diff --git a/.agents/skills/codex-review-loop/SKILL.md b/.agents/skills/codex-review-loop/SKILL.md index 03942cea3c..96eb8ac3d7 100644 --- a/.agents/skills/codex-review-loop/SKILL.md +++ b/.agents/skills/codex-review-loop/SKILL.md @@ -59,7 +59,13 @@ Launch the adversarial review as a background process. cat | bash /scripts/codex-subagent.sh --uncommitted ``` 2. Inform the user the review is running (~20-50 min). -3. The script parses Codex output and returns the final review text. +3. The script parses Codex output and returns the final review text. Review + rollouts intentionally remain persistent. If terminal output is lost, use + `codex resume --include-non-interactive` to locate the review, or + `codex resume ` when its ID is known. Do not add + `--ephemeral` to the wrapper. The wrapper also relies on the configured + non-interactive approval/sandbox policy because Codex CLI 0.147.0 removed + the historical `--full-auto` argument from `exec review`. ### Error handling diff --git a/.agents/skills/codex-review-loop/scripts/codex-subagent.sh b/.agents/skills/codex-review-loop/scripts/codex-subagent.sh index 4cbd93e29f..01bf32f68b 100755 --- a/.agents/skills/codex-review-loop/scripts/codex-subagent.sh +++ b/.agents/skills/codex-review-loop/scripts/codex-subagent.sh @@ -33,7 +33,11 @@ while [[ $# -gt 0 ]]; do esac done -CODEX_ARGS+=("--full-auto" "--ephemeral") +# Reviews commonly outlive the invoking terminal or exceed its output cap, so +# do not add `--ephemeral`: the persistent rollout is the recovery path after +# either failure. Also do not restore the historical `--full-auto` argument; +# Codex CLI 0.147.0 removed it from `exec review`, which already uses the +# configured non-interactive approval and sandbox policy. # --- Model overrides --- if [[ -n "${CODEX_REVIEW_MODEL:-}" ]]; then diff --git a/.all-contributorsrc b/.all-contributorsrc index f8898e5f97..06a87f0397 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -1223,6 +1223,96 @@ "code" ] }, + { + "login": "kidclone3", + "name": "DuyBui", + "avatar_url": "https://avatars.githubusercontent.com/u/54184969?v=4", + "profile": "https://github.com/kidclone3", + "contributions": [ + "code", + "test" + ] + }, + { + "login": "kevinsslin", + "name": "Kevin Lin", + "avatar_url": "https://avatars.githubusercontent.com/u/86810837?v=4", + "profile": "https://github.com/kevinsslin", + "contributions": [ + "code", + "test" + ] + }, + { + "login": "Borealin", + "name": "Borealin", + "avatar_url": "https://avatars.githubusercontent.com/u/41241077?v=4", + "profile": "https://github.com/Borealin", + "contributions": [ + "code", + "test" + ] + }, + { + "login": "BrenticusMaximus", + "name": "BrenticusMaximus", + "avatar_url": "https://avatars.githubusercontent.com/u/32489248?v=4", + "profile": "https://github.com/BrenticusMaximus", + "contributions": [ + "code", + "test" + ] + }, + { + "login": "sakthimaran-venom", + "name": "Sakthimaran", + "avatar_url": "https://avatars.githubusercontent.com/u/233523816?v=4", + "profile": "https://github.com/sakthimaran-venom", + "contributions": [ + "code", + "test" + ] + }, + { + "login": "evan-choi", + "name": "Evan", + "avatar_url": "https://avatars.githubusercontent.com/u/9690415?v=4", + "profile": "https://github.com/evan-choi", + "contributions": [ + "code" + ] + }, + { + "login": "chaoxu", + "name": "Chao Xu", + "avatar_url": "https://avatars.githubusercontent.com/u/18860?v=4", + "profile": "https://chaoxu.prof/", + "contributions": [ + "code", + "test" + ] + }, + { + "login": "zenasharp", + "name": "zenasharp", + "avatar_url": "https://avatars.githubusercontent.com/u/170236008?v=4", + "profile": "https://github.com/zenasharp", + "contributions": [ + "code", + "test", + "doc" + ] + }, + { + "login": "hanseo0507", + "name": "HanSu Lee", + "avatar_url": "https://avatars.githubusercontent.com/u/56479293?v=4", + "profile": "https://github.com/hanseo0507", + "contributions": [ + "code", + "test" + ] + }, { "login": "hongzexin", "name": "Jason HONG", @@ -1231,7 +1321,7 @@ "contributions": [ "code", "test", - "doc" + "maintenance" ] } ], diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5b126ab8af..af3f057368 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -193,7 +193,7 @@ PR titles must follow the same format — that's the title release-please reads. 2. Make atomic commits with Conventional Commit titles. 3. Run the lint/test gate locally (see above). 4. Open a PR using the template. Link the relevant issue. -5. Codex Review (and a human maintainer) will review. Address feedback by +5. CodeRabbit (and a human maintainer) will review. Address feedback by pushing follow-up commits — no force-pushing during active review. 6. Once approved and CI is green, a maintainer squash-merges with a clean Conventional Commits title. @@ -215,16 +215,11 @@ Before a PR is squash-merged into `main`: `CI Required` check is the branch-protection check to require: it depends on every CI job and also runs for merge queue synthetic merge groups, so a stale PR head cannot bypass a broken merge result. -2. **`@codex review` must be clean — or its findings addressed — on the - merge-target head.** Every PR triggers `@codex review` at least once - against the head that's about to be merged. Local `codex review - --base origin/main` runs are encouraged but don't substitute for the - cloud review (the cloud `@codex review` reliably catches things the - local run misses). - The `🤖 codex: ok` label is maintained by the trusted - `Codex review labels` workflow from current-head CI and current-head - Codex review evidence. Treat the label as an audit aid, not as a - substitute for branch protection or merge queue checks. +2. **Actionable CodeRabbit findings must be fixed or explicitly addressed + or dismissed in-thread on the merge-target head.** Review the current-head + CodeRabbit findings before merging; no finding may be silently skipped. + Local `codex review --base origin/main` runs remain an encouraged extra + tool, but they are not a merge gate and do not substitute for CodeRabbit. - **P1 findings**: fix in the PR, or justify in-thread with a short write-up of why the finding doesn't apply. No silent skipping. - **P2 findings**: fix in the PR, or open a follow-up issue and link @@ -297,7 +292,7 @@ self-merge escape hatch applies: - If a collaborator's PR has been waiting on a maintainer merge for **more than 14 days** with **all merge gates met** (CI green, - `@codex review` clean or findings addressed, `mergeable=CLEAN`, no + CodeRabbit findings addressed, `mergeable=CLEAN`, no outstanding requested-changes review, no objection from any other active collaborator in the thread), the PR author may self-merge. - Self-merge under this clause **must** include a comment on the PR @@ -311,8 +306,8 @@ self-merge escape hatch applies: These rules are intentionally lightweight. They don't require: -- A second human reviewer in addition to `@codex review` for every PR. - Codex review + the PR author + a maintainer merge is the baseline. +- A second human reviewer in addition to CodeRabbit for every PR. + CodeRabbit review + the PR author + a maintainer merge is the baseline. - Squash-merge commit message rewriting beyond the Conventional Commits title. The PR description ends up in the body; that's enough. - A formal escalation process for disagreements. If a P1 finding is diff --git a/.github/release-please-manifest.json b/.github/release-please-manifest.json index c41415c5ec..4d625b2f88 100644 --- a/.github/release-please-manifest.json +++ b/.github/release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.23.0" + ".": "1.24.0" } diff --git a/.github/scripts/check_simplicity_budgets.py b/.github/scripts/check_simplicity_budgets.py index 51d22c8933..e688013261 100644 --- a/.github/scripts/check_simplicity_budgets.py +++ b/.github/scripts/check_simplicity_budgets.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -"""Enforce simplicity budgets on README, .env.example, and the dashboard core nav. +"""Enforce simplicity budgets on README, .env.example, the dashboard core nav, and the tracked root tree. Budgets live in .github/simplicity-budgets.toml and are enforced by .github/workflows/simplicity-budgets.yml. Intentionally stdlib-only so it runs -on the runner's python3 before project dependencies are installed. +on the runner's python3 before project dependencies are installed; the +[root_files] check additionally shells out to `git ls-tree` against the +checkout's HEAD. Override: the 'simplicity-budget-approved' PR label (passed in via the PR_LABELS env var as a JSON array of label names) downgrades violations to @@ -12,8 +14,8 @@ Exit codes: 0 = within budget (or overridden), 1 = over budget, 2 = configuration error (the budget config is missing or malformed, a -budgeted file or the nav array is missing, or an ALL-CONTRIBUTORS-LIST -block is opened but never closed). +budgeted file or the nav array is missing, the tracked root tree cannot +be listed, or an ALL-CONTRIBUTORS-LIST block is opened but never closed). """ from __future__ import annotations @@ -21,6 +23,7 @@ import json import os import re +import subprocess import sys import tomllib from pathlib import Path @@ -129,6 +132,34 @@ def count_nav_items(path: Path, array: str) -> int: return len(re.findall(r"\bto:\s*[\"']", match.group("body"))) +def _escape_annotation_value(value: str) -> str: + """Escape a contributor-controlled value for a workflow-command line ('%' first, per Actions rules).""" + for char, escape in (("%", "%25"), ("\r", "%0D"), ("\n", "%0A"), (":", "%3A"), (",", "%2C")): + value = value.replace(char, escape) + return value + + +def list_tracked_root_entries() -> list[str]: + """List tracked repository-root entries from HEAD; exit 2 loudly if git cannot.""" + try: + proc = subprocess.run( + ["git", "ls-tree", "--name-only", "-z", "HEAD"], + capture_output=True, + encoding="utf-8", + # Non-UTF-8 filename bytes become \x escapes: they can never match + # an allowlist entry, so they surface as a named violation instead + # of a decode crash. + errors="backslashreplace", + check=True, + ) + except FileNotFoundError: + _config_error("[root_files] git executable not found; the root-entry budget needs a git checkout") + except subprocess.CalledProcessError as exc: + detail = (exc.stderr or "").strip() or f"exit code {exc.returncode}" + _config_error(f"[root_files] 'git ls-tree --name-only HEAD' failed: {detail}") + return [entry for entry in proc.stdout.split("\0") if entry] + + def _override_labels() -> list[str]: raw = os.environ.get("PR_LABELS") or "[]" try: @@ -164,9 +195,26 @@ def main() -> int: except (KeyError, TypeError, ValueError) as exc: _config_error(f"budget config '{CONFIG_PATH}' is missing or has a malformed section/key: {exc!r}") + # [root_files] is optional: absent means the root-entry budget is not + # enforced (older configs keep working), present-but-malformed is a + # config error like any other section. + root_allowed: set[str] | None = None + root_cfg = config.get("root_files") + if root_cfg is not None: + try: + allowed_entries = root_cfg["allowed"] + except (KeyError, TypeError) as exc: + _config_error(f"budget config '{CONFIG_PATH}' has a malformed [root_files] section: {exc!r}") + if not isinstance(allowed_entries, list) or not all(isinstance(entry, str) for entry in allowed_entries): + _config_error(f"budget config '{CONFIG_PATH}' [root_files] 'allowed' must be an array of strings") + root_allowed = set(allowed_entries) + readme_lines = strip_contributors_block(_read_lines(readme_path, "readme")) env_lines = _read_lines(env_path, "env_example") nav_items = count_nav_items(nav_path, nav_array) + unexpected_root_entries: list[str] = [] + if root_allowed is not None: + unexpected_root_entries = sorted(set(list_tracked_root_entries()) - root_allowed) metrics: list[tuple[str, Path, int, int]] = [ ( @@ -192,12 +240,25 @@ def main() -> int: if actual > budget: violations.append((name, path, actual, budget)) - if not violations: + if root_allowed is not None: + status = "OK" if not unexpected_root_entries else "OVER" + print(f"tracked root entries outside allowlist: {len(unexpected_root_entries)}/0 {status}") + + if not violations and not unexpected_root_entries: return 0 annotation = "warning" if overridden else "error" for name, path, actual, budget in violations: print(f"::{annotation} file={path}::simplicity budget exceeded: {name}: {actual} > {budget}") + for entry in unexpected_root_entries: + # Entry names come from the tree, not the trusted config: escape them + # so a crafted filename cannot break or forge workflow-command lines. + shown = _escape_annotation_value(entry) + print( + f"::{annotation} file={shown}::simplicity budget exceeded: tracked root entry '{shown}' is not in " + f"the [root_files] allowlist — add it to {CONFIG_PATH} in the same diff, or a maintainer applies " + f"the '{OVERRIDE_LABEL}' PR label" + ) if overridden: print(f"Budgets exceeded, but the '{OVERRIDE_LABEL}' label is applied; passing with warnings. {OVERRIDE_HELP}") diff --git a/.github/scripts/sync_codex_ok_labels.py b/.github/scripts/sync_codex_ok_labels.py deleted file mode 100755 index 17b3c1a275..0000000000 --- a/.github/scripts/sync_codex_ok_labels.py +++ /dev/null @@ -1,1910 +0,0 @@ -#!/usr/bin/env python3 -"""Synchronize GitHub Codex review labels from current-head Codex reviews.""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import subprocess -import sys -import time -from collections.abc import Callable -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from typing import Any -from urllib.parse import quote - -CODEX_OK_LABEL = "🤖 codex: ok" -CODEX_NEEDS_WORK_LABEL = "🤖 codex: needs work" -NEEDS_REBASE_LABEL = "needs rebase" -LEGACY_CODEX_LABELS = {"🤖 codex-ok"} -CODEX_REVIEW_AUTHORS = { - "chatgpt-codex-connector", - "chatgpt-codex-connector[bot]", - "openai-codex", - "openai-codex[bot]", -} -CODEX_CLEAN_RE = re.compile( - r"(didn['’]t find any major issues|no major issues found|no major issues)", - re.IGNORECASE, -) -CODEX_FINDING_RE = re.compile(r"(?:\bP[0-3]\s+Badge\b|badge/P[0-3]-|(?m:(?:^|\n)\s*(?:\*\*)?(?:\[P[0-3]\]|P[0-3]\b)))") -# Anchored to the real quota envelope ("You have reached your Codex usage limits -# for code reviews. ...") so ordinary reviews that merely discuss usage limits do -# not latch the backoff. -CODEX_USAGE_LIMIT_RE = re.compile( - r"^\s*You(?: have|['’]ve) reached your Codex usage limits", - re.IGNORECASE, -) -CLEAN_REACTION_CONTENTS = frozenset({"THUMBS_UP", "+1"}) -DEFAULT_CODEX_USAGE_LIMIT_BACKOFF_HOURS = 24.0 -DEFAULT_CODEX_REVIEW_RESPONSE_WAIT_SECONDS = 10.0 -SUCCESS_CHECK_STATES = {"SUCCESS", "NEUTRAL", "SKIPPED"} -FAIL_CHECK_STATES = {"ACTION_REQUIRED", "CANCELLED", "ERROR", "FAILURE", "STALE", "TIMED_OUT"} -PENDING_CHECK_STATES = {"EXPECTED", "IN_PROGRESS", "PENDING", "QUEUED", "REQUESTED", "WAITING"} -UNMERGEABLE_STATES = {"DIRTY", "BLOCKED"} -NEEDS_REBASE_STATES = {"CONFLICTING", "DIRTY"} -NO_REBASE_STATES = {"BEHIND", "BLOCKED", "CLEAN", "DRAFT", "HAS_HOOKS", "UNSTABLE"} -CODEX_LB_REQUIRED_CHECKS = frozenset( - { - "Frontend lint (eslint)", - "Frontend type check (tsc)", - "Frontend tests (vitest + coverage)", - "Frontend build (vite)", - "Lint (ruff)", - "Type check (ty)", - "Tests (pytest, unit)", - "Tests (pytest, integration-core)", - "Tests (pytest, integration-bridge)", - "Tests (pytest, e2e)", - "Tests (pytest, PostgreSQL)", - "Migration check (alembic)", - "Migration check (alembic, PostgreSQL)", - "Package (build)", - "Docker build", - "Helm lint + template + kubeconform", - "Helm smoke install (kind)", - "CI Required", - } -) -REQUIRED_CHECKS_BY_REPO = { - "Soju06/codex-lb": CODEX_LB_REQUIRED_CHECKS, -} -PR_TIMELINE_QUERY = """ -query($owner: String!, $name: String!, $number: Int!, $before: String) { - repository(owner: $owner, name: $name) { - pullRequest(number: $number) { - headRefOid - commits(last: 1) { - nodes { - commit { - oid - } - } - } - timelineItems( - last: 100 - before: $before - itemTypes: [ - PULL_REQUEST_COMMIT - ISSUE_COMMENT - PULL_REQUEST_REVIEW - HEAD_REF_FORCE_PUSHED_EVENT - ] - ) { - pageInfo { - hasPreviousPage - startCursor - } - nodes { - __typename - ... on PullRequestCommit { - commit { - oid - } - } - ... on HeadRefForcePushedEvent { - afterCommit { - oid - } - } - ... on IssueComment { - author { - login - } - bodyText - createdAt - url - reactions(first: 100) { - nodes { - content - createdAt - user { - login - } - } - } - } - ... on PullRequestReview { - databaseId - author { - login - } - bodyText - submittedAt - url - commit { - oid - } - } - } - } - } - } -} -""" - -PR_REVIEW_THREADS_QUERY = """ -query($owner: String!, $name: String!, $number: Int!, $after: String) { - repository(owner: $owner, name: $name) { - pullRequest(number: $number) { - reviewThreads(first: 100, after: $after) { - pageInfo { - hasNextPage - endCursor - } - nodes { - isResolved - isOutdated - comments(first: 20) { - nodes { - author { - login - } - body - url - commit { - oid - } - originalCommit { - oid - } - } - } - } - } - } - } -} -""" - - -class GhError(RuntimeError): - """A GitHub CLI call failed.""" - - -_RATE_LIMIT_MARKER = "API rate limit exceeded" -_TRANSIENT_GH_MARKERS = ("HTTP 500", "HTTP 502", "HTTP 503", "HTTP 504", "Unicorn!") -_GH_RETRY_ATTEMPTS = 5 -_GH_RETRY_BASE_DELAY_SECONDS = 2.0 -_fallback_token_active = False - - -def _activate_fallback_token() -> bool: - """Switch gh calls to GH_FALLBACK_TOKEN once after a rate-limit failure. - - The primary token is typically a user PAT whose quota is shared with - other consumers; github.token carries a separate per-repository quota, - so a single runtime switch keeps the sync alive through PAT exhaustion. - """ - global _fallback_token_active - if _fallback_token_active: - return False - fallback = os.environ.get("GH_FALLBACK_TOKEN", "").strip() - if not fallback or fallback == os.environ.get("GH_TOKEN", ""): - return False - os.environ["GH_TOKEN"] = fallback - _fallback_token_active = True - return True - - -def _gh_args_safe_to_retry(args: list[str]) -> bool: - """Return whether a failed gh invocation is safe to re-run automatically.""" - - if not args or args[0] != "api": - return False - if "graphql" in args: - return True - method = "GET" - for index, arg in enumerate(args): - if arg == "--method" and index + 1 < len(args): - method = args[index + 1].upper() - break - return method == "GET" - - -def _is_retryable_gh_failure(detail: str) -> bool: - return any(marker in detail for marker in _TRANSIENT_GH_MARKERS) - - -def _gh_retry_delay(attempt_index: int) -> float: - return min(_GH_RETRY_BASE_DELAY_SECONDS * (2**attempt_index), 30.0) - - -@dataclass(frozen=True) -class SyncDecision: - repo: str - number: int - head_sha: str - has_ok_label: bool - wants_ok_label: bool - ok_action: str - has_needs_work_label: bool - wants_needs_work_label: bool - needs_work_action: str - has_needs_rebase_label: bool - wants_needs_rebase_label: bool - needs_rebase_action: str - legacy_labels: frozenset[str] - reason: str - review_url: str | None - review_state: str - checks_state: str - merge_state: str - trigger_codex_review: bool - approve_workflow_run_ids: tuple[int, ...] - - -def run_gh( - args: list[str], - *, - input_json: Any | None = None, - timeout_seconds: int = 30, - fallback_retry: bool = True, -) -> Any: - command = ["gh", *args] - input_text = json.dumps(input_json) if input_json is not None else None - safe_to_retry = _gh_args_safe_to_retry(args) - attempts = _GH_RETRY_ATTEMPTS if safe_to_retry else 1 - for attempt_index in range(attempts): - try: - proc = subprocess.run( - command, - check=False, - input=input_text, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - timeout=timeout_seconds, - ) - except subprocess.TimeoutExpired as exc: - raise GhError(f"{' '.join(command)}: timed out after {timeout_seconds}s") from exc - - if proc.returncode == 0: - text = proc.stdout.strip() - if not text: - return None - return json.loads(text) - - detail = proc.stderr.strip() or proc.stdout.strip() - if _RATE_LIMIT_MARKER in detail and _activate_fallback_token(): - if not fallback_retry: - # Identity-sensitive commands (e.g. the @codex review comment) - # must not silently retry under the fallback token's identity; - # later calls still benefit from the activated fallback. - raise GhError( - f"{' '.join(command)}: rate-limited; switched to GH_FALLBACK_TOKEN " - f"without retrying this identity-sensitive command ({detail})" - ) - print( - "warning: active token rate-limited; retrying with GH_FALLBACK_TOKEN", - file=sys.stderr, - ) - return run_gh(args, input_json=input_json, timeout_seconds=timeout_seconds) - if safe_to_retry and _is_retryable_gh_failure(detail) and attempt_index + 1 < attempts: - delay = _gh_retry_delay(attempt_index) - print( - f"warning: {' '.join(command)} failed transiently ({detail}); retrying in {delay:g}s", - file=sys.stderr, - ) - time.sleep(delay) - continue - raise GhError(f"{' '.join(command)}: {detail}") - - raise GhError(f"{' '.join(command)}: failed after retries") - - -def gh_api(path: str, *, method: str = "GET", input_json: Any | None = None) -> Any: - if not path.startswith("/"): - path = f"/{path}" - - args = ["api", "--method", method, path] - if input_json is not None: - args.append("--input") - args.append("-") - return run_gh(args, input_json=input_json) - - -def graphql(query: str, **fields: object) -> dict[str, Any]: - args = ["api", "graphql", "-f", f"query={query}"] - for key, value in fields.items(): - args.extend(["-F", f"{key}={value}"]) - payload = run_gh(args) - if not isinstance(payload, dict): - raise GhError("gh api graphql returned a non-object payload") - return payload - - -def paged_api(path: str) -> list[dict[str, Any]]: - page = 1 - items: list[dict[str, Any]] = [] - sep = "&" if "?" in path else "?" - while True: - payload = gh_api(f"{path}{sep}per_page=100&page={page}") - if not payload: - return items - if isinstance(payload, dict): - page_items = None - for key in ("items", "check_runs", "workflow_runs"): - if key in payload: - page_items = payload[key] - break - else: - page_items = payload - if not isinstance(page_items, list): - raise GhError(f"{path}: expected list payload, got {type(payload).__name__}") - items.extend(item for item in page_items if isinstance(item, dict)) - if len(page_items) < 100: - return items - page += 1 - - -def repo_path(repo: str) -> str: - parts = repo.strip().split("/") - if len(parts) != 2 or not all(parts): - raise ValueError(f"repo must be owner/name, got {repo!r}") - return f"{parts[0]}/{parts[1]}" - - -def list_open_pr_numbers(repo: str) -> list[int]: - pulls = paged_api(f"/repos/{repo}/pulls?state=open") - return [int(pr["number"]) for pr in pulls if isinstance(pr.get("number"), int)] - - -def issue_label_names(repo: str, number: int) -> set[str]: - labels = paged_api(f"/repos/{repo}/issues/{number}/labels") - return {str(label.get("name")) for label in labels if isinstance(label.get("name"), str)} - - -def author_login(item: dict[str, Any]) -> str: - user = item.get("user") - login = user.get("login") if isinstance(user, dict) else None - return str(login or "") - - -def is_clean_codex_body(body: object) -> bool: - return isinstance(body, str) and CODEX_CLEAN_RE.search(body) is not None - - -def is_needs_work_codex_body(body: object) -> bool: - return isinstance(body, str) and CODEX_FINDING_RE.search(body) is not None - - -def is_codex_usage_limit_body(body: object) -> bool: - return isinstance(body, str) and CODEX_USAGE_LIMIT_RE.search(body) is not None - - -def body_mentions_head(body: object, head_sha: str) -> bool: - if not isinstance(body, str): - return False - return head_sha in body or head_sha[:12] in body or head_sha[:8] in body - - -def review_node_commit_oid(node: dict[str, Any]) -> str | None: - commit = node.get("commit") - oid = commit.get("oid") if isinstance(commit, dict) else None - return oid if isinstance(oid, str) else None - - -def review_node_database_id(node: dict[str, Any]) -> int | None: - value = node.get("databaseId") - return value if isinstance(value, int) else None - - -def node_body(node: dict[str, Any]) -> str: - body = node.get("bodyText") - if not isinstance(body, str): - body = node.get("body") - return body if isinstance(body, str) else "" - - -def node_url(node: dict[str, Any]) -> str | None: - url = node.get("url") or node.get("html_url") - return str(url) if isinstance(url, str) else None - - -def node_author_login(node: dict[str, Any]) -> str: - author = node.get("author") - login = author.get("login") if isinstance(author, dict) else None - if isinstance(login, str): - return login - return author_login(node) - - -def is_timeline_codex_author(node: dict[str, Any], allowed: set[str]) -> bool: - return node_author_login(node) in allowed - - -def is_codex_review_request_comment(node: dict[str, Any]) -> bool: - if node.get("__typename") != "IssueComment": - return False - return node_body(node).strip().casefold() == "@codex review" - - -def reaction_user_login(node: dict[str, Any]) -> str: - user = node.get("user") - login = user.get("login") if isinstance(user, dict) else None - return str(login or "") - - -def codex_request_reaction_state(node: dict[str, Any], *, allowed_authors: set[str]) -> str: - if not is_codex_review_request_comment(node): - return "none" - - reactions = node.get("reactions") - reaction_nodes = reactions.get("nodes") if isinstance(reactions, dict) else [] - if not isinstance(reaction_nodes, list): - return "none" - - state = "none" - for reaction in reaction_nodes: - if not isinstance(reaction, dict): - continue - if reaction_user_login(reaction) not in allowed_authors: - continue - content = str(reaction.get("content") or "").upper() - if content in CLEAN_REACTION_CONTENTS: - state = "clean" - elif content == "EYES" and state == "none": - state = "pending" - return state - - -def timeline_head_oid(node: dict[str, Any]) -> str | None: - if node.get("__typename") == "PullRequestCommit": - commit = node.get("commit") - oid = commit.get("oid") if isinstance(commit, dict) else None - return oid if isinstance(oid, str) else None - if node.get("__typename") == "HeadRefForcePushedEvent": - commit = node.get("afterCommit") - oid = commit.get("oid") if isinstance(commit, dict) else None - return oid if isinstance(oid, str) else None - return None - - -def timeline_node_timestamp(node: dict[str, Any]) -> str | None: - for key in ("createdAt", "submittedAt", "committedDate"): - value = node.get(key) - if isinstance(value, str) and value: - return value - return None - - -def parse_github_timestamp(value: object) -> datetime | None: - if not isinstance(value, str) or not value: - return None - try: - return datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError: - return None - - -def current_viewer_login() -> str: - payload = gh_api("/user") - login = payload.get("login") if isinstance(payload, dict) else None - if not isinstance(login, str) or not login: - raise GhError("gh api /user did not return a login") - return login - - -def resolve_codex_request_sender() -> str | None: - """Resolve the login GitHub records as the author of our `@codex review` comments. - - GitHub App installation tokens cannot call ``GET /user``, so prefer the app - slug the workflow exports (installation comments are authored as - ``[bot]``) and fall back to ``GET /user`` for PAT-backed runs. - The slug describes the primary token only, so it is ignored once the run - has switched to ``GH_FALLBACK_TOKEN``. Returns ``None`` when no path can - resolve a login. - """ - - slug = os.environ.get("GH_APP_SLUG", "").strip() - if slug and not _fallback_token_active: - return slug if slug.endswith("[bot]") else f"{slug}[bot]" - try: - return current_viewer_login() - except GhError as exc: - print(f"warning: cannot resolve @codex review sender via GET /user: {exc}", file=sys.stderr, flush=True) - return None - - -def is_normal_codex_response_node(node: dict[str, Any]) -> bool: - body = node_body(node) - if is_codex_usage_limit_body(body): - return False - return node.get("__typename") in {"IssueComment", "PullRequestReview", "PullRequestReviewComment"} and bool( - body.strip() - ) - - -@dataclass -class CodexReviewUsageBackoff: - request_author: str - allowed_authors: set[str] - window: timedelta - now: datetime - latest_usage_limit_at: datetime | None = None - latest_usage_limit_url: str | None = None - latest_normal_response_at: datetime | None = None - - def observe(self, timeline_nodes: list[dict[str, Any]]) -> None: - active_request_author: str | None = None - for node in timeline_nodes: - timestamp = parse_github_timestamp(timeline_node_timestamp(node)) - if is_codex_review_request_comment(node): - active_request_author = node_author_login(node) - if active_request_author == self.request_author: - self._observe_clean_request_reactions(node) - continue - if timestamp is None or timestamp < self.now - self.window: - continue - if active_request_author != self.request_author: - continue - if not is_timeline_codex_author(node, self.allowed_authors): - continue - if is_codex_usage_limit_body(node_body(node)): - if self.latest_usage_limit_at is None or timestamp > self.latest_usage_limit_at: - self.latest_usage_limit_at = timestamp - self.latest_usage_limit_url = node_url(node) - elif is_normal_codex_response_node(node): - if self.latest_normal_response_at is None or timestamp > self.latest_normal_response_at: - self.latest_normal_response_at = timestamp - - def _observe_clean_request_reactions(self, node: dict[str, Any]) -> None: - """Count clean THUMBS_UP reactions on the sender's requests as normal responses.""" - - reactions = node.get("reactions") - reaction_nodes = reactions.get("nodes") if isinstance(reactions, dict) else None - if not isinstance(reaction_nodes, list): - return - for reaction in reaction_nodes: - if not isinstance(reaction, dict): - continue - if reaction_user_login(reaction) not in self.allowed_authors: - continue - if str(reaction.get("content") or "").upper() not in CLEAN_REACTION_CONTENTS: - continue - timestamp = parse_github_timestamp(reaction.get("createdAt")) - if timestamp is None or timestamp < self.now - self.window: - continue - if self.latest_normal_response_at is None or timestamp > self.latest_normal_response_at: - self.latest_normal_response_at = timestamp - - def is_limited(self) -> bool: - if self.latest_usage_limit_at is None: - return False - return self.latest_normal_response_at is None or self.latest_normal_response_at < self.latest_usage_limit_at - - def skip_warning(self, decision: SyncDecision) -> str: - when = self.latest_usage_limit_at.isoformat() if self.latest_usage_limit_at else "unknown time" - suffix = f" ({self.latest_usage_limit_url})" if self.latest_usage_limit_url else "" - return ( - f"request Codex review on {decision.repo}#{decision.number}: skipped because " - f"{self.request_author} has a recent Codex usage-limit reply at {when}{suffix}" - ) - - -def recent_issue_comment_timelines(repo: str, *, since: datetime) -> list[list[dict[str, Any]]]: - """Return repo-wide recent issue comments grouped per issue as timeline nodes. - - Sender quota evidence can live on pull requests outside the current - selection (single --pr runs, closed PRs), so the backoff also observes - every issue comment in the window, grouped per issue to keep request -> - reply attribution intact. - """ - - since_text = since.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") - comments = paged_api(f"/repos/{repo}/issues/comments?since={quote(since_text, safe='')}") - nodes_by_issue: dict[str, list[dict[str, Any]]] = {} - for comment in comments: - body = comment.get("body") - if not isinstance(body, str): - continue - issue_url = str(comment.get("issue_url") or "") - nodes_by_issue.setdefault(issue_url, []).append( - { - "__typename": "IssueComment", - "author": {"login": author_login(comment)}, - "bodyText": body, - "createdAt": comment.get("created_at"), - "url": comment.get("html_url"), - } - ) - return [sorted(nodes, key=lambda node: str(node.get("createdAt") or "")) for nodes in nodes_by_issue.values()] - - -def backoff_timeline_observer(backoff: CodexReviewUsageBackoff) -> Callable[[int, list[dict[str, Any]]], None]: - def observe(_number: int, timeline_nodes: list[dict[str, Any]]) -> None: - backoff.observe(timeline_nodes) - - return observe - - -def unresolved_review_comment_urls(repo: str, number: int) -> set[str]: - owner, name = repo.split("/", 1) - after: str | None = None - urls: set[str] = set() - - while True: - fields: dict[str, object] = {"owner": owner, "name": name, "number": number} - if after is not None: - fields["after"] = after - payload = graphql(PR_REVIEW_THREADS_QUERY, **fields) - pr = payload.get("data", {}).get("repository", {}).get("pullRequest") - if not isinstance(pr, dict): - raise GhError(f"{repo}#{number}: GraphQL did not return a pull request") - - threads = pr.get("reviewThreads") - if not isinstance(threads, dict): - raise GhError(f"{repo}#{number}: GraphQL did not return review threads") - nodes = threads.get("nodes", []) - if not isinstance(nodes, list): - raise GhError(f"{repo}#{number}: GraphQL did not return review thread nodes") - - for thread in nodes: - if not isinstance(thread, dict): - continue - if thread.get("isResolved") or thread.get("isOutdated"): - continue - comments = thread.get("comments") - comment_nodes = comments.get("nodes") if isinstance(comments, dict) else [] - if not isinstance(comment_nodes, list): - continue - for comment in comment_nodes: - if not isinstance(comment, dict): - continue - url = comment.get("url") - if isinstance(url, str): - urls.add(url) - - page_info = threads.get("pageInfo") - if not isinstance(page_info, dict) or not page_info.get("hasNextPage"): - break - end_cursor = page_info.get("endCursor") - if not isinstance(end_cursor, str) or not end_cursor: - break - after = end_cursor - - return urls - - -def pull_review_comment_nodes(repo: str, number: int, *, head_sha: str) -> list[dict[str, Any]]: - comments = paged_api(f"/repos/{repo}/pulls/{number}/comments") - unresolved_urls = unresolved_review_comment_urls(repo, number) - nodes: list[dict[str, Any]] = [] - for comment in comments: - body = comment.get("body") - if not isinstance(body, str): - continue - commit_id = comment.get("commit_id") - original_commit_id = comment.get("original_commit_id") - review_id = comment.get("pull_request_review_id") - commit_matches_head = commit_id == head_sha - original_matches_head = original_commit_id == head_sha - body_mentions_current_head = body_mentions_head(body, head_sha) - if not commit_matches_head and not original_matches_head and not body_mentions_current_head: - continue - effective_commit_id = original_commit_id if original_matches_head else commit_id - effective_review_id = review_id if original_matches_head else None - html_url = comment.get("html_url") or comment.get("url") - if is_needs_work_codex_body(body) and html_url not in unresolved_urls: - continue - user = comment.get("user") - login = user.get("login") if isinstance(user, dict) else None - nodes.append( - { - "__typename": "PullRequestReviewComment", - "author": {"login": login} if isinstance(login, str) else None, - "bodyText": body, - "createdAt": comment.get("created_at"), - "url": html_url, - "commit": {"oid": effective_commit_id} if isinstance(effective_commit_id, str) else None, - "pullRequestReviewDatabaseId": effective_review_id if isinstance(effective_review_id, int) else None, - } - ) - return nodes - - -def merge_review_comment_nodes( - timeline_nodes: list[dict[str, Any]], - comment_nodes: list[dict[str, Any]], -) -> list[dict[str, Any]]: - if not comment_nodes: - return timeline_nodes - - comments_by_review_id: dict[int, list[dict[str, Any]]] = {} - unplaced: list[dict[str, Any]] = [] - for node in comment_nodes: - review_id = node.get("pullRequestReviewDatabaseId") - if isinstance(review_id, int): - comments_by_review_id.setdefault(review_id, []).append(node) - else: - unplaced.append(node) - - merged: list[dict[str, Any]] = [] - placed_ids: set[int] = set() - for node in timeline_nodes: - merged.append(node) - if node.get("__typename") != "PullRequestReview": - continue - review_id = review_node_database_id(node) - if review_id is None: - continue - for comment in comments_by_review_id.get(review_id, []): - merged.append(comment) - placed_ids.add(id(comment)) - - for node in comment_nodes: - if id(node) not in placed_ids and node not in unplaced: - unplaced.append(node) - for node in unplaced: - timestamp = timeline_node_timestamp(node) - if timestamp is None: - merged.append(node) - continue - insert_at = len(merged) - for index, candidate in enumerate(merged): - candidate_timestamp = timeline_node_timestamp(candidate) - if candidate_timestamp is not None and candidate_timestamp > timestamp: - insert_at = index - break - merged.insert(insert_at, node) - return merged - - -def find_current_head_codex_review_state( - timeline_nodes: list[dict[str, Any]], - *, - head_sha: str, - allowed_authors: set[str], -) -> tuple[str, dict[str, Any] | None]: - head_index = None - for index, node in enumerate(timeline_nodes): - if timeline_head_oid(node) == head_sha: - head_index = index - - if head_index is None: - return "none", None - - latest_state = "none" - latest_node: dict[str, Any] | None = None - for node in timeline_nodes[head_index + 1 :]: - reaction_state = codex_request_reaction_state(node, allowed_authors=allowed_authors) - if reaction_state == "clean": - latest_state = "clean" - latest_node = node - continue - if reaction_state == "pending" and latest_state == "none": - latest_state = "pending" - latest_node = node - continue - - if not is_timeline_codex_author(node, allowed_authors): - continue - - if node.get("__typename") == "PullRequestReview": - body = node_body(node) - commit_oid = review_node_commit_oid(node) - if commit_oid != head_sha and not body_mentions_head(body, head_sha): - continue - if is_needs_work_codex_body(body): - latest_state = "needs_work" - latest_node = node - continue - latest_state = "clean" - latest_node = node - continue - - if node.get("__typename") == "PullRequestReviewComment": - body = node_body(node) - commit_oid = review_node_commit_oid(node) - if commit_oid != head_sha and not body_mentions_head(body, head_sha): - continue - if is_needs_work_codex_body(body): - latest_state = "needs_work" - latest_node = node - continue - - if node.get("__typename") == "IssueComment" and is_clean_codex_body(node_body(node)): - latest_state = "clean" - latest_node = node - - return latest_state, latest_node - - -def find_current_head_clean_review( - timeline_nodes: list[dict[str, Any]], - *, - head_sha: str, - allowed_authors: set[str], -) -> dict[str, Any] | None: - state, node = find_current_head_codex_review_state( - timeline_nodes, - head_sha=head_sha, - allowed_authors=allowed_authors, - ) - return node if state == "clean" else None - - -def has_codex_news_after_current_head( - timeline_nodes: list[dict[str, Any]], - *, - head_sha: str, - allowed_authors: set[str], -) -> bool: - head_index = None - for index, node in enumerate(timeline_nodes): - if timeline_head_oid(node) == head_sha: - head_index = index - - if head_index is None: - return False - - for node in timeline_nodes[head_index + 1 :]: - if is_codex_review_request_comment(node): - return True - if not is_timeline_codex_author(node, allowed_authors): - continue - if node.get("__typename") == "PullRequestReview": - body = node_body(node) - commit_oid = review_node_commit_oid(node) - if commit_oid != head_sha and not body_mentions_head(body, head_sha): - continue - return True - if node.get("__typename") == "PullRequestReviewComment": - body = node_body(node) - commit_oid = review_node_commit_oid(node) - if commit_oid != head_sha and not body_mentions_head(body, head_sha): - continue - return True - if node.get("__typename") == "IssueComment": - return True - - return False - - -def pr_timeline_evidence(repo: str, number: int) -> tuple[str, list[dict[str, Any]]]: - owner, name = repo.split("/", 1) - before: str | None = None - head_sha: str | None = None - timeline_nodes: list[dict[str, Any]] = [] - - while True: - fields: dict[str, object] = {"owner": owner, "name": name, "number": number} - if before is not None: - fields["before"] = before - payload = graphql(PR_TIMELINE_QUERY, **fields) - pr = payload.get("data", {}).get("repository", {}).get("pullRequest") - if not isinstance(pr, dict): - raise GhError(f"{repo}#{number}: GraphQL did not return a pull request") - - page_head_sha = pr.get("headRefOid") - commit_nodes = pr.get("commits", {}).get("nodes", []) - last_commit = commit_nodes[-1].get("commit", {}) if commit_nodes else {} - commit_sha = last_commit.get("oid") - if not isinstance(page_head_sha, str) or not page_head_sha: - raise GhError(f"{repo}#{number}: GraphQL did not return headRefOid") - if commit_sha != page_head_sha: - raise GhError(f"{repo}#{number}: headRefOid {page_head_sha} disagrees with commits.last {commit_sha}") - if head_sha is None: - head_sha = page_head_sha - elif head_sha != page_head_sha: - raise GhError(f"{repo}#{number}: headRefOid changed while paging timeline") - - timeline = pr.get("timelineItems") - if not isinstance(timeline, dict): - raise GhError(f"{repo}#{number}: GraphQL did not return timeline items") - nodes = timeline.get("nodes", []) - if not isinstance(nodes, list): - raise GhError(f"{repo}#{number}: GraphQL did not return timeline nodes") - page_nodes = [node for node in nodes if isinstance(node, dict)] - timeline_nodes = page_nodes + timeline_nodes - if any(timeline_head_oid(node) == head_sha for node in page_nodes): - break - - page_info = timeline.get("pageInfo") - if not isinstance(page_info, dict) or not page_info.get("hasPreviousPage"): - break - start_cursor = page_info.get("startCursor") - if not isinstance(start_cursor, str) or not start_cursor: - break - before = start_cursor - - if head_sha is None: - raise GhError(f"{repo}#{number}: GraphQL did not return headRefOid") - return head_sha, merge_review_comment_nodes( - timeline_nodes, - pull_review_comment_nodes(repo, number, head_sha=head_sha), - ) - - -def classify_check_state( - check_runs: list[dict[str, Any]], - combined_status: dict[str, Any], - *, - required_check_names: frozenset[str] = frozenset(), -) -> str: - states: list[str] = [] - seen_check_names: set[str] = set() - named_check_runs: dict[str, dict[str, Any]] = {} - unnamed_check_runs: list[dict[str, Any]] = [] - - authoritative_ci_workflow = authoritative_ci_workflow_id(check_runs) - authoritative_ci_run = authoritative_ci_workflow_run_id( - check_runs, - workflow_id=authoritative_ci_workflow, - ) - if authoritative_ci_run is not None and authoritative_ci_workflow is not None: - check_runs = [ - item - for item in check_runs - if github_actions_workflow_id(item) != authoritative_ci_workflow - or github_actions_workflow_run_id(item) in {None, authoritative_ci_run} - ] - - for item in check_runs: - name = item.get("name") - if isinstance(name, str): - previous = named_check_runs.get(name) - if previous is None or check_run_recency_key(item) >= check_run_recency_key(previous): - named_check_runs[name] = item - else: - unnamed_check_runs.append(item) - - for item in [*named_check_runs.values(), *unnamed_check_runs]: - name = item.get("name") - if isinstance(name, str): - seen_check_names.add(name) - conclusion = str(item.get("conclusion") or "").upper() - status = str(item.get("status") or "").upper() - states.append(conclusion or status or "UNKNOWN") - - for item in combined_status.get("statuses", []) if isinstance(combined_status, dict) else []: - if isinstance(item, dict): - states.append(str(item.get("state") or "").upper()) - - if not states: - return "none" - if any(state in FAIL_CHECK_STATES for state in states): - return "failure" - if any(state in PENDING_CHECK_STATES for state in states): - return "pending" - if required_check_names and not required_check_names <= seen_check_names: - return "pending" - if all(state in SUCCESS_CHECK_STATES for state in states): - return "success" - return "unknown" - - -def check_run_recency_key(item: dict[str, Any]) -> tuple[str, str]: - return ( - str( - item.get("started_at") - or item.get("startedAt") - or item.get("created_at") - or item.get("createdAt") - or item.get("completed_at") - or item.get("completedAt") - or "" - ), - str(item.get("completed_at") or item.get("completedAt") or ""), - ) - - -def github_actions_workflow_run_id(item: dict[str, Any]) -> str | None: - details_url = item.get("details_url") or item.get("detailsUrl") - if not isinstance(details_url, str): - return None - match = re.search(r"/actions/runs/(\d+)(?:/|$)", details_url) - return match.group(1) if match is not None else None - - -def authoritative_ci_workflow_run_id( - check_runs: list[dict[str, Any]], - *, - workflow_id: str | None, -) -> str | None: - if workflow_id is None: - return None - workflow_runs = [ - item - for item in check_runs - if github_actions_workflow_id(item) == workflow_id and github_actions_workflow_run_id(item) is not None - ] - if not workflow_runs: - return None - latest_run = max(workflow_runs, key=github_actions_workflow_run_recency_key) - return github_actions_workflow_run_id(latest_run) - - -def github_actions_workflow_run_recency_key(item: dict[str, Any]) -> tuple[str, tuple[str, str], int]: - run_id = github_actions_workflow_run_id(item) - return ( - str(item.get("_github_actions_run_started_at") or item.get("_github_actions_run_created_at") or ""), - check_run_recency_key(item), - int(run_id) if isinstance(run_id, str) and run_id.isdigit() else 0, - ) - - -def github_actions_workflow_id(item: dict[str, Any]) -> str | None: - workflow_id = item.get("_github_actions_workflow_id") - return str(workflow_id) if isinstance(workflow_id, (int, str)) else None - - -def authoritative_ci_workflow_id(check_runs: list[dict[str, Any]]) -> str | None: - required_runs = [item for item in check_runs if item.get("name") == "CI Required"] - if not required_runs: - return None - latest_required = max(required_runs, key=check_run_recency_key) - return github_actions_workflow_id(latest_required) - - -def annotate_github_actions_workflow_ids(repo: str, check_runs: list[dict[str, Any]]) -> list[dict[str, Any]]: - workflow_metadata_by_run: dict[str, tuple[str, str | None]] = {} - for run_id in {github_actions_workflow_run_id(item) for item in check_runs} - {None}: - assert run_id is not None - try: - workflow_run = gh_api(f"/repos/{repo}/actions/runs/{run_id}") - except GhError: - continue - workflow_id = workflow_run.get("workflow_id") if isinstance(workflow_run, dict) else None - if isinstance(workflow_id, (int, str)): - # GitHub preserves ``created_at`` when an existing run id is rerun, - # while ``run_started_at`` advances to the current attempt. - run_started_at = workflow_run.get("run_started_at") or workflow_run.get("created_at") - workflow_metadata_by_run[run_id] = ( - str(workflow_id), - str(run_started_at) if isinstance(run_started_at, str) else None, - ) - - annotated: list[dict[str, Any]] = [] - for item in check_runs: - run_id = github_actions_workflow_run_id(item) - metadata = workflow_metadata_by_run.get(run_id) if run_id is not None else None - if metadata is None: - annotated.append(item) - continue - workflow_id, run_started_at = metadata - annotated_item = {**item, "_github_actions_workflow_id": workflow_id} - if run_started_at is not None: - annotated_item["_github_actions_run_started_at"] = run_started_at - annotated.append(annotated_item) - return annotated - - -def commit_checks_state(repo: str, head_sha: str) -> str: - check_runs = paged_api(f"/repos/{repo}/commits/{head_sha}/check-runs") - check_runs = annotate_github_actions_workflow_ids(repo, check_runs) - combined_status = gh_api(f"/repos/{repo}/commits/{head_sha}/status") - return classify_check_state( - check_runs, - combined_status if isinstance(combined_status, dict) else {}, - required_check_names=REQUIRED_CHECKS_BY_REPO.get(repo, frozenset()), - ) - - -def decision_requires_writes(decision: SyncDecision) -> bool: - """Return whether applying this decision would mutate GitHub state.""" - - return ( - decision.ok_action != "keep" - or decision.needs_work_action != "keep" - or decision.needs_rebase_action != "keep" - or bool(decision.legacy_labels) - or bool(decision.approve_workflow_run_ids) - ) - - -def pr_merge_state(repo: str, number: int) -> str: - payload = run_gh( - ["pr", "view", str(number), "--repo", repo, "--json", "mergeStateStatus,mergeable"], - timeout_seconds=30, - ) - if not isinstance(payload, dict): - raise GhError(f"{repo}#{number}: expected pull request object") - mergeable = str(payload.get("mergeable") or "").upper() - merge_state = str(payload.get("mergeStateStatus") or "").upper() - if mergeable == "CONFLICTING": - return "CONFLICTING" - return merge_state or "UNKNOWN" - - -def needs_rebase_label_target(merge_state: str, *, has_label: bool) -> bool: - """Sync confirmed conflicts and preserve the label when GitHub is ambiguous.""" - - if merge_state in NEEDS_REBASE_STATES: - return True - if merge_state in NO_REBASE_STATES: - return False - return has_label - - -def workflow_runs_requiring_approval(repo: str, head_sha: str) -> tuple[int, ...]: - runs = paged_api(f"/repos/{repo}/actions/runs?event=pull_request&head_sha={head_sha}") - run_ids: list[int] = [] - for run in runs: - status = str(run.get("status") or "").lower() - conclusion = str(run.get("conclusion") or "").lower() - run_id = run.get("id") - if not isinstance(run_id, int): - continue - if status == "action_required" or conclusion == "action_required": - run_ids.append(run_id) - return tuple(run_ids) - - -def unresolved_codex_finding_thread_urls( - repo: str, - number: int, - *, - head_sha: str, - allowed_authors: set[str], -) -> tuple[str, ...]: - owner, name = repo.split("/", 1) - after: str | None = None - urls: list[str] = [] - - while True: - fields: dict[str, object] = {"owner": owner, "name": name, "number": number} - if after is not None: - fields["after"] = after - payload = graphql(PR_REVIEW_THREADS_QUERY, **fields) - pr = payload.get("data", {}).get("repository", {}).get("pullRequest") - if not isinstance(pr, dict): - raise GhError(f"{repo}#{number}: GraphQL did not return a pull request") - - threads = pr.get("reviewThreads") - if not isinstance(threads, dict): - raise GhError(f"{repo}#{number}: GraphQL did not return review threads") - nodes = threads.get("nodes", []) - if not isinstance(nodes, list): - raise GhError(f"{repo}#{number}: GraphQL did not return review thread nodes") - - for thread in nodes: - if not isinstance(thread, dict): - continue - if thread.get("isResolved") or thread.get("isOutdated"): - continue - comments = thread.get("comments") - comment_nodes = comments.get("nodes") if isinstance(comments, dict) else [] - if not isinstance(comment_nodes, list): - continue - for comment in comment_nodes: - if not isinstance(comment, dict): - continue - author = comment.get("author") - login = author.get("login") if isinstance(author, dict) else None - if login not in allowed_authors: - continue - if not is_needs_work_codex_body(comment.get("body")): - continue - body = comment.get("body") - commit = comment.get("commit") - commit_oid = commit.get("oid") if isinstance(commit, dict) else None - original_commit = comment.get("originalCommit") - original_oid = original_commit.get("oid") if isinstance(original_commit, dict) else None - body_mentions_current_head = body_mentions_head(body, head_sha) - if body_mentions_current_head: - pass - elif commit_oid == head_sha: - pass - elif original_oid == head_sha: - pass - else: - continue - url = comment.get("url") - urls.append(str(url) if isinstance(url, str) else "unresolved Codex review thread") - - page_info = threads.get("pageInfo") - if not isinstance(page_info, dict) or not page_info.get("hasNextPage"): - break - end_cursor = page_info.get("endCursor") - if not isinstance(end_cursor, str) or not end_cursor: - break - after = end_cursor - - return tuple(urls) - - -def is_github_app_write_denial(exc: BaseException) -> bool: - """Return True when GitHub rejected a write from the current token.""" - - text = str(exc) - return "Resource not accessible by integration" in text and "HTTP 403" in text - - -def write_warning(action: str, exc: BaseException) -> str: - return f"{action}: skipped because the GitHub token cannot write this resource ({exc})" - - -def is_missing_issue_label(exc: BaseException) -> bool: - """Return True when GitHub reports that an issue label is already absent.""" - - text = str(exc) - return "HTTP 404" in text and "Label does not exist" in text - - -def gh_api_write( - path: str, - *, - method: str = "GET", - input_json: Any | None = None, - tolerate_permission_errors: bool, - tolerate_missing: bool = False, - action: str, -) -> str | None: - try: - gh_api(path, method=method, input_json=input_json) - except GhError as exc: - if tolerate_missing and is_missing_issue_label(exc): - return None - if tolerate_permission_errors and is_github_app_write_denial(exc): - return write_warning(action, exc) - raise - return None - - -def run_gh_write( - args: list[str], - *, - timeout_seconds: int, - tolerate_permission_errors: bool, - action: str, - fallback_retry: bool = True, -) -> str | None: - try: - run_gh(args, timeout_seconds=timeout_seconds, fallback_retry=fallback_retry) - except GhError as exc: - if tolerate_permission_errors and is_github_app_write_denial(exc): - return write_warning(action, exc) - raise - return None - - -def ensure_label( - repo: str, - label: str, - *, - color: str, - description: str, - apply: bool, - tolerate_permission_errors: bool = False, -) -> tuple[str, ...]: - if not apply: - return () - try: - gh_api(f"/repos/{repo}/labels/{quote(label, safe='')}") - return () - except GhError as exc: - if "HTTP 404" not in str(exc): - raise - - try: - warning = gh_api_write( - f"/repos/{repo}/labels", - method="POST", - input_json={ - "name": label, - "color": color, - "description": description, - }, - tolerate_permission_errors=tolerate_permission_errors, - action=f"create label {repo}:{label}", - ) - return (warning,) if warning else () - except GhError as exc: - if "already_exists" not in str(exc) and "already exists" not in str(exc).lower(): - raise - return () - - -def decide_pr( - repo: str, - number: int, - *, - allowed_authors: set[str], - ignore_checks: bool, - timeline_observer: Callable[[int, list[dict[str, Any]]], None] | None = None, -) -> SyncDecision: - head_sha, timeline_nodes = pr_timeline_evidence(repo, number) - if timeline_observer is not None: - timeline_observer(number, timeline_nodes) - labels = issue_label_names(repo, number) - checks_state = commit_checks_state(repo, head_sha) - merge_state = pr_merge_state(repo, number) - review_state, review_node = find_current_head_codex_review_state( - timeline_nodes, - head_sha=head_sha, - allowed_authors=allowed_authors, - ) - unresolved_finding_urls = unresolved_codex_finding_thread_urls( - repo, - number, - head_sha=head_sha, - allowed_authors=allowed_authors, - ) - has_codex_news = has_codex_news_after_current_head( - timeline_nodes, - head_sha=head_sha, - allowed_authors=allowed_authors, - ) - has_ok_label = CODEX_OK_LABEL in labels - has_needs_work_label = CODEX_NEEDS_WORK_LABEL in labels - has_needs_rebase_label = NEEDS_REBASE_LABEL in labels - wants_needs_rebase_label = needs_rebase_label_target( - merge_state, - has_label=has_needs_rebase_label, - ) - legacy_labels = frozenset(label for label in labels if label in LEGACY_CODEX_LABELS) - - reason_parts: list[str] = [] - wants_ok_label = review_state == "clean" - wants_needs_work_label = review_state == "needs_work" - if review_state == "none": - reason_parts.append("no provable clean Codex review for current head") - elif review_state == "pending": - reason_parts.append("Codex review request is acknowledged but still pending") - elif review_state == "needs_work": - reason_parts.append("Codex raised current-head review issues") - else: - reason_parts.append("clean Codex review matches current head") - - if unresolved_finding_urls: - wants_ok_label = False - wants_needs_work_label = True - reason_parts.append(f"unresolved Codex review threads: {len(unresolved_finding_urls)}") - - if not ignore_checks and checks_state != "success": - wants_ok_label = False - reason_parts.append(f"checks are {checks_state}") - if not ignore_checks and merge_state in UNMERGEABLE_STATES | {"CONFLICTING"}: - wants_ok_label = False - reason_parts.append(f"merge state is {merge_state.lower()}") - if not ignore_checks and merge_state == "UNKNOWN" and not has_ok_label: - wants_ok_label = False - reason_parts.append("merge state is still unknown") - - trigger_codex_review = ( - review_state == "none" - and checks_state == "success" - and merge_state not in UNMERGEABLE_STATES | {"CONFLICTING"} - and merge_state != "UNKNOWN" - and not has_codex_news - ) - if trigger_codex_review: - reason_parts.append("current-head CI is green and no Codex news exists after head") - - approve_workflow_run_ids: tuple[int, ...] = () - if ( - review_state == "clean" - and not unresolved_finding_urls - and merge_state not in UNMERGEABLE_STATES | {"CONFLICTING"} - and merge_state != "UNKNOWN" - ): - approve_workflow_run_ids = workflow_runs_requiring_approval(repo, head_sha) - if approve_workflow_run_ids: - reason_parts.append( - "workflow runs need approval: " + ",".join(str(run_id) for run_id in approve_workflow_run_ids) - ) - - if wants_ok_label and not has_ok_label: - ok_action = "add" - elif not wants_ok_label and has_ok_label: - ok_action = "remove" - else: - ok_action = "keep" - if wants_needs_work_label and not has_needs_work_label: - needs_work_action = "add" - elif not wants_needs_work_label and has_needs_work_label: - needs_work_action = "remove" - else: - needs_work_action = "keep" - if wants_needs_rebase_label and not has_needs_rebase_label: - needs_rebase_action = "add" - elif not wants_needs_rebase_label and has_needs_rebase_label: - needs_rebase_action = "remove" - else: - needs_rebase_action = "keep" - - review_url = unresolved_finding_urls[0] if unresolved_finding_urls else None - if review_url is None and isinstance(review_node, dict): - review_url = node_url(review_node) - - return SyncDecision( - repo=repo, - number=number, - head_sha=head_sha, - has_ok_label=has_ok_label, - wants_ok_label=wants_ok_label, - ok_action=ok_action, - has_needs_work_label=has_needs_work_label, - wants_needs_work_label=wants_needs_work_label, - needs_work_action=needs_work_action, - has_needs_rebase_label=has_needs_rebase_label, - wants_needs_rebase_label=wants_needs_rebase_label, - needs_rebase_action=needs_rebase_action, - legacy_labels=legacy_labels, - reason="; ".join(reason_parts), - review_url=review_url, - review_state=review_state, - checks_state=checks_state, - merge_state=merge_state, - trigger_codex_review=trigger_codex_review, - approve_workflow_run_ids=approve_workflow_run_ids, - ) - - -def apply_decision(decision: SyncDecision, *, tolerate_permission_errors: bool = False) -> tuple[str, ...]: - warnings: list[str] = [] - - def record(warning: str | None) -> None: - if warning: - warnings.append(warning) - - if decision.ok_action == "add": - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels", - method="POST", - input_json={"labels": [CODEX_OK_LABEL]}, - tolerate_permission_errors=tolerate_permission_errors, - action=f"add {CODEX_OK_LABEL} to {decision.repo}#{decision.number}", - ) - ) - elif decision.ok_action == "remove": - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels/{quote(CODEX_OK_LABEL, safe='')}", - method="DELETE", - tolerate_permission_errors=tolerate_permission_errors, - tolerate_missing=True, - action=f"remove {CODEX_OK_LABEL} from {decision.repo}#{decision.number}", - ) - ) - if decision.needs_work_action == "add": - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels", - method="POST", - input_json={"labels": [CODEX_NEEDS_WORK_LABEL]}, - tolerate_permission_errors=tolerate_permission_errors, - action=f"add {CODEX_NEEDS_WORK_LABEL} to {decision.repo}#{decision.number}", - ) - ) - elif decision.needs_work_action == "remove": - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels/{quote(CODEX_NEEDS_WORK_LABEL, safe='')}", - method="DELETE", - tolerate_permission_errors=tolerate_permission_errors, - tolerate_missing=True, - action=f"remove {CODEX_NEEDS_WORK_LABEL} from {decision.repo}#{decision.number}", - ) - ) - if decision.needs_rebase_action == "add": - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels", - method="POST", - input_json={"labels": [NEEDS_REBASE_LABEL]}, - tolerate_permission_errors=tolerate_permission_errors, - action=f"add {NEEDS_REBASE_LABEL} to {decision.repo}#{decision.number}", - ) - ) - elif decision.needs_rebase_action == "remove": - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels/{quote(NEEDS_REBASE_LABEL, safe='')}", - method="DELETE", - tolerate_permission_errors=tolerate_permission_errors, - tolerate_missing=True, - action=f"remove {NEEDS_REBASE_LABEL} from {decision.repo}#{decision.number}", - ) - ) - for label in decision.legacy_labels: - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels/{quote(label, safe='')}", - method="DELETE", - tolerate_permission_errors=tolerate_permission_errors, - tolerate_missing=True, - action=f"remove legacy {label} from {decision.repo}#{decision.number}", - ) - ) - return tuple(warnings) - - -def trigger_codex_review( - decision: SyncDecision, - *, - body: str, - tolerate_permission_errors: bool = False, -) -> tuple[str, ...]: - warning = run_gh_write( - [ - "api", - "--method", - "POST", - f"/repos/{decision.repo}/issues/{decision.number}/comments", - "-f", - f"body={body}", - ], - timeout_seconds=30, - tolerate_permission_errors=tolerate_permission_errors, - action=f"request Codex review on {decision.repo}#{decision.number}", - fallback_retry=False, - ) - return (warning,) if warning else () - - -def approve_workflow_runs( - decision: SyncDecision, - *, - tolerate_permission_errors: bool = False, -) -> tuple[str, ...]: - warnings: list[str] = [] - for run_id in decision.approve_workflow_run_ids: - warning = gh_api_write( - f"/repos/{decision.repo}/actions/runs/{run_id}/approve", - method="POST", - tolerate_permission_errors=tolerate_permission_errors, - action=f"approve workflow run {run_id} for {decision.repo}#{decision.number}", - ) - if warning: - warnings.append(warning) - return tuple(warnings) - - -def parse_args(argv: list[str]) -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=("Idempotently sync GitHub Codex review labels based on current-head Codex review state.") - ) - parser.add_argument("--repo", action="append", required=True, help="GitHub repo as owner/name. May repeat.") - parser.add_argument("--pr", action="append", type=int, help="PR number to sync. May repeat.") - parser.add_argument("--all-open", action="store_true", help="Sync all open PRs in each --repo.") - parser.add_argument("--apply", action="store_true", help="Actually write labels. Default is dry-run.") - parser.add_argument( - "--no-trigger-missing-codex", - action="store_true", - help="Do not post @codex review when current-head CI is green and Codex has no current-head news.", - ) - parser.add_argument( - "--no-approve-workflow-runs", - action="store_true", - help=( - "Do not approve action_required fork workflow runs after a current-head " - "clean Codex review on a mergeable PR." - ), - ) - parser.add_argument( - "--codex-review-command", - default="@codex review", - help="Issue comment body used to request a missing Codex review.", - ) - parser.add_argument( - "--codex-usage-limit-backoff-hours", - type=float, - default=DEFAULT_CODEX_USAGE_LIMIT_BACKOFF_HOURS, - help=( - "Skip new @codex review comments when the same comment sender account received a Codex usage-limit " - "reply within this many hours, unless that same account has a newer normal Codex response." - ), - ) - parser.add_argument( - "--codex-review-response-wait-seconds", - type=float, - default=DEFAULT_CODEX_REVIEW_RESPONSE_WAIT_SECONDS, - help=( - "After posting the first @codex review without recent quota evidence, wait this long, reread the PR " - "timeline, and stop further review requests if Codex replied with a usage limit." - ), - ) - parser.add_argument( - "--ignore-checks", - action="store_true", - help="Ignore current-head CI state when deciding the ok label. Normally do not use this.", - ) - parser.add_argument( - "--tolerate-write-permission-errors", - action="store_true", - help=( - "Log and continue when GitHub returns Resource not accessible by integration " - "for label/comment/approval writes. Read/classification errors still fail." - ), - ) - parser.add_argument( - "--tolerate-read-errors", - action="store_true", - help=( - "Log and continue when a selected PR cannot be classified because of a GitHub " - "read/API error. Intended for broad --all-open best-effort maintenance runs." - ), - ) - parser.add_argument( - "--reviewer-login", - action="append", - default=[], - help="Allowed Codex reviewer login. May repeat; defaults include chatgpt-codex-connector[bot].", - ) - return parser.parse_args(argv) - - -def main(argv: list[str] | None = None) -> int: - args = parse_args(argv or sys.argv[1:]) - repos = [repo_path(repo) for repo in args.repo] - allowed_authors = CODEX_REVIEW_AUTHORS | set(args.reviewer_login) - had_error = False - # Per-sender quota state is shared across every --repo in the run: a usage - # limit observed in one repository suppresses review requests in the rest. - # Timelines classified before the backoff exists are retained so evidence - # from repositories without their own triggers still counts. - usage_backoff: CodexReviewUsageBackoff | None = None - unobserved_timelines: list[list[dict[str, Any]]] = [] - codex_sender_unresolved = False - - for repo in repos: - setup_warnings: list[str] = [] - setup_warnings.extend( - ensure_label( - repo, - CODEX_OK_LABEL, - color="0e8a16", - description="Current PR head has green CI and a clean Codex review", - apply=args.apply, - tolerate_permission_errors=args.tolerate_write_permission_errors, - ) - ) - setup_warnings.extend( - ensure_label( - repo, - CODEX_NEEDS_WORK_LABEL, - color="d93f0b", - description="Codex raised issues on the current PR head that still need work", - apply=args.apply, - tolerate_permission_errors=args.tolerate_write_permission_errors, - ) - ) - setup_warnings.extend( - ensure_label( - repo, - NEEDS_REBASE_LABEL, - color="fbca04", - description="Needs rebase or conflict repair against current main", - apply=args.apply, - tolerate_permission_errors=args.tolerate_write_permission_errors, - ) - ) - for warning in setup_warnings: - print(f"warning: {warning}", file=sys.stderr, flush=True) - numbers = list_open_pr_numbers(repo) if args.all_open else list(args.pr or []) - if not numbers: - print(f"{repo}: no PRs selected; pass --pr or --all-open", file=sys.stderr) - had_error = True - continue - - timeline_nodes_by_number: dict[int, list[dict[str, Any]]] = {} - - def observe_timeline(_number: int, timeline_nodes: list[dict[str, Any]]) -> None: - timeline_nodes_by_number[_number] = timeline_nodes - - decisions: list[SyncDecision] = [] - classified_count = 0 - for number in sorted(set(numbers)): - try: - decision = decide_pr( - repo, - number, - allowed_authors=allowed_authors, - ignore_checks=args.ignore_checks, - timeline_observer=observe_timeline, - ) - except GhError as exc: - if not args.tolerate_read_errors: - had_error = True - print(f"{repo}#{number}: {exc}", file=sys.stderr, flush=True) - continue - except Exception as exc: # noqa: BLE001 - had_error = True - print(f"{repo}#{number}: {exc}", file=sys.stderr, flush=True) - continue - - classified_count += 1 - decisions.append(decision) - - if args.tolerate_read_errors and classified_count == 0: - had_error = True - print( - f"{repo}: all selected PRs failed classification; refusing a false-green tolerant run", - file=sys.stderr, - flush=True, - ) - - if args.apply and not args.no_trigger_missing_codex: - unobserved_timelines.extend(timeline_nodes_by_number.values()) - repo_has_triggers = any(decision.trigger_codex_review for decision in decisions) - if repo_has_triggers: - # Quota evidence may live outside the selected PRs (single - # --pr runs, closed PRs), so also observe the repo's recent - # issue comments before posting anything here. - try: - unobserved_timelines.extend( - recent_issue_comment_timelines( - repo, - since=datetime.now(UTC) - timedelta(hours=args.codex_usage_limit_backoff_hours), - ) - ) - except GhError as exc: - print( - f"warning: {repo}: could not gather repo-wide Codex quota evidence: {exc}", - file=sys.stderr, - flush=True, - ) - if usage_backoff is None and not codex_sender_unresolved and repo_has_triggers: - sender = resolve_codex_request_sender() - if sender is None: - # Only the trigger/backoff path depends on the sender; - # label sync and workflow approvals proceed regardless. - codex_sender_unresolved = True - print( - f"{repo}: cannot determine @codex review sender; " - "skipping review triggers but continuing label sync", - file=sys.stderr, - flush=True, - ) - else: - usage_backoff = CodexReviewUsageBackoff( - request_author=sender, - allowed_authors=allowed_authors, - window=timedelta(hours=args.codex_usage_limit_backoff_hours), - now=datetime.now(UTC), - ) - if usage_backoff is not None: - for timeline_nodes in unobserved_timelines: - usage_backoff.observe(timeline_nodes) - unobserved_timelines.clear() - - for decision in decisions: - try: - write_warnings: tuple[str, ...] = () - trigger_codex_review_now = decision.trigger_codex_review and not args.no_trigger_missing_codex - if args.apply and (decision_requires_writes(decision) or trigger_codex_review_now): - # Under --all-open every PR is classified before any is - # applied; evidence (head, checks, reviews, mergeability) - # may have moved meanwhile. Reclassify immediately before - # writing and act on the fresh decision only. The fresh - # timeline also feeds the shared backoff so a quota reply - # that arrived after bulk classification suppresses the - # remaining review requests. - try: - fresh_decision = decide_pr( - decision.repo, - decision.number, - allowed_authors=allowed_authors, - ignore_checks=args.ignore_checks, - timeline_observer=( - backoff_timeline_observer(usage_backoff) if usage_backoff is not None else None - ), - ) - except GhError as exc: - if not args.tolerate_read_errors: - had_error = True - print( - f"{decision.repo}#{decision.number}: apply-time reclassification failed: {exc}", - file=sys.stderr, - flush=True, - ) - continue - if fresh_decision.head_sha != decision.head_sha: - print( - f"warning: {decision.repo}#{decision.number}: head moved from " - f"{decision.head_sha[:12]} to {fresh_decision.head_sha[:12]} after classification; " - "skipping stale decision", - file=sys.stderr, - flush=True, - ) - continue - trigger_codex_review_now = trigger_codex_review_now and fresh_decision.trigger_codex_review - decision = fresh_decision - if args.apply: - accumulated_warnings: list[str] = [] - accumulated_warnings.extend( - apply_decision( - decision, - tolerate_permission_errors=args.tolerate_write_permission_errors, - ) - ) - if decision.approve_workflow_run_ids and not args.no_approve_workflow_runs: - accumulated_warnings.extend( - approve_workflow_runs( - decision, - tolerate_permission_errors=args.tolerate_write_permission_errors, - ) - ) - if trigger_codex_review_now and _fallback_token_active: - # Comments would now be authored by the fallback token's - # identity, not the resolved sender, so quota replies - # could no longer be attributed. Stop posting. - accumulated_warnings.append( - f"request Codex review on {decision.repo}#{decision.number}: skipped because " - "the run switched to GH_FALLBACK_TOKEN and the resolved sender no longer " - "matches the active token" - ) - trigger_codex_review_now = False - if trigger_codex_review_now and codex_sender_unresolved: - accumulated_warnings.append( - f"request Codex review on {decision.repo}#{decision.number}: skipped because " - "the @codex review sender could not be resolved" - ) - trigger_codex_review_now = False - if trigger_codex_review_now and usage_backoff is not None and usage_backoff.is_limited(): - accumulated_warnings.append(usage_backoff.skip_warning(decision)) - trigger_codex_review_now = False - if trigger_codex_review_now: - trigger_warnings = trigger_codex_review( - decision, - body=args.codex_review_command, - tolerate_permission_errors=args.tolerate_write_permission_errors, - ) - accumulated_warnings.extend(trigger_warnings) - review_request_posted = not trigger_warnings - if ( - review_request_posted - and usage_backoff is not None - and usage_backoff.latest_normal_response_at is None - ): - if args.codex_review_response_wait_seconds > 0: - time.sleep(args.codex_review_response_wait_seconds) - _head_sha, timeline_nodes = pr_timeline_evidence(decision.repo, decision.number) - usage_backoff.observe(timeline_nodes) - write_warnings = tuple(accumulated_warnings) - mode = "apply" if args.apply else "dry-run" - print( - f"{mode} {decision.repo}#{decision.number}: " - f"head={decision.head_sha[:12]} checks={decision.checks_state} " - f"merge={decision.merge_state} review={decision.review_state} " - f"ok={decision.has_ok_label}->{decision.wants_ok_label}/{decision.ok_action} " - f"needs_work={decision.has_needs_work_label}->{decision.wants_needs_work_label}/" - f"{decision.needs_work_action} " - f"needs_rebase={decision.has_needs_rebase_label}->{decision.wants_needs_rebase_label}/" - f"{decision.needs_rebase_action} " - f"legacy={','.join(sorted(decision.legacy_labels)) or '-'} " - f"approve_runs={','.join(str(run_id) for run_id in decision.approve_workflow_run_ids) or '-'} " - f"trigger_codex={trigger_codex_review_now} " - f"reason={decision.reason}", - flush=True, - ) - if decision.review_url: - print(f" review_url={decision.review_url}", flush=True) - for warning in write_warnings: - print(f" write_warning={warning}", flush=True) - except Exception as exc: # noqa: BLE001 - had_error = True - print(f"{decision.repo}#{decision.number}: {exc}", file=sys.stderr, flush=True) - - return 1 if had_error else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.github/simplicity-budgets.toml b/.github/simplicity-budgets.toml index ded1760f3c..7adf5af4c5 100644 --- a/.github/simplicity-budgets.toml +++ b/.github/simplicity-budgets.toml @@ -26,3 +26,43 @@ max_lines = 60 path = "frontend/src/components/layout/app-header.tsx" array = "CORE_NAV_ITEMS" max_items = 5 + +[root_files] +# Complete allowlist of tracked repository-root entries (files and +# directories), compared against `git ls-tree --name-only HEAD`. A new root +# entry is a reviewable one-line diff here; anything not listed fails the +# check. Keep the list sorted. +allowed = [ + ".agents", + ".all-contributorsrc", + ".claude", + ".dockerignore", + ".env.example", + ".github", + ".gitignore", + ".pre-commit-config.yaml", + "AGENTS.md", + "CHANGELOG.md", + "CLAUDE.md", + "Dockerfile", + "Dockerfile.distroless", + "LICENSE", + "Makefile", + "PRINCIPLES.md", + "README.md", + "README.zh-CN.md", + "app", + "config", + "deploy", + "docker-compose.prod.yml", + "docker-compose.yml", + "docs", + "frontend", + "mkdocs.yml", + "openspec", + "pyproject.toml", + "renovate.json", + "scripts", + "tests", + "uv.lock", +] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd8009942a..78b39d39d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,10 +190,9 @@ jobs: run: make frontend-typecheck frontend-test: - # Keep this exact job name: the repository ruleset and - # .github/scripts/sync_codex_ok_labels.py treat "Frontend tests - # (vitest + coverage)" as a required check context. PR runs skip the - # coverage instrumentation below, but the check name must not change. + # Keep this exact job name: the repository ruleset requires "Frontend + # tests (vitest + coverage)" as a check context. PR runs skip the coverage + # instrumentation below, but the check name must not change. name: Frontend tests (vitest + coverage) runs-on: ubuntu-24.04 needs: changes @@ -294,7 +293,7 @@ jobs: key: playwright-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('frontend/bun.lock') }} - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -322,7 +321,7 @@ jobs: persist-credentials: false - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -344,7 +343,7 @@ jobs: persist-credentials: false - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -398,7 +397,7 @@ jobs: - name: Set up uv if: needs.changes.outputs.backend == 'true' - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -453,7 +452,7 @@ jobs: - name: Set up uv if: needs.changes.outputs.backend == 'true' - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -464,11 +463,10 @@ jobs: run: make test-integration-core-${{ matrix.shard }} # Aggregate for the shards above. Keep this exact job name: the repository - # ruleset and .github/scripts/sync_codex_ok_labels.py treat - # "Tests (pytest, integration-core)" as a required check context. The shard - # jobs never skip at job level (they use the placeholder-step pattern), so - # anything other than an all-success matrix result must fail here — - # skipped or cancelled shards are not laundered into a passing check. + # ruleset requires "Tests (pytest, integration-core)" as a check context. + # The shard jobs never skip at job level (they use the placeholder-step + # pattern), so anything other than an all-success matrix result must fail + # here — skipped or cancelled shards are not laundered into a passing check. test-integration-core-required: name: Tests (pytest, integration-core) runs-on: ubuntu-24.04 @@ -552,7 +550,7 @@ jobs: - name: Set up uv if: needs.changes.outputs.backend == 'true' - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -575,7 +573,7 @@ jobs: persist-credentials: false - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -611,7 +609,7 @@ jobs: persist-credentials: false - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -648,7 +646,7 @@ jobs: bun-1.3.14-${{ runner.os }}-${{ runner.arch }}- - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -699,7 +697,7 @@ jobs: ignore-unfixed: true - name: Upload Trivy scan results to GitHub Security - uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd if: always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) with: sarif_file: trivy-results.sarif diff --git a/.github/workflows/codex-review-labels.yml b/.github/workflows/codex-review-labels.yml deleted file mode 100644 index 62eef30fbe..0000000000 --- a/.github/workflows/codex-review-labels.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Codex review labels - -on: - pull_request_target: - types: [opened, synchronize, reopened, ready_for_review, converted_to_draft] - workflow_run: - workflows: ["CI"] - types: [completed] - issue_comment: - types: [created, edited] - pull_request_review: - types: [submitted, edited, dismissed] - schedule: - - cron: "*/15 * * * *" - -permissions: - actions: write - checks: read - contents: read - issues: write - pull-requests: read - statuses: read - -concurrency: - group: codex-review-labels-${{ github.event.pull_request.number || github.event.issue.number || github.event.workflow_run.head_sha || github.run_id }} - cancel-in-progress: false - -jobs: - sync-pr: - name: Sync Codex labels for PR - runs-on: ubuntu-24.04 - if: >- - ${{ - github.event_name == 'pull_request_target' || - github.event_name == 'pull_request_review' || - (github.event_name == 'issue_comment' && github.event.issue.pull_request) - }} - - steps: - - name: Checkout trusted base - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - persist-credentials: false - ref: ${{ github.event.repository.default_branch }} - - - name: Mint label sync App token - id: app-token - if: ${{ vars.CODEX_LABEL_SYNC_APP_ID != '' }} - continue-on-error: true - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 - with: - app-id: ${{ vars.CODEX_LABEL_SYNC_APP_ID }} - private-key: ${{ secrets.CODEX_LABEL_SYNC_APP_PRIVATE_KEY }} - permission-actions: write - permission-checks: read - permission-contents: read - permission-issues: write - permission-pull-requests: read - permission-statuses: read - - - name: Sync labels - env: - GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.CODEX_LABEL_SYNC_TOKEN || secrets.RELEASE_PLEASE_TOKEN || github.token }} - GH_FALLBACK_TOKEN: ${{ github.token }} - # App installation tokens cannot call GET /user; the script derives the - # @codex review sender login from this slug as "[bot]" instead. - GH_APP_SLUG: ${{ steps.app-token.outputs.token && steps.app-token.outputs.app-slug || '' }} - PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} - REPO: ${{ github.repository }} - run: | - if [ ! -f .github/scripts/sync_codex_ok_labels.py ]; then - echo ".github/scripts/sync_codex_ok_labels.py is not available on the default branch yet; skipping bootstrap run" - exit 0 - fi - python3 .github/scripts/sync_codex_ok_labels.py --repo "$REPO" --pr "$PR_NUMBER" --apply --tolerate-write-permission-errors - - sync-after-ci: - name: Sync Codex labels after CI - runs-on: ubuntu-24.04 - if: >- - ${{ - github.event_name == 'schedule' || - ( - github.event_name == 'workflow_run' && - github.event.workflow_run.event == 'pull_request' - ) - }} - - steps: - - name: Checkout trusted base - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - persist-credentials: false - ref: ${{ github.event.repository.default_branch }} - - - name: Mint label sync App token - id: app-token - if: ${{ vars.CODEX_LABEL_SYNC_APP_ID != '' }} - continue-on-error: true - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 - with: - app-id: ${{ vars.CODEX_LABEL_SYNC_APP_ID }} - private-key: ${{ secrets.CODEX_LABEL_SYNC_APP_PRIVATE_KEY }} - permission-actions: write - permission-checks: read - permission-contents: read - permission-issues: write - permission-pull-requests: read - permission-statuses: read - - - name: Sync labels - env: - GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.CODEX_LABEL_SYNC_TOKEN || secrets.RELEASE_PLEASE_TOKEN || github.token }} - GH_FALLBACK_TOKEN: ${{ github.token }} - # App installation tokens cannot call GET /user; the script derives the - # @codex review sender login from this slug as "[bot]" instead. - GH_APP_SLUG: ${{ steps.app-token.outputs.token && steps.app-token.outputs.app-slug || '' }} - REPO: ${{ github.repository }} - run: | - if [ ! -f .github/scripts/sync_codex_ok_labels.py ]; then - echo ".github/scripts/sync_codex_ok_labels.py is not available on the default branch yet; skipping bootstrap run" - exit 0 - fi - python3 .github/scripts/sync_codex_ok_labels.py --repo "$REPO" --all-open --apply --tolerate-write-permission-errors --tolerate-read-errors diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 639c580268..f6556c25b8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -39,7 +39,7 @@ jobs: persist-credentials: false - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 80fc06b8cf..87bf575033 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -53,7 +53,7 @@ jobs: - name: Set up uv if: ${{ steps.release-branch.outputs.branch != '' }} - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f36b06ac1b..73bacc9796 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,7 +71,7 @@ jobs: run: cd frontend && bun run build - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: false @@ -212,7 +212,7 @@ jobs: ignore-unfixed: true - name: Upload Trivy scan results to GitHub Security - uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd if: always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) with: sarif_file: trivy-results.sarif diff --git a/.github/workflows/simplicity-budgets.yml b/.github/workflows/simplicity-budgets.yml index bf91705b60..b66baa6814 100644 --- a/.github/workflows/simplicity-budgets.yml +++ b/.github/workflows/simplicity-budgets.yml @@ -6,10 +6,8 @@ name: Simplicity budgets # This is deliberately a SEPARATE workflow from ci.yml, with different # pull_request trigger types: `labeled`/`unlabeled` re-evaluate the # simplicity-budget-approved override the moment a maintainer toggles it. -# Adding those types to ci.yml instead would re-run the full CI matrix on -# every label churn from codex-review-labels.yml (15-minute cron + -# workflow_run syncs of the codex labels), which is why they live here on a -# seconds-long job. +# They remain scoped to this seconds-long workflow so override-label changes +# do not re-run the full CI matrix. # # The override label is fetched live from the API (not read from the event # payload): fork PR payloads and re-runs of old runs would otherwise see a diff --git a/.github/workflows/windows-startup.yml b/.github/workflows/windows-startup.yml index d6110e57cf..f4bc0f1d4b 100644 --- a/.github/workflows/windows-startup.yml +++ b/.github/workflows/windows-startup.yml @@ -18,7 +18,7 @@ jobs: persist-credentials: false - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true diff --git a/.gitignore b/.gitignore index 92b567da04..77f8a1b4f0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ poc/ __pycache__/ *.py[cod] .pytest_cache/ +.hypothesis/ .mypy_cache/ .ruff_cache/ @@ -57,5 +58,6 @@ certs/ .omx/ PROMPT.md SUMMARY.md +DECISIONS.md .superpowers/ .agents/worktrees/ diff --git a/AGENTS.md b/AGENTS.md index 49ddd745f9..84a616afde 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,7 +67,7 @@ in [`.github/CONTRIBUTING.md`](.github/CONTRIBUTING.md). The sections an AI assistant most often needs are: - [Merge gates](.github/CONTRIBUTING.md#merge-gates) — CI green + - `@codex review` clean (or findings addressed) + `mergeable=CLEAN` + + actionable CodeRabbit findings addressed + `mergeable=CLEAN` + OpenSpec change folder for behavior changes + `Fixes #N` / `Closes #N` for issue cover + the five simplicity rules (PRINCIPLES.md P1-P5; see @@ -80,8 +80,8 @@ an AI assistant most often needs are: comment invoking the clause. An assistant preparing a merge MUST verify the gates against the -actual GitHub state (status check rollup, codex review submissions, -`mergeable` field) rather than asserting them from local history. +actual GitHub state (status check rollup, current-head CodeRabbit review +threads, `mergeable` field) rather than asserting them from local history. Local `uv run pytest` / `uv run ruff` / `codex review --base origin/main` are encouraged but not substitutes for the cloud gates. @@ -96,12 +96,10 @@ These rules encode recurring review blockers observed across codex-lb PRs. examples in `context.md` or change notes, and run strict OpenSpec validation before calling the PR ready. Code/tests alone are not enough when OpenSpec is required. -- Codex review state must come from current-head GitHub evidence. Check labels, - latest Codex review/comment/reaction, and GraphQL review threads before using - or claiming `🤖 codex: ok`. Usage-limit, environment, or missing-review - results mean missing evidence, not approval. Unresolved non-outdated P-level - Codex threads block readiness even when a top-level review comment looks - clean. +- CodeRabbit review state must come from current-head GitHub evidence. + Unresolved, non-outdated actionable review threads block readiness until + their findings are fixed or explicitly addressed or dismissed in-thread; + a top-level summary does not override active thread evidence. - Proxy failover and retry patches must prove account ownership and settlement invariants. File-pinned requests must not cross accounts; API-key reservations must settle before error-health writes; excluded accounts must actually leave diff --git a/CHANGELOG.md b/CHANGELOG.md index ade8935ba1..0eeaf3b439 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,120 @@ # Changelog +## [1.24.0](https://github.com/Soju06/codex-lb/compare/v1.23.0...v1.24.0) (2026-08-26) + + +### Features + +* **api-keys:** allow per-key reasoning effort policies ([#1642](https://github.com/Soju06/codex-lb/issues/1642)) ([ed31b7d](https://github.com/Soju06/codex-lb/commit/ed31b7da3d225aac30fc42c7c0d128f10955b58e)) +* **config:** timeout-invariant linter — validate deadline/TTL inequalities at startup and in CI ([#1622](https://github.com/Soju06/codex-lb/issues/1622)) ([d148dd9](https://github.com/Soju06/codex-lb/commit/d148dd9a42dca3088e8063aca8a21682f9bf7fb6)) +* **db:** report SQLite write transactions that outlive the busy timeout ([#1752](https://github.com/Soju06/codex-lb/issues/1752)) ([6464e96](https://github.com/Soju06/codex-lb/commit/6464e96f78bdc62c643d9285143b72152e9a0742)) +* **frontend:** configure model-source reasoning efforts ([#1848](https://github.com/Soju06/codex-lb/issues/1848)) ([eab7155](https://github.com/Soju06/codex-lb/commit/eab71553aee660fdb31122e9feb61a0e6c367904)) +* **model-sources:** advertise operator-declared reasoning efforts ([#1661](https://github.com/Soju06/codex-lb/issues/1661)) ([f1c8d5c](https://github.com/Soju06/codex-lb/commit/f1c8d5cd19947d76191fea8da8c07a69df493f83)) +* **model-sources:** embeddings source capability ([#1776](https://github.com/Soju06/codex-lb/issues/1776)) ([4d0f0ff](https://github.com/Soju06/codex-lb/commit/4d0f0ffc64df11a6800397497512abeb5478bce2)) +* **proxy:** report websocket cleanup phase ([#1726](https://github.com/Soju06/codex-lb/issues/1726)) ([0c8d921](https://github.com/Soju06/codex-lb/commit/0c8d921906735352ef60c0a445be455daa35249d)) +* **proxy:** support Ultrafast service tier ([#1734](https://github.com/Soju06/codex-lb/issues/1734)) ([d522a4d](https://github.com/Soju06/codex-lb/commit/d522a4de0740b530b22775ed6d2fe3959e3ca178)) +* **reports:** Add API Key Filtering to Reports Dashboard ([#1728](https://github.com/Soju06/codex-lb/issues/1728)) ([1f65f80](https://github.com/Soju06/codex-lb/commit/1f65f8093458d551979bd237d8304ed70b50c390)) +* **reset-credits:** add refresh scheduler enable toggle ([#1701](https://github.com/Soju06/codex-lb/issues/1701)) ([6509dd0](https://github.com/Soju06/codex-lb/commit/6509dd0d4a577908e5940f35ba4c5ab6d66f23bc)) +* **telemetry:** anonymous usage telemetry with informed opt-out consent ([#1618](https://github.com/Soju06/codex-lb/issues/1618)) ([debd7cf](https://github.com/Soju06/codex-lb/commit/debd7cf63c173e1e7b2982ff171bf1564c150c5a)) +* **telemetry:** report consent state and send a decision-time opt-out signal ([#1835](https://github.com/Soju06/codex-lb/issues/1835)) ([1541ee8](https://github.com/Soju06/codex-lb/commit/1541ee83105edd9a06062f6bcc617f9dab595a66)) +* **ui:** customize dashboard request-log columns ([#1503](https://github.com/Soju06/codex-lb/issues/1503)) ([138aa9f](https://github.com/Soju06/codex-lb/commit/138aa9f15c6ebea998335afae06475d8834fe7d6)) +* **ui:** surface reasoning token usage ([#1801](https://github.com/Soju06/codex-lb/issues/1801)) ([8e7589e](https://github.com/Soju06/codex-lb/commit/8e7589e4286869b9030eb1b19726297c63852ed3)) + + +### Bug Fixes + +* **accounts:** recover Free accounts after reset ([#1700](https://github.com/Soju06/codex-lb/issues/1700)) ([b43d0c8](https://github.com/Soju06/codex-lb/commit/b43d0c8bb3101932436dc9ff6fadb9034b4d3b03)) +* **auth:** guard the refresh singleflight negative cache by successor ownership ([#1652](https://github.com/Soju06/codex-lb/issues/1652)) ([4ace71e](https://github.com/Soju06/codex-lb/commit/4ace71e8044e5180905f7a53f2d00d010cd43233)) +* **cache:** keep an aborted invalidation bump queued ([#1748](https://github.com/Soju06/codex-lb/issues/1748)) ([7148810](https://github.com/Soju06/codex-lb/commit/71488100f71b5b132ed9c86b8cddbb63991842a9)) +* **chat:** omit unset tools on mapped Responses payloads ([#1725](https://github.com/Soju06/codex-lb/issues/1725)) ([db5776c](https://github.com/Soju06/codex-lb/commit/db5776c288b5c709d665c1dcaa9298ae8c390e6a)) +* **compact:** omit oversized non-state tool tail ([#1235](https://github.com/Soju06/codex-lb/issues/1235)) ([edb3734](https://github.com/Soju06/codex-lb/commit/edb37348f19a5015c3b90e0509ed92179e3716da)) +* **compact:** recover previous-response-pinned compaction from quota-excluded owners ([#1780](https://github.com/Soju06/codex-lb/issues/1780)) ([120af75](https://github.com/Soju06/codex-lb/commit/120af7520cae7d51fb66c731d9d4578e416ce93f)) +* **dashboard:** distinguish first-run empty states from filter mismatch ([#1729](https://github.com/Soju06/codex-lb/issues/1729)) ([e439043](https://github.com/Soju06/codex-lb/commit/e439043d4ad2869ca992d3d9bbb632b814406013)) +* **dashboard:** exclude cancelled/client_disconnected from error rate ([#1696](https://github.com/Soju06/codex-lb/issues/1696)) ([8a7d956](https://github.com/Soju06/codex-lb/commit/8a7d9560571478a435f66f082b7656a6e9313f14)) +* **dashboard:** pin web asset MIME types against poisoned OS registries ([#1709](https://github.com/Soju06/codex-lb/issues/1709)) ([2164b8c](https://github.com/Soju06/codex-lb/commit/2164b8c0656b3a8539ccdf8a4a8655a788af9ae8)), closes [#1698](https://github.com/Soju06/codex-lb/issues/1698) +* **dashboard:** preserve cancelled request count ([#1766](https://github.com/Soju06/codex-lb/issues/1766)) ([ef9c68e](https://github.com/Soju06/codex-lb/commit/ef9c68e0fbc553ff83c374e4dcb7a3c079b163a9)) +* **dashboard:** separate quota and purchased credits ([#1670](https://github.com/Soju06/codex-lb/issues/1670)) ([4a08d96](https://github.com/Soju06/codex-lb/commit/4a08d968f34aeec423892c02775769335590619f)) +* **dashboard:** show cancellation totals in reports ([#1772](https://github.com/Soju06/codex-lb/issues/1772)) ([5d27f7f](https://github.com/Soju06/codex-lb/commit/5d27f7f0cc2fea37ce2e3786ea08660f008dd694)) +* **dashboard:** show cancelled request logs ([#1769](https://github.com/Soju06/codex-lb/issues/1769)) ([5f2f726](https://github.com/Soju06/codex-lb/commit/5f2f7266163f93b8fe5df6295fae6342bc51945b)) +* **dashboard:** surface upstream route metadata ([#1767](https://github.com/Soju06/codex-lb/issues/1767)) ([e359d49](https://github.com/Soju06/codex-lb/commit/e359d490befc87d27f198b528a67685ee7a8503e)) +* **db:** add postgres shm_size and raise default pool headroom ([#1791](https://github.com/Soju06/codex-lb/issues/1791)) ([539cf93](https://github.com/Soju06/codex-lb/commit/539cf934ea654efbc915bad17ea84a87c07afa4d)) +* **db:** bound wedged SQLite session teardown and reclaim the connection ([#1778](https://github.com/Soju06/codex-lb/issues/1778)) ([9eedb2c](https://github.com/Soju06/codex-lb/commit/9eedb2c8f4aa809715484e1c7607e6f0219b4ccf)) +* **db:** repair retired identity/warmup migration stamp ([#1847](https://github.com/Soju06/codex-lb/issues/1847)) ([b6c217f](https://github.com/Soju06/codex-lb/commit/b6c217fada24c8a7b7e2c77af6bce3e2d7a2d3e2)) +* **docker:** upgrade util-linux family in runtime image for CVE-2026-53615 ([#1796](https://github.com/Soju06/codex-lb/issues/1796)) ([0a4c0a1](https://github.com/Soju06/codex-lb/commit/0a4c0a1071cfe2e91357d8dc34b433cb511b1aec)) +* **helm:** bind TTFT dashboard SQL datasource ([#1827](https://github.com/Soju06/codex-lb/issues/1827)) ([8abd507](https://github.com/Soju06/codex-lb/commit/8abd50778dd15131fac01f6a40d8e07a1bcebf54)) +* **http-bridge:** classify recovery error frames and poison same-anchor eventless failures ([#1841](https://github.com/Soju06/codex-lb/issues/1841)) ([01f089c](https://github.com/Soju06/codex-lb/commit/01f089c359dadc2cc75b0719addc646e116624a5)) +* **http-bridge:** dedupe retry circuit failures per send ([#1743](https://github.com/Soju06/codex-lb/issues/1743)) ([5780a27](https://github.com/Soju06/codex-lb/commit/5780a27f8c77f033ece2d144cb25ff89fb9db679)) +* **http-bridge:** keep idle retirements out of retry circuit ([#1677](https://github.com/Soju06/codex-lb/issues/1677)) ([7c46719](https://github.com/Soju06/codex-lb/commit/7c4671980094135b9094278d2ebb374c7cb22655)) +* **http-bridge:** keep missing-created watchdog armed after prelude and handle stale API-key activity ([#1580](https://github.com/Soju06/codex-lb/issues/1580)) ([0eb0ee7](https://github.com/Soju06/codex-lb/commit/0eb0ee7939309dfd264f56a88d1bbcb519fde5c6)) +* **http-bridge:** preserve goal-restart recovery across reconnects ([#1680](https://github.com/Soju06/codex-lb/issues/1680)) ([5dc6081](https://github.com/Soju06/codex-lb/commit/5dc6081e41b7abe8670ea4565758246ef9f173b9)) +* **http-bridge:** refuse foreign claims on live DRAINING leases ([#1722](https://github.com/Soju06/codex-lb/issues/1722)) ([b50cb86](https://github.com/Soju06/codex-lb/commit/b50cb86659f0cebdc4abd8e200556a54e62e17da)) +* **models:** apply context-window overrides to /v1 input context fields ([#1808](https://github.com/Soju06/codex-lb/issues/1808)) ([c750dcf](https://github.com/Soju06/codex-lb/commit/c750dcfe64961c7d538c75367e9ee509fe8c9052)) +* **models:** correct GPT-5.6 context windows ([#1691](https://github.com/Soju06/codex-lb/issues/1691)) ([8488bc4](https://github.com/Soju06/codex-lb/commit/8488bc462a46be07ae70f805ff6bf351f0ba4d97)) +* **models:** raise GPT-5.6 bootstrap max_context_window to 872k ([#1813](https://github.com/Soju06/codex-lb/issues/1813)) ([1add104](https://github.com/Soju06/codex-lb/commit/1add1041b61e7b20ea104a91dd6331f03db904c7)) +* **proxy:** abandon unavailable owner on thread-scoped goal restart ([#1764](https://github.com/Soju06/codex-lb/issues/1764)) ([17ae866](https://github.com/Soju06/codex-lb/commit/17ae866e2f6fd3d2daa1f1c4a0b2a8d0a5d2f25b)) +* **proxy:** absorb replay-safe compaction recovery ([#1849](https://github.com/Soju06/codex-lb/issues/1849)) ([c597226](https://github.com/Soju06/codex-lb/commit/c597226cf139ec1e80117b72f02b7a18bef3645f)) +* **proxy:** add explicit Daybreak capability routing ([#1742](https://github.com/Soju06/codex-lb/issues/1742)) ([0031e3d](https://github.com/Soju06/codex-lb/commit/0031e3d468747d63d8573139eabcaa0594891aef)) +* **proxy:** bind account-bound retries to dispatch owner ([#1829](https://github.com/Soju06/codex-lb/issues/1829)) ([3381938](https://github.com/Soju06/codex-lb/commit/3381938aa278d7f3cd371bdd76c7914856586bf8)) +* **proxy:** classify parameterless previous response errors ([#1818](https://github.com/Soju06/codex-lb/issues/1818)) ([eeab46a](https://github.com/Soju06/codex-lb/commit/eeab46a5edf5be16ff2915edc21a7f6a9a424717)) +* **proxy:** close non-stream chat collect and map error status ([#1712](https://github.com/Soju06/codex-lb/issues/1712)) ([a85f71d](https://github.com/Soju06/codex-lb/commit/a85f71dbec6f56d417bd663c3e63e7431267554e)) +* **proxy:** compact transport switch + trigger canonicalization (supersedes [#1749](https://github.com/Soju06/codex-lb/issues/1749)) ([#1809](https://github.com/Soju06/codex-lb/issues/1809)) ([0481ed9](https://github.com/Soju06/codex-lb/commit/0481ed996ab128ae67ff311a9c69699e98890d7b)) +* **proxy:** complete disconnect cleanup — pool leak, charged reservation, mutable terminal reason ([#1645](https://github.com/Soju06/codex-lb/issues/1645)) ([6cf7e61](https://github.com/Soju06/codex-lb/commit/6cf7e61d7719654a62fb3132a21ae5f19ad8dba1)) +* **proxy:** demote quarantined bridge reattach keys ([#1730](https://github.com/Soju06/codex-lb/issues/1730)) ([5e1f568](https://github.com/Soju06/codex-lb/commit/5e1f568f3a2772b4d1162a5e325d056cae7daa39)) +* **proxy:** do not rewrite thread locality for a file-pin owner ([#1765](https://github.com/Soju06/codex-lb/issues/1765)) ([34ef7b2](https://github.com/Soju06/codex-lb/commit/34ef7b262ecabc597be0c7dd73978dc24973ee82)) +* **proxy:** drop malformed compact item ids ([#1815](https://github.com/Soju06/codex-lb/issues/1815)) ([812265d](https://github.com/Soju06/codex-lb/commit/812265d11c0faa470da1db1b689442afa2b93869)) +* **proxy:** durably recover hard HTTP bridge operations ([#1657](https://github.com/Soju06/codex-lb/issues/1657)) ([7a0b671](https://github.com/Soju06/codex-lb/commit/7a0b67192140ab307b911719189c52f4fa87033d)) +* **proxy:** fence successor bridge claims against the retiring predecessor ([#1751](https://github.com/Soju06/codex-lb/issues/1751)) ([2c0dc5b](https://github.com/Soju06/codex-lb/commit/2c0dc5b8eec16d3e8c413f144cd62c583f43847d)) +* **proxy:** guard model-transition owner-conflict fork ([#1619](https://github.com/Soju06/codex-lb/issues/1619)) ([52092bc](https://github.com/Soju06/codex-lb/commit/52092bc91fd81d7a18655eab403f02ff6895dd80)) +* **proxy:** hold fenced hard turns through cooldown ([#1739](https://github.com/Soju06/codex-lb/issues/1739)) ([6ff51cd](https://github.com/Soju06/codex-lb/commit/6ff51cd69c564499e4371ee755eecab8d80b262d)) +* **proxy:** keep abrupt eventless websocket drops account-neutral ([#1777](https://github.com/Soju06/codex-lb/issues/1777)) ([6c97ad6](https://github.com/Soju06/codex-lb/commit/6c97ad6265100c4ac3489e15a247d73ab45866fb)) +* **proxy:** keep file-pin owner on soft 1011 reconnect ([#1761](https://github.com/Soju06/codex-lb/issues/1761)) ([f694c44](https://github.com/Soju06/codex-lb/commit/f694c449479dd1b204e93ee27bf599f9a5c6b86c)) +* **proxy:** keep stream idle timeouts account-neutral ([#1718](https://github.com/Soju06/codex-lb/issues/1718)) ([64da340](https://github.com/Soju06/codex-lb/commit/64da340ab72580d8762ba9cda8a1fed9c9bb30be)) +* **proxy:** normalize single-account warmup failures ([#1774](https://github.com/Soju06/codex-lb/issues/1774)) ([f92bc90](https://github.com/Soju06/codex-lb/commit/f92bc906ee06079e307866cff548b75435b46c49)) +* **proxy:** O(1) shared-future admission waits + event-loop lag watchdog ([#1842](https://github.com/Soju06/codex-lb/issues/1842)) ([ed2c94d](https://github.com/Soju06/codex-lb/commit/ed2c94d4b8ece64455233e5293d44ec0f263e6bc)) +* **proxy:** persist file ownership across replicas ([#1521](https://github.com/Soju06/codex-lb/issues/1521)) ([2cd52e4](https://github.com/Soju06/codex-lb/commit/2cd52e44b4136bdc76b425cca1bd767335f43754)) +* **proxy:** preserve compact terminal error type ([#1824](https://github.com/Soju06/codex-lb/issues/1824)) ([78d63e5](https://github.com/Soju06/codex-lb/commit/78d63e5a840c1cc6a34a03cb002eac9b3e041f7e)) +* **proxy:** reject truncated chat completion streams ([#1833](https://github.com/Soju06/codex-lb/issues/1833)) ([6ba083d](https://github.com/Soju06/codex-lb/commit/6ba083d7df4f82c2d2e6aa08e9a1e1fb47b59faa)) +* **proxy:** release the API-key reservation on all exits of the models endpoints ([#1653](https://github.com/Soju06/codex-lb/issues/1653)) ([7007885](https://github.com/Soju06/codex-lb/commit/7007885dad6572e2332739f808528c5b8b4a0857)) +* **proxy:** report suppressed duplicate tool-call terminals ([#1706](https://github.com/Soju06/codex-lb/issues/1706)) ([25d6374](https://github.com/Soju06/codex-lb/commit/25d6374a8a671900a6bf4dc89f248e7834efad76)) +* **proxy:** retain image reservation recovery ownership ([#1822](https://github.com/Soju06/codex-lb/issues/1822)) ([bd67c64](https://github.com/Soju06/codex-lb/commit/bd67c640012692786f6beddd3d61c67cea759c47)) +* **proxy:** route source-owned models off the WebSocket transport ([#1659](https://github.com/Soju06/codex-lb/issues/1659)) ([08b84a9](https://github.com/Soju06/codex-lb/commit/08b84a95ad5d4de88b3ac4ebb37185781a603f81)) +* **proxy:** scope backend Codex affinity by thread identity ([#1703](https://github.com/Soju06/codex-lb/issues/1703)) ([35bbb00](https://github.com/Soju06/codex-lb/commit/35bbb006bc2ec76a43273f08f5a29ea150f11f4c)) +* **proxy:** separate websocket scope cleanup budget ([#1723](https://github.com/Soju06/codex-lb/issues/1723)) ([fd97cb8](https://github.com/Soju06/codex-lb/commit/fd97cb856970e46a5e6e4265e0064fca4f24fb02)) +* **proxy:** settle compact failover before account health ([#1717](https://github.com/Soju06/codex-lb/issues/1717)) ([3093203](https://github.com/Soju06/codex-lb/commit/30932034c3188efbddb33a68e0c438bd5db87db3)) +* **proxy:** settle terminal spool append failures ([#1775](https://github.com/Soju06/codex-lb/issues/1775)) ([4e48f35](https://github.com/Soju06/codex-lb/commit/4e48f355b519fb20e16e89a5e5c6b2375bb08161)) +* **proxy:** stop abandoning an unresolved inflight session-creation future ([#1644](https://github.com/Soju06/codex-lb/issues/1644)) ([57618c8](https://github.com/Soju06/codex-lb/commit/57618c87528aaaac4fecc4c6ad57dd07fbfa108b)) +* **proxy:** sweep idle bridge sessions without request traffic ([#1747](https://github.com/Soju06/codex-lb/issues/1747)) ([3159ebe](https://github.com/Soju06/codex-lb/commit/3159ebedfc48789ee6b9c678c4f48e842a2a3555)) +* **proxy:** wait on usage-refresh singleflight without asyncio.shield ([#1897](https://github.com/Soju06/codex-lb/issues/1897)) ([798203f](https://github.com/Soju06/codex-lb/commit/798203ff9d9d8f30a9b53e181d34fc9935ba5444)), closes [#1896](https://github.com/Soju06/codex-lb/issues/1896) +* **quota-planner:** compare warmup reset epochs in UTC ([#1623](https://github.com/Soju06/codex-lb/issues/1623)) ([e4fa3f2](https://github.com/Soju06/codex-lb/commit/e4fa3f273f45ac9eaeafc28047005584c954ef3c)) +* **reports:** format full Cost values with grouping separators ([#1814](https://github.com/Soju06/codex-lb/issues/1814)) ([028a75c](https://github.com/Soju06/codex-lb/commit/028a75c33701494834c054718fb58e31d72c4d99)) +* **review:** keep Codex review sessions resumable ([#1678](https://github.com/Soju06/codex-lb/issues/1678)) ([e34db2d](https://github.com/Soju06/codex-lb/commit/e34db2d218925f7764e57b0571dcf736d33084da)) +* **server:** serve h2c upgrade offers as plain HTTP/1.1 instead of rejecting them ([#1782](https://github.com/Soju06/codex-lb/issues/1782)) ([8d265c3](https://github.com/Soju06/codex-lb/commit/8d265c3f73bda4c2adb7632d888dad5413b1b121)) +* **usage:** fence leaked live-usage-ingestor tasks and settle their failures deterministically ([#1783](https://github.com/Soju06/codex-lb/issues/1783)) ([66fd103](https://github.com/Soju06/codex-lb/commit/66fd1033165133e943a5818d75c40bc86e4a6b49)) +* **usage:** settle live snapshots after account consolidation ([#1773](https://github.com/Soju06/codex-lb/issues/1773)) ([3f66c28](https://github.com/Soju06/codex-lb/commit/3f66c288a230d6eb73c39b397c558427c21e167f)) +* **warmup:** warm paid-to-free transitions ([#1825](https://github.com/Soju06/codex-lb/issues/1825)) ([68892e7](https://github.com/Soju06/codex-lb/commit/68892e7afff21cff8910e2ee5456317eff6e231b)) + + +### Performance Improvements + +* **accounts:** bound the account-listing live tail with a 2h fold lag and a 30s summary cache ([#1792](https://github.com/Soju06/codex-lb/issues/1792)) ([c1caa44](https://github.com/Soju06/codex-lb/commit/c1caa4468cdf2f94f9abe675b63e63a13dbdd383)) +* **accounts:** make account deletion a fast mark + background batch drain ([#1795](https://github.com/Soju06/codex-lb/issues/1795)) ([d4f9e23](https://github.com/Soju06/codex-lb/commit/d4f9e23cd623d67beee2df153723839e391d44d1)) +* **api-keys,proxy:** shape ORM hot-path queries ([#1788](https://github.com/Soju06/codex-lb/issues/1788)) ([7dacb04](https://github.com/Soju06/codex-lb/commit/7dacb04181390b85bd42b335a73e27ad4d90ec2f)) +* **api-keys:** skip usage reservations when no limit applies ([#1789](https://github.com/Soju06/codex-lb/issues/1789)) ([8a2d066](https://github.com/Soju06/codex-lb/commit/8a2d0660e2b216a93593757f2fec242e69f92724)) +* coalesce same-owner sticky session TTL refresh upserts ([#1790](https://github.com/Soju06/codex-lb/issues/1790)) ([076aab8](https://github.com/Soju06/codex-lb/commit/076aab854ff0b334d77ab2104175d672384065c5)) +* **dashboard:** cap projections bulk usage-history read per account ([#1779](https://github.com/Soju06/codex-lb/issues/1779)) ([d4c43ef](https://github.com/Soju06/codex-lb/commit/d4c43ef88f8d4a548fb952a82901ea482995a633)) +* **middleware:** convert BaseHTTPMiddleware layers to pure ASGI ([#1787](https://github.com/Soju06/codex-lb/issues/1787)) ([94057cc](https://github.com/Soju06/codex-lb/commit/94057ccf91a7ed9f32eb15dcfa0487c969dc2a83)) +* **proxy:** disable permessage-deflate on direct-egress upstream websockets ([#1786](https://github.com/Soju06/codex-lb/issues/1786)) ([2e4a580](https://github.com/Soju06/codex-lb/commit/2e4a580c1c1834f00b6d4c62598caf1ffd0446ee)) +* **proxy:** relay unmodified SSE frames verbatim ([#1785](https://github.com/Soju06/codex-lb/issues/1785)) ([980572e](https://github.com/Soju06/codex-lb/commit/980572eb8ee5ed70acb9edfc5a7a2acc35f46216)) +* **proxy:** validate stream payloads only for lifecycle events ([#1784](https://github.com/Soju06/codex-lb/issues/1784)) ([9d9f099](https://github.com/Soju06/codex-lb/commit/9d9f099197326f37eea4042e27c5b09606f99043)) + + +### Documentation + +* **dashboard:** clarify routing, sticky affinity, quota thresholds, warm-up, and eligibility copy ([#1781](https://github.com/Soju06/codex-lb/issues/1781)) ([6ff22e0](https://github.com/Soju06/codex-lb/commit/6ff22e0e528fb7bdd6c69c059178681254b143af)) +* **openspec:** archive 90 landed changes and sync their specs ([#1713](https://github.com/Soju06/codex-lb/issues/1713)) ([c3f0c56](https://github.com/Soju06/codex-lb/commit/c3f0c568cb4dda4547f5d765951ba0748e7c923a)) +* **openspec:** archive landed performance and reliability changes ([#1694](https://github.com/Soju06/codex-lb/issues/1694)) ([6b3db74](https://github.com/Soju06/codex-lb/commit/6b3db74e7b8a8201f6362e06a7debb517e48db27)) +* **proxy:** document cluster-wide account cap partitioning ([#1750](https://github.com/Soju06/codex-lb/issues/1750)) ([560fb50](https://github.com/Soju06/codex-lb/commit/560fb503b3ada0e5a544cd3de3af42b3c5e43ebe)) + ## [1.23.0](https://github.com/Soju06/codex-lb/compare/v1.22.0...v1.23.0) (2026-08-11) diff --git a/Dockerfile b/Dockerfile index 9b6d0d83be..a7387c6998 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1.7 -FROM ghcr.io/astral-sh/uv:0.12.3 AS uv-bin +FROM ghcr.io/astral-sh/uv:0.12.5 AS uv-bin FROM oven/bun:1.3.14-alpine AS frontend-build @@ -40,9 +40,8 @@ WORKDIR /app RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --only-upgrade \ - bsdutils libc-bin libc6 libblkid1 libcap2 liblastlog2-2 libmount1 \ - libsmartcols1 libssl3t64 libsystemd0 libudev1 libuuid1 login mount \ - openssl sed util-linux \ + bsdutils libblkid1 libc-bin libc6 libcap2 libmount1 libsmartcols1 libssl3t64 \ + libsystemd0 libudev1 libuuid1 openssl sed util-linux \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ openssl-provider-legacy \ && rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile.distroless b/Dockerfile.distroless index 5b34691f0e..a3e61efad8 100644 --- a/Dockerfile.distroless +++ b/Dockerfile.distroless @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1.7 -FROM ghcr.io/astral-sh/uv:0.12.3 AS uv-bin +FROM ghcr.io/astral-sh/uv:0.12.5 AS uv-bin FROM oven/bun:1.3.14-alpine AS frontend-build diff --git a/Makefile b/Makefile index 9e5c571958..6e11247a63 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ POSTGRES_PYTEST_TARGETS := \ tests/integration/test_db_commit_durability.py \ tests/test_request_logs_options_api.py \ tests/integration/test_account_usage_rollup.py \ + tests/integration/test_account_deletion_background.py \ tests/integration/test_request_usage_time_rollup.py \ tests/integration/test_request_usage_rollup_parity.py \ tests/integration/test_migrations.py::test_request_usage_time_rollups_migration_upgrade_and_downgrade \ @@ -33,10 +34,16 @@ POSTGRES_PYTEST_TARGETS := \ tests/integration/test_repositories.py::test_replace_reauthorized_discards_pending_downgrade_evidence \ tests/integration/test_repositories.py::test_upsert_account_slot_discards_pending_downgrade_evidence_on_reimport \ tests/integration/test_migrations.py::test_account_plan_downgrade_observations_migration_upgrade_and_downgrade \ + tests/integration/test_migrations.py::test_account_pending_deletion_migration_upgrade_and_downgrade \ tests/integration/test_usage_repository.py::test_bulk_history_since_primary_query_plan_is_index_only_postgresql \ tests/integration/test_usage_repository.py::test_bulk_history_since_cutoff_query_plan_is_index_only_postgresql \ tests/integration/test_usage_repository.py::test_bulk_history_since_secondary_query_plan_is_index_only_postgresql \ tests/integration/test_usage_repository.py::test_bulk_history_since_covered_read_matches_non_covered_read_postgresql \ + tests/integration/test_usage_repository.py::test_bulk_history_since_per_account_row_cap_keeps_newest_rows \ + tests/integration/test_usage_repository.py::test_bulk_history_since_row_cap_respects_per_account_cutoffs_postgresql \ + tests/integration/test_usage_repository.py::test_bulk_history_since_row_cap_exempts_uncapped_recent_floor_postgresql \ + tests/integration/test_usage_repository.py::test_bulk_history_since_capped_query_plan_is_index_only_postgresql \ + tests/integration/test_usage_repository.py::test_bulk_history_since_capped_floor_query_plan_is_index_only_postgresql \ tests/integration/test_migrations.py::test_usage_history_bulk_covering_indexes_migration_upgrade_and_downgrade \ tests/integration/test_migrations.py::test_usage_history_covering_index_migration_repairs_invalid_leftover_postgresql \ tests/integration/test_migrations.py::test_usage_history_autovacuum_tuning_migration_sets_and_resets_reloptions_postgresql @@ -90,7 +97,7 @@ lint: architecture-check uv run ruff format --check . architecture-check: - python scripts/check_proxy_architecture.py + uv run python scripts/check_proxy_architecture.py typecheck: uv sync --dev --frozen @@ -114,9 +121,9 @@ test-integration-core: frontend-build # guards that the shards always partition the full selection exactly. test-integration-core-shard: frontend-build uv sync --dev --frozen - python .github/scripts/pytest_shards.py --shard-count $(INTEGRATION_CORE_SHARD_COUNT) --verify + uv run python .github/scripts/pytest_shards.py --shard-count $(INTEGRATION_CORE_SHARD_COUNT) --verify PYTHONFAULTHANDLER=1 uv run pytest $(PYTEST_ARGS) \ - $$(python .github/scripts/pytest_shards.py --shard-count $(INTEGRATION_CORE_SHARD_COUNT) --shard $(SHARD)) + $$(uv run python .github/scripts/pytest_shards.py --shard-count $(INTEGRATION_CORE_SHARD_COUNT) --shard $(SHARD)) test-integration-core-1: $(MAKE) test-integration-core-shard SHARD=1 @@ -163,7 +170,7 @@ package: frontend-build uv run python -c "import app; import app.main; print('import ok')" rm -rf build dist *.egg-info uvx --from build==1.3.0 python -m build - python scripts/verify-wheel-assets.py + uv run python scripts/verify-wheel-assets.py .PHONY: docker docker: diff --git a/README.md b/README.md index 6b0bd131aa..a2a3e1b71e 100644 --- a/README.md +++ b/README.md @@ -282,9 +282,23 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/e rknightion
rknightion
💻 ⚠️ glopyglerky
glopyglerky
💻 ⚠️ Ahmad Maulana Iqbal
Ahmad Maulana Iqbal
💻 ⚠️ + Dvredin
Dvredin
💻 - Jason HONG
Jason HONG
💻 ⚠️ 📖 + DuyBui
DuyBui
💻 ⚠️ + Kevin Lin
Kevin Lin
💻 ⚠️ + Borealin
Borealin
💻 ⚠️ + BrenticusMaximus
BrenticusMaximus
💻 ⚠️ + Sakthimaran
Sakthimaran
💻 ⚠️ + Evan
Evan
💻 + Chao Xu
Chao Xu
💻 ⚠️ + + + zenasharp
zenasharp
💻 ⚠️ 📖 + HanSu Lee
HanSu Lee
💻 ⚠️ + + + Jason HONG
Jason HONG
💻 ⚠️ 🚧 diff --git a/SUMMARY.md b/SUMMARY.md deleted file mode 100644 index 79c3a6b038..0000000000 --- a/SUMMARY.md +++ /dev/null @@ -1,140 +0,0 @@ -# Summary - -## Root cause - -This bug was a three-fault chain: - -1. `/v1/responses` payloads carrying `{"type":"input_image","file_id":"file_*"}` or `{"type":"input_image","image_url":"sediment://file_*"}` were forwarded upstream even though the Responses surface only accepts inline `data:` URLs for conversation `input_image` parts. -2. codex-lb persisted only `file_id -> account_id`, so after `/backend-api/files/{file_id}/uploaded` completed it had no stored `download_url` / `mime_type` to pull the uploaded bytes back and rewrite them into the codex-style inline image form. -3. When upstream rejected that bad shape, the HTTP responses bridge saw a clean close (`close_code=1000`) with zero `response.*` events and treated it as transient, looping through `retry_precreated` / `retry_fresh_upstream` until the request budget expired. - -## What changed - -### `app/core/clients/proxy.py` - -- Added `_ws_transport_payload_budget_bytes(settings)` so auto transport selection respects the deploy's `max_sse_event_bytes` with 2 MiB headroom for the websocket envelope and control frames. -- `stream_responses()` now computes the post-inline serialized payload size immediately after `_inline_input_image_urls()`, covering both: - - `app/modules/proxy/service.py::_rewrite_input_image_file_references` - - `app/core/clients/proxy.py::_inline_input_image_urls` -- `_resolve_stream_transport()` now routes `auto` requests over HTTP before the existing codex-header / model-registry websocket heuristics when that rewritten payload estimate exceeds the websocket budget. -- Explicit `upstream_stream_transport = "websocket"` and `upstream_stream_transport = "http"` still win unchanged. - -### `app/core/clients/image_processor.py` - -- Added a new codex-faithful prompt image processor. -- Mirrors the upstream codex image contract: - - accepts only PNG / JPEG / GIF / WebP - - preserves PNG / JPEG / WebP bytes verbatim when already within 2048x2048 - - re-encodes GIF as PNG - - resizes oversized images to fit 2048x2048 - - uses JPEG quality 85 and lossless WebP on resized output -- Adds a 32-entry in-process LRU cache keyed by `sha1(bytes) + mode`. - -### `app/core/clients/files.py` - -- Added `fetch_file_bytes(download_url, expected_mime, max_bytes)`. -- Downloads finalize SAS blobs with a hard byte cap so a single attachment cannot blow the websocket frame budget after base64 expansion. - -### `app/core/openai/requests.py` - -- Added `_input_image_file_reference()` for: - - `input_image.file_id` - - `input_image.image_url = "sediment://file_*"` -- Extended `extract_input_file_ids()` so routing sees both `input_file` and uploaded `input_image` references. -- Added `extract_input_image_file_references()` so the proxy can rewrite only the precise `input_image` parts, without touching any other conversation content. - -### `app/modules/proxy/service.py` - -- Replaced the old tuple pin with `_FilePinEntry(account_id, download_url, mime_type, file_name, expires_at)`. -- `create_file()` still pins the upload owner immediately so finalize stays on the same upstream account. -- `finalize_file()` now upgrades the pin with `download_url` / `mime_type` / `file_name` once upstream returns `status=success`. -- Pin expiry is clamped to the shorter of: - - `_FILE_ACCOUNT_PIN_TTL_SECONDS` (30 minutes) - - the SAS `se=` expiry embedded in `download_url`, when present -- Added `_lookup_file_pin()`. -- Added `_rewrite_input_image_file_references()`: - - finds only `input_image.file_id` / `sediment://file_*` - - fetches the uploaded bytes from the pinned SAS `download_url` - - runs the codex-faithful image processor - - rewrites the original part to inline `image_url: "data:..."`, preserving `detail` when supplied and defaulting it to `auto` otherwise - - leaves all non-targeted conversation content byte-for-byte untouched - - logs a synthetic `image-inline-rewrite` request-log row for observability -- Wired the rewrite into: - - HTTP `/v1/responses` / backend responses streaming path - - HTTP bridge path - - websocket `response.create` prepare path - - `/responses/compact` -- Added `_classify_upstream_close()` and `response_event_count` tracking. -- HTTP bridge `retry_precreated` now fails fast with `502 upstream_rejected_input` when upstream closes with `close_code=1000` before any `response.*` event. -- `stream_http_responses()` now rewrites uploaded `input_image` references before branch selection, estimates the post-rewrite JSON payload size, and bypasses the HTTP responses bridge per request when that rewritten body exceeds the WebSocket frame budget. -- The bypass uses a local `dataclasses.replace(runtime_config, enabled=False)` copy only, so bridge state stays unchanged globally and smaller follow-up requests still use the bridge normally. - -### `tests/unit/test_image_processor.py` - -- Added coverage for passthrough, resize, GIF->PNG re-encode, unsupported formats, garbage bytes, ORIGINAL mode, and cache-hit identity. - -### `tests/unit/test_files_client.py` - -- Added coverage for `fetch_file_bytes()` success and `file_too_large` enforcement. - -### `tests/unit/test_openai_requests.py` - -- Added coverage for `input_image.file_id`, `sediment://file_*`, and `extract_input_image_file_references()`. - -### `tests/unit/test_proxy_utils.py` - -- Added coverage for: - - `_lookup_file_pin()` - - `_rewrite_input_image_file_references()` single and multiple rewrites - - missing pin -> `400 file_not_found` - - oversized download -> `400 file_too_large` - - preserving non-image conversation content - - returning the pinned account for routing - - clean-close classifier - - HTTP bridge precreated retry suppression on rejected input - - large rewritten payloads forcing HTTP only in `auto` - - large rewritten payloads bypassing the HTTP responses bridge selector - - smaller / unknown payload sizes preserving websocket preference - - explicit transport overrides still winning - - websocket budget calculation from `max_sse_event_bytes` - -### OpenSpec - -- Amended `openspec/changes/add-backend-api-files-protocol/`: - - `proposal.md` - - `tasks.md` - - `specs/responses-api-compat/spec.md` -- Documented accepted `input_file` / uploaded `input_image` shapes, the inline rewrite contract, the 16 MiB cap, the “rewrite only the targeted `input_image` parts” rule, the auto HTTP fallback for oversized rewritten payloads, and the clean-close fail-fast behavior. -- Added the bridge-bypass scenario so the OpenSpec now covers the default bridge-enabled `/responses` path as well as `_resolve_stream_transport()`. - -### Dependency / lockfile - -- `pyproject.toml` now declares `pillow>=10.0`. -- `uv.lock` was updated so the direct dependency is in sync. -- Pillow was added explicitly even though it was already present transitively because this code now imports `from PIL import Image` directly in production. - -## Caveats - -- SAS expiry vs pin TTL: - - file pins now expire at the earlier of 30 minutes or the SAS `se=` timestamp when present - - if the SAS URL expires before the follow-up `/responses` call arrives, inline rewrite fails closed instead of attempting a stale fetch -- Cache misses: - - the image processor cache is in-process only - - a different worker or a cold process simply re-downloads and re-processes the image -- Partial multi-image rewrites: - - if any referenced upload pin is missing / expired / unfetchable, the whole request fails - - there is no partial-forward behavior - -## Verification - -- `uv run --frozen ruff check app tests` -- `uv run --frozen ruff format --check app tests` -- `uv run --frozen ty check app` -- `uv run --frozen pytest tests/unit -q` -- `uv run --frozen pytest tests/integration/test_proxy_files.py -q` -- `uv run --frozen pytest tests/integration/test_proxy_responses.py -q` - -## Could not verify - -- `openspec validate add-backend-api-files-protocol --strict --no-interactive` - - the `openspec` CLI is not installed in this workspace (`openspec: command not found`) diff --git a/app/cli.py b/app/cli.py index f5d19146c5..93dae15a68 100644 --- a/app/cli.py +++ b/app/cli.py @@ -125,6 +125,12 @@ def _load_graceful_drain_server(): return GracefulDrainServer +def _load_http_protocol_class() -> Any: + from app.core.http_protocol import load_http_protocol_class + + return load_http_protocol_class() + + def _load_shutdown_drain_timeout_seconds() -> int: from app.core.config.settings import get_settings @@ -140,6 +146,11 @@ def _run_server(app: str, **kwargs: Any) -> None: # this explicitly prevents Uvicorn from treating ambient # WEB_CONCURRENCY as an unsupported multiprocess launch. workers=1, + # Serve valid HTTP/1.1 requests that opportunistically offer an h2c + # upgrade (JetBrains/Ktor clients) instead of rejecting them; the + # stock httptools protocol drops the body or answers 400. See + # app/core/http_protocol.py and issue #1757. + http=_load_http_protocol_class(), timeout_graceful_shutdown=drain_timeout_seconds, **kwargs, ) diff --git a/app/core/auth/dependencies.py b/app/core/auth/dependencies.py index d63893bf1f..52eb32a173 100644 --- a/app/core/auth/dependencies.py +++ b/app/core/auth/dependencies.py @@ -19,6 +19,7 @@ guest_principal, ) from app.core.auth.dashboard_mode import DashboardAuthMode, get_dashboard_request_auth +from app.core.clients.proxy import CODEX_LB_REQUIRED_CAPABILITY_HEADER from app.core.clients.usage import UsageFetchError, fetch_usage from app.core.config.settings import get_settings from app.core.config.settings_cache import get_settings_cache @@ -29,7 +30,7 @@ from app.core.upstream_proxy import UpstreamProxyRouteError, resolve_upstream_route from app.core.utils.time import utcnow from app.db.models import AccountStatus -from app.db.session import get_request_session +from app.db.session import get_background_session from app.modules.accounts.repository import AccountsRepository from app.modules.api_keys.repository import ApiKeysRepository from app.modules.api_keys.service import ApiKeyData, ApiKeyInvalidError, ApiKeysService @@ -63,7 +64,11 @@ async def validate_proxy_api_key( request: Request, credentials: HTTPAuthorizationCredentials | None = Security(_bearer), ) -> ApiKeyData | None: + """A required-capability header authenticates even when global proxy API-key auth is disabled.""" + authorization = None if credentials is None else f"Bearer {credentials.credentials}" + if request.headers.getlist(CODEX_LB_REQUIRED_CAPABILITY_HEADER): + return await validate_required_proxy_api_key_authorization(authorization) return await validate_proxy_api_key_authorization(authorization, request=request) @@ -108,7 +113,7 @@ async def _validate_api_key_token(token: str) -> ApiKeyData: return cached version_before_read = cache.version - async with get_request_session() as session: + async with get_background_session() as session: service = ApiKeysService(ApiKeysRepository(session)) try: validated = await service.validate_key(token) @@ -301,7 +306,7 @@ async def validate_codex_usage_identity(request: Request) -> ApiKeyData | None: return await _validate_api_key_token(token) raise ProxyAuthError("Missing chatgpt-account-id header") - async with get_request_session() as session: + async with get_background_session() as session: accounts_repo = AccountsRepository(session) account = await accounts_repo.get_active_by_chatgpt_account_id(account_id) if account is None: @@ -341,7 +346,7 @@ async def validate_codex_usage_identity(request: Request) -> ApiKeyData | None: usage_payload.workspace_id, usage_payload.workspace_label, ) - async with get_request_session() as session: + async with get_background_session() as session: accounts_repo = AccountsRepository(session) workspace_account = await accounts_repo.get_by_id(expected_account_id) if workspace_account is not None and workspace_account.chatgpt_account_id == account_id: @@ -366,6 +371,14 @@ async def validate_codex_usage_identity(request: Request) -> ApiKeyData | None: return None +async def validate_codex_provider_usage_identity(request: Request) -> ApiKeyData | None: + """Bind provider capability intent to a proxy API-key principal before usage I/O.""" + + if request.headers.getlist(CODEX_LB_REQUIRED_CAPABILITY_HEADER): + return await validate_required_proxy_api_key_authorization(request.headers.get("authorization")) + return await validate_codex_usage_identity(request) + + def _extract_bearer_token(authorization: str | None) -> str | None: if authorization is None: return None diff --git a/app/core/auth/guardian.py b/app/core/auth/guardian.py index 70b33a3b0d..3e92b638dc 100644 --- a/app/core/auth/guardian.py +++ b/app/core/auth/guardian.py @@ -16,8 +16,8 @@ from app.core.utils.time import to_utc_naive, utcnow from app.db.models import Account, AccountStatus from app.db.session import get_background_session -from app.modules.accounts.auth_manager import AccountsRepositoryPort, AuthManager -from app.modules.accounts.background_repository import BackgroundAccountsRepository +from app.modules.accounts.auth_manager import AuthManager +from app.modules.accounts.repository import AccountsRepository from app.modules.proxy.account_cache import get_account_selection_cache logger = logging.getLogger(__name__) @@ -176,49 +176,49 @@ async def _refresh_candidate(self, account_id: str, semaphore: asyncio.Semaphore max_age_seconds=self.max_age_seconds, ): return - manager = self.auth_manager_factory(repo) - try: - refresh_task = asyncio.create_task(manager.ensure_fresh(account, force=True)) + manager = self.auth_manager_factory(repo) try: - await asyncio.shield(refresh_task) - except asyncio.CancelledError: - with contextlib.suppress(Exception): - await refresh_task - raise - except RefreshError as exc: - self._record_failure(account_id) - if exc.is_permanent: - get_account_selection_cache().invalidate() - logger.warning( - "Auth Guardian refresh failed account_id=%s account_alias=%s status=%s code=%s permanent=%s " - "transport=%s", - account.id, - _safe_account_alias(account), - source_status, - exc.code, - exc.is_permanent, - exc.transport_error, - ) - return - except Exception as exc: - self._record_failure(account_id) - logger.warning( - "Auth Guardian refresh failed account_id=%s account_alias=%s status=%s error_type=%s", + refresh_task = asyncio.create_task(manager.ensure_fresh(account, force=True)) + try: + await asyncio.shield(refresh_task) + except asyncio.CancelledError: + with contextlib.suppress(Exception): + await refresh_task + raise + except RefreshError as exc: + self._record_failure(account_id) + if exc.is_permanent: + get_account_selection_cache().invalidate() + logger.warning( + "Auth Guardian refresh failed account_id=%s account_alias=%s status=%s code=%s permanent=%s " + "transport=%s", + account.id, + _safe_account_alias(account), + source_status, + exc.code, + exc.is_permanent, + exc.transport_error, + ) + return + except Exception as exc: + self._record_failure(account_id) + logger.warning( + "Auth Guardian refresh failed account_id=%s account_alias=%s status=%s error_type=%s", + account.id, + _safe_account_alias(account), + source_status, + exc.__class__.__name__, + exc_info=True, + ) + return + self._failures.pop(account_id, None) + get_account_selection_cache().invalidate() + logger.info( + "Auth Guardian refreshed account_id=%s account_alias=%s status=%s", account.id, _safe_account_alias(account), source_status, - exc.__class__.__name__, - exc_info=True, ) - return - self._failures.pop(account_id, None) - get_account_selection_cache().invalidate() - logger.info( - "Auth Guardian refreshed account_id=%s account_alias=%s status=%s", - account.id, - _safe_account_alias(account), - source_status, - ) def _in_backoff(self, account_id: str) -> bool: failure = self._failures.get(account_id) @@ -321,13 +321,13 @@ async def _count_live_bridge_ring_members() -> int: @asynccontextmanager -async def _default_accounts_repo_factory() -> AsyncIterator[BackgroundAccountsRepository]: - yield BackgroundAccountsRepository() +async def _default_accounts_repo_factory() -> AsyncIterator[AccountsRepository]: + async with get_background_session() as session: + yield AccountsRepository(session) -def _default_auth_manager_factory(_repo: _AccountsRepositoryLike) -> _AuthManagerLike: - repo = BackgroundAccountsRepository() - return AuthManager(cast(AccountsRepositoryPort, repo), refresh_repo_factory=_default_accounts_repo_factory) +def _default_auth_manager_factory(repo: _AccountsRepositoryLike) -> _AuthManagerLike: + return AuthManager(cast(AccountsRepository, repo), refresh_repo_factory=_default_accounts_repo_factory) def _jitter_delay(max_seconds: float) -> float: diff --git a/app/core/cache/invalidation.py b/app/core/cache/invalidation.py index 587108ea29..9e93be5c34 100644 --- a/app/core/cache/invalidation.py +++ b/app/core/cache/invalidation.py @@ -284,8 +284,29 @@ async def _flush_pending_bumps(self) -> None: # later bump instead of being coalesced into the version already # being written. self._pending_bumps.discard(namespace) - if not await self.bump(namespace): + try: + if not await self.bump(namespace): + self._pending_bumps.add(namespace) + except asyncio.CancelledError: + # The marker is cleared before the write, so an aborted write + # would otherwise leave the namespace neither written nor + # pending, breaking the required retry. Restored even when the + # abort is ambiguous — a redundant bump only re-runs peers' + # idempotent callbacks. + self._pending_bumps.add(namespace) + raise + except Exception: + # ``bump()`` reports normal failure by returning False, so a + # raise is abnormal — but re-raising would abort the flush and, + # since the loop is sorted, a persistently raising namespace + # would starve every namespace sorting after it on every cycle. + # Restore it and keep flushing the rest. self._pending_bumps.add(namespace) + logger.warning( + "cache_invalidation flush bump raised for namespace %s; kept pending", + _NAMESPACE_LOG_LABELS.get(namespace, "unknown"), + exc_info=True, + ) async def _poll_once(self) -> bool: """Flush pending bumps and reconcile observed versions once. diff --git a/app/core/clients/proxy.py b/app/core/clients/proxy.py index d39f068154..511a0cc9ba 100644 --- a/app/core/clients/proxy.py +++ b/app/core/clients/proxy.py @@ -58,11 +58,11 @@ ) from app.core.openai.exceptions import ClientPayloadError from app.core.openai.model_registry import get_model_registry -from app.core.openai.models import CompactResponsePayload, OpenAIError +from app.core.openai.models import CompactResponsePayload, OpenAIError, normalize_compaction_item_id from app.core.openai.parsing import ( + classify_event_type, parse_compact_response_payload, parse_error_payload, - parse_sse_event, ) from app.core.openai.requests import ( ResponsesCompactRequest, @@ -82,13 +82,14 @@ is_proxy_endpoint_failure, process_network_error_code, ) +from app.core.runtime_logging import safe_log_field from app.core.types import JsonObject, JsonValue from app.core.upstream_proxy import ResolvedUpstreamRoute from app.core.usage.live_hub import publish_live_usage from app.core.usage.live_snapshots import EVENT_MARKER, parse_rate_limit_event_text, parse_rate_limit_headers from app.core.utils.json_guards import is_json_mapping from app.core.utils.request_id import get_request_id -from app.core.utils.sse import format_sse_event, parse_sse_data_json +from app.core.utils.sse import format_sse_event, parse_sse_data_json, sse_event_type_from_block CODEX_INSTALLATION_ID_HEADER = "x-codex-installation-id" CODEX_TURN_METADATA_HEADER = "x-codex-turn-metadata" @@ -125,6 +126,11 @@ "response.audio.delta": "response.output_audio.delta", "response.audio_transcript.delta": "response.output_audio_transcript.delta", } +# Bare (unquoted) alias names gate the block-level alias normalizer: they +# match both the JSON `"type":""` in a data line and a stale +# `event: ` framing line. False positives (an alias name inside delta +# text) just take the full-parse path. +_SSE_EVENT_TYPE_ALIAS_MARKERS = tuple(_SSE_EVENT_TYPE_ALIASES) _SSE_LINE_BOUNDARY_RE = re.compile(r"\r\n|\r|\n") _RESPONSE_STREAM_TERMINAL_EVENT_TYPES = frozenset( { @@ -437,11 +443,41 @@ class SSEResponseProtocol(Protocol): class _CodexSSEContent: def __init__(self, response: Any) -> None: - self._response = response + content = getattr(response, "content", None) + if isinstance(content, bytes | bytearray): + self._body: bytes | None = bytes(content) + elif isinstance(content, str): + # Duck-typed upstream responses may expose a decoded string body + # (mirrors _codex_response_body); str has no iter_chunked. + self._body = content.encode() + else: + self._body = None + self._content = content def iter_chunked(self, size: int) -> "SSEChunkIteratorProtocol": - del size - return cast(SSEChunkIteratorProtocol, self._response.content.iter_chunked(1024)) + if self._body is not None: + return cast(SSEChunkIteratorProtocol, _BytesSSEChunkIterator(bytes(self._body), size)) + if self._content is None: + raise TypeError("SSE response content is missing") + return cast(SSEChunkIteratorProtocol, self._content.iter_chunked(size)) + + +class _BytesSSEChunkIterator: + def __init__(self, body: bytes, size: int) -> None: + self._body = body + self._size = max(1, size) + self._offset = 0 + + def __aiter__(self) -> "_BytesSSEChunkIterator": + return self + + async def __anext__(self) -> bytes: + if self._offset >= len(self._body): + raise StopAsyncIteration + end = min(len(self._body), self._offset + self._size) + chunk = self._body[self._offset : end] + self._offset = end + return chunk class _CodexSSEResponse: @@ -450,6 +486,7 @@ class _CodexSSEResponse: def __init__(self, response: Any) -> None: self._response = response self.status = _codex_response_status(response) + self.headers = _codex_response_headers(response) self.content = _CodexSSEContent(response) async def json(self, *, content_type: str | None = None) -> JsonValue: @@ -478,6 +515,7 @@ def __init__( upstream_error_code: str | None = None, failed_session: aiohttp.ClientSession | None = None, retry_after_seconds: int | None = None, + reservation_released: bool = False, ) -> None: super().__init__(f"Proxy response error ({status_code})") self.status_code = status_code @@ -490,6 +528,7 @@ def __init__( self.upstream_error_code = upstream_error_code self.failed_session = failed_session self.retry_after_seconds = retry_after_seconds + self.reservation_released = reservation_released def is_confirmed_pre_dispatch_transport_error(exc: ProxyResponseError) -> bool: @@ -971,26 +1010,24 @@ def _maybe_log_upstream_request_start( if privacy_policy.redacts_sensitive_details: account_id = "" - payload_summary = "sensitive private payload redacted" payload_json = None if "upstream_summary" in trace_channels: logger.info( - "upstream_request_start request_id=%s kind=%s method=%s target=%s account_id=%s headers=%s payload=%s", - request_id, - kind, - method, - target, - account_id, - header_keys, - payload_summary, + "upstream_request_start request_id=%s kind=%s method=%s target=%s account_id=%s headers=%s payload=omitted", + safe_log_field(request_id), + safe_log_field(kind), + safe_log_field(method), + safe_log_field(target), + safe_log_field(account_id), + safe_log_field(",".join(header_keys)), ) if "upstream_payload" in trace_channels and payload_json is not None: logger.info( - "upstream_request_payload request_id=%s kind=%s target=%s payload=%s", - request_id, - kind, - target, - payload_json, + "upstream_request_payload request_id=%s kind=%s target=%s payload_bytes=%s", + safe_log_field(request_id), + safe_log_field(kind), + safe_log_field(target), + len(payload_json.encode("utf-8")), ) @@ -1202,6 +1239,324 @@ async def _cancel_pending_chunk(task: asyncio.Task[bytes]) -> None: yield bytes(buffer).decode("utf-8", errors="replace") +async def _compact_response_payload_from_sse( + resp: SSEResponse, idle_timeout_seconds: float, max_event_bytes: int +) -> JsonValue: + last_payload: dict[str, JsonValue] | None = None + output_items: dict[int, dict[str, JsonValue]] = {} + unindexed_output_items: list[dict[str, JsonValue]] = [] + async for event_block in _iter_sse_events(resp, idle_timeout_seconds, max_event_bytes): + payload = parse_sse_data_json(event_block) + if payload is None: + continue + last_payload = payload + event_type = payload.get("type") + if event_type in {"response.output_item.added", "response.output_item.done"}: + output_index = payload.get("output_index") + item = payload.get("item") + if not isinstance(item, dict): + continue + if isinstance(output_index, int): + output_items[output_index] = dict(item) + elif event_type == "response.output_item.done": + # Some compatible upstream responses omit output_index on the + # terminal item even though response.completed has no output. + unindexed_output_items.append(dict(item)) + if event_type == "response.completed": + response = payload.get("response") + if isinstance(response, dict): + existing_output = response.get("output") + if (output_items or unindexed_output_items) and not ( + isinstance(existing_output, list) and existing_output + ): + merged_response = dict(response) + merged_response["output"] = [ + *[item for _, item in sorted(output_items.items())], + *unindexed_output_items, + ] + return merged_response + return response + raise ValueError("response.completed event missing response object") + if event_type in {"response.failed", "response.incomplete", "error"}: + raise _proxy_response_error_from_compact_sse_terminal(payload, event_type) + if last_payload is not None: + raise ValueError("upstream SSE ended before response.completed") + raise ValueError("empty upstream SSE response") + + +async def _compact_response_payload_from_success_response( + resp: Any, + *, + idle_timeout_seconds: float, + max_event_bytes: int, +) -> JsonValue: + headers = _codex_response_headers(resp) + content_type = next((value for key, value in headers.items() if key.lower() == "content-type"), "") + content = getattr(resp, "content", None) + if "text/event-stream" in content_type.lower() or ( + not content_type and callable(getattr(content, "iter_chunked", None)) + ): + return await _compact_response_payload_from_sse(cast(SSEResponse, resp), idle_timeout_seconds, max_event_bytes) + return await _codex_response_json(resp) + + +def _normalize_compact_response_payload_shape(payload: JsonValue) -> JsonValue: + if not is_json_mapping(payload): + return payload + object_value = payload.get("object") + if isinstance(object_value, str) and object_value.startswith("response.compact"): + return payload + compaction_item = _compact_output_item_from_payload(payload) + if compaction_item is None: + return payload + normalized: dict[str, JsonValue] = { + "object": "response.compaction", + "output": [compaction_item], + } + for key in ("id", "status", "usage", "service_tier"): + value = payload.get(key) + if value is not None: + normalized[key] = value + return normalized + + +def _responses_compact_payload_for_responses_endpoint(payload: ResponsesCompactRequest) -> dict[str, JsonValue]: + payload_dict = dict(payload.to_payload()) + input_value = payload_dict.get("input") + input_items = list(input_value) if isinstance(input_value, list) else [input_value] + if not (input_items and is_json_mapping(input_items[-1]) and input_items[-1].get("type") == "compaction_trigger"): + input_items.append({"type": "compaction_trigger"}) + payload_dict["input"] = input_items + return payload_dict + + +def _compact_output_item_from_payload(payload: Mapping[str, JsonValue]) -> dict[str, JsonValue] | None: + output = payload.get("output") + if isinstance(output, list): + for raw_item in output: + if not is_json_mapping(raw_item): + continue + item_type = raw_item.get("type") + if isinstance(item_type, str) and item_type in {"compaction", "compaction_summary"}: + normalized = _normalize_compact_output_item(raw_item) + if normalized is not None: + return normalized + # Remote compaction output places the compaction summary after any + # historical message items, so the message-shaped fallback must pick + # the last usable message instead of leaking earlier history. + for raw_item in reversed(output): + if not is_json_mapping(raw_item): + continue + item_type = raw_item.get("type") + if item_type == "message": + normalized = _compact_output_item_from_message(raw_item) + if normalized is not None: + return normalized + summary = payload.get("compaction_summary") + if is_json_mapping(summary): + return _normalize_compact_output_item(summary) + return None + + +def _compact_output_item_from_message(item: Mapping[str, JsonValue]) -> dict[str, JsonValue] | None: + text = _compact_message_text(item) + if not text: + return None + normalized: dict[str, JsonValue] = { + "type": "compaction", + "encrypted_content": text, + } + item_id = normalize_compaction_item_id(item.get("id")) + if item_id is not None: + normalized["id"] = item_id + status = item.get("status") + if isinstance(status, str) and status.strip(): + normalized["status"] = status + return normalized + + +def _compact_message_text(item: Mapping[str, JsonValue]) -> str | None: + direct_text = item.get("text") + if isinstance(direct_text, str) and direct_text: + return direct_text + content = item.get("content") + content_parts: list[Mapping[str, JsonValue]] + if is_json_mapping(content): + content_parts = [content] + elif isinstance(content, list): + content_parts = [part for part in content if is_json_mapping(part)] + else: + content_parts = [] + text_parts: list[str] = [] + for part in content_parts: + text = part.get("text") + if isinstance(text, str) and text: + text_parts.append(text) + if text_parts: + return "".join(text_parts) + return None + + +def _normalize_compact_output_item(item: Mapping[str, JsonValue]) -> dict[str, JsonValue] | None: + encrypted_content = item.get("encrypted_content") + if not isinstance(encrypted_content, str): + return None + normalized: dict[str, JsonValue] = { + "type": "compaction", + "encrypted_content": encrypted_content, + } + item_id = normalize_compaction_item_id(item.get("id")) + if item_id is not None: + normalized["id"] = item_id + status = item.get("status") + if isinstance(status, str) and status.strip(): + normalized["status"] = status + return normalized + + +def _proxy_response_error_from_compact_sse_terminal( + payload: Mapping[str, JsonValue], + event_type: object, +) -> ProxyResponseError: + error_payload = _compact_sse_terminal_error_payload(payload, event_type) + error_code, error_message = _error_details_from_envelope(error_payload) + status_code = _compact_sse_terminal_status_code(payload, error_payload=error_payload) + return ProxyResponseError( + status_code, + error_payload, + failure_phase="upstream", + failure_detail=error_message, + upstream_status_code=status_code, + upstream_error_code=error_code, + ) + + +def _proxy_response_error_from_compact_sse_stream_exception( + exc: StreamIdleTimeoutError | StreamEventTooLargeError, + *, + upstream_status_code: int | None, +) -> ProxyResponseError: + if isinstance(exc, StreamIdleTimeoutError): + return ProxyResponseError( + 502, + openai_error("stream_idle_timeout", "Upstream stream idle timeout"), + failure_phase="upstream", + failure_detail="stream_idle_timeout", + failure_exception_type=type(exc).__name__, + upstream_status_code=upstream_status_code, + upstream_error_code="stream_idle_timeout", + ) + return ProxyResponseError( + 502, + openai_error("stream_event_too_large", str(exc)), + failure_phase="upstream", + failure_detail=str(exc), + failure_exception_type=type(exc).__name__, + upstream_status_code=upstream_status_code, + upstream_error_code="stream_event_too_large", + ) + + +def _compact_sse_terminal_error_payload( + payload: Mapping[str, JsonValue], + event_type: object, +) -> OpenAIErrorEnvelope: + error = parse_error_payload(dict(payload)) + if error: + return {"error": _openai_error_detail(error)} + if event_type == "error": + error_code = payload.get("code") + error_message = payload.get("message") + if isinstance(error_code, str) and error_code and isinstance(error_message, str) and error_message: + error_type = payload.get("error_type") + if not isinstance(error_type, str) or not error_type.strip(): + error_type = "server_error" + detail: OpenAIErrorDetail = { + "code": error_code, + "message": error_message, + "type": error_type, + } + param = payload.get("param") + if isinstance(param, str) and param: + detail["param"] = param + return {"error": detail} + response = payload.get("response") + if is_json_mapping(response): + response_error = parse_error_payload(dict(response)) + if response_error: + return {"error": _openai_error_detail(response_error)} + message = _extract_upstream_message(cast(Mapping[str, Any], payload)) + if not message and is_json_mapping(response): + message = _extract_upstream_message(cast(Mapping[str, Any], response)) + code = "incomplete" if event_type == "response.incomplete" else "upstream_error" + return openai_error(code, message or f"Upstream SSE terminal event: {event_type}") + + +def _compact_sse_terminal_status_code( + payload: Mapping[str, JsonValue], + *, + error_payload: OpenAIErrorEnvelope | None = None, +) -> int: + response = payload.get("response") + candidates: list[JsonValue] = [] + if is_json_mapping(response): + candidates.extend( + [ + response.get("status_code"), + response.get("statusCode"), + response.get("status"), + ] + ) + candidates.extend([payload.get("status_code"), payload.get("statusCode"), payload.get("status")]) + for value in candidates: + if isinstance(value, int) and not isinstance(value, bool) and 400 <= value <= 599: + return value + candidates_for_error: tuple[Mapping[str, JsonValue], ...] = tuple( + candidate for candidate in (error_payload, response, payload) if is_json_mapping(candidate) + ) + for candidate in candidates_for_error: + error = parse_error_payload(dict(candidate)) + if error is None: + if candidate is payload and payload.get("type") == "error": + root_error = {key: payload[key] for key in ("code", "message", "param", "error_type") if key in payload} + error = OpenAIError.model_validate( + { + **root_error, + "type": root_error.get("error_type"), + } + ) + else: + continue + inferred_status = _status_code_from_openai_error(error) + if inferred_status is not None: + return inferred_status + return 502 + + +def _status_code_from_openai_error(error: OpenAIError) -> int | None: + error_type = error.type + error_code = error.code + if error_type == "authentication_error" or error_code in { + "invalid_api_key", + "invalid_authentication", + "token_invalidated", + }: + return 401 + if error_type == "permission_error" or error_code == "insufficient_permissions": + return 403 + if error_code == "not_found": + return 404 + if error_type == "rate_limit_error" or error_code in { + "rate_limit_exceeded", + "usage_limit_reached", + "insufficient_quota", + }: + return 429 + if error_type == "invalid_request_error": + return 400 + return None + + async def _error_response_body(resp: ErrorResponse) -> tuple[object | None, str | None]: try: return await resp.json(content_type=None), None @@ -1350,11 +1705,73 @@ def _normalize_sse_data_line(line: str) -> str: return line +def _normalize_sse_event_type_line(line: str) -> str: + if not line.startswith("event:"): + return line + value = line[6:] + if value.startswith(" "): + value = value[1:] + normalized_type = _SSE_EVENT_TYPE_ALIASES.get(value) + if normalized_type is None: + return line + return f"event: {normalized_type}" + + +def _normalize_multi_data_sse_block( + event_block: str, + lines: list[str], + line_separator: str, + terminator: str, +) -> str: + # Fragments of a payload split across multiple `data:` lines are not + # individually decodable, so alias detection must run on the combined + # payload (the SSE spec joins data-line values with "\n"). Decode it + # before touching the `event:` framing line so both surfaces are + # rewritten together; if the combined payload cannot be decoded, leave + # the whole block — framing line included — untouched rather than + # emitting a partially rewritten frame. + payload = parse_sse_data_json(event_block) + if payload is None: + return event_block + + data_replacement: str | None = None + event_type = payload.get("type") + if isinstance(event_type, str) and event_type in _SSE_EVENT_TYPE_ALIASES: + payload["type"] = _SSE_EVENT_TYPE_ALIASES[event_type] + data_replacement = f"data: {json.dumps(payload, ensure_ascii=True, separators=(',', ':'))}" + + normalized_lines: list[str] = [] + changed = False + data_line_emitted = False + for line in lines: + if line.startswith("data:"): + if data_replacement is None: + normalized_lines.append(line) + elif not data_line_emitted: + # The rewritten payload re-serializes compactly, so the + # fragments collapse into one canonical `data:` line. + normalized_lines.append(data_replacement) + data_line_emitted = True + changed = True + continue + normalized_line = _normalize_sse_event_type_line(line) + if normalized_line != line: + changed = True + normalized_lines.append(normalized_line) + if not changed: + return event_block + + normalized = line_separator.join(normalized_lines) + if terminator: + return normalized + terminator + return normalized + + def _normalize_sse_event_block(event_block: str) -> str: if not event_block: return event_block - if '"type":' not in event_block: + if not any(marker in event_block for marker in _SSE_EVENT_TYPE_ALIAS_MARKERS): return event_block if event_block.endswith("\r\n\r\n"): @@ -1378,10 +1795,17 @@ def _normalize_sse_event_block(event_block: str) -> str: if not lines: return event_block + if sum(1 for line in lines if line.startswith("data:")) > 1: + return _normalize_multi_data_sse_block(event_block, lines, line_separator, terminator) + normalized_lines: list[str] = [] changed = False for line in lines: - normalized_line = _normalize_sse_data_line(line) + # Rewrite both surfaces of a legacy alias: the JSON payload's `type` + # and the SSE `event:` framing line. Rewriting only the data line + # would emit mismatched framing when the block is relayed verbatim + # downstream instead of being re-serialized. + normalized_line = _normalize_sse_event_type_line(_normalize_sse_data_line(line)) if normalized_line != line: changed = True normalized_lines.append(normalized_line) @@ -1400,37 +1824,42 @@ def _normalize_stream_event_payload(payload: dict[str, JsonValue]) -> dict[str, normalized = dict(payload) normalized["type"] = _SSE_EVENT_TYPE_ALIASES[event_type] return normalized - error = parse_error_payload(payload) - if error is not None: - detail = error.model_dump(exclude_none=True) - event = response_failed_event( - _normalize_error_code(detail.get("code"), detail.get("type")), - detail.get("message", "Upstream websocket error"), - error_type=detail.get("type") or "server_error", - response_id=get_request_id(), - error_param=detail.get("param"), - ) - _copy_quota_error_metadata(event["response"]["error"], detail) - return cast(dict[str, JsonValue], event) - if event_type == "error": - message = _extract_upstream_message(payload) or "Upstream websocket error" - code = payload.get("code") - error_type = payload.get("error_type") or payload.get("type") - normalized_code = _normalize_error_code( - code if isinstance(code, str) else None, - error_type if isinstance(error_type, str) else None, - ) - if not isinstance(code, str) and normalized_code == "error": - normalized_code = "upstream_error" - return cast( - dict[str, JsonValue], - response_failed_event( - normalized_code, - message, - error_type=error_type if isinstance(error_type, str) and error_type != "error" else "server_error", + # Error-envelope schema validation is the only pydantic work on this hot + # path: classify from the parsed dict first and validate only error-shaped + # frames (``type == "error"`` or a top-level ``error`` envelope) so delta + # frames never reach the pydantic adapter. + if classify_event_type(payload) == "error" or isinstance(payload.get("error"), dict): + error = parse_error_payload(payload) + if error is not None: + detail = error.model_dump(exclude_none=True) + event = response_failed_event( + _normalize_error_code(detail.get("code"), detail.get("type")), + detail.get("message", "Upstream websocket error"), + error_type=detail.get("type") or "server_error", response_id=get_request_id(), - ), - ) + error_param=detail.get("param"), + ) + _copy_quota_error_metadata(event["response"]["error"], detail) + return cast(dict[str, JsonValue], event) + if event_type == "error": + message = _extract_upstream_message(payload) or "Upstream websocket error" + code = payload.get("code") + error_type = payload.get("error_type") or payload.get("type") + normalized_code = _normalize_error_code( + code if isinstance(code, str) else None, + error_type if isinstance(error_type, str) else None, + ) + if not isinstance(code, str) and normalized_code == "error": + normalized_code = "upstream_error" + return cast( + dict[str, JsonValue], + response_failed_event( + normalized_code, + message, + error_type=error_type if isinstance(error_type, str) and error_type != "error" else "server_error", + response_id=get_request_id(), + ), + ) return payload @@ -1439,6 +1868,21 @@ def _normalize_stream_payload_for_http_block( *, enforce_openai_sdk_contract: bool = True, ) -> tuple[str, str | None]: + # Cheap path for the dominant delta traffic: a canonically framed block + # exposes its event type on the `event:` line, so no JSON parse is needed. + # Full parsing remains for `error` frames and any block carrying an + # `"error"` substring (the SDK-contract rewrite in + # `_normalize_stream_event_payload` keys off a top-level error envelope), + # legacy alias types (rewritten payloads), and non-canonical or data-only + # framing (the event type then comes from the payload itself). + cheap_event_type = sse_event_type_from_block(event_block) + if ( + cheap_event_type is not None + and cheap_event_type != "error" + and cheap_event_type not in _SSE_EVENT_TYPE_ALIASES + and '"error"' not in event_block + ): + return event_block, cheap_event_type if not enforce_openai_sdk_contract: payload = parse_sse_data_json(event_block) if payload is None: @@ -1806,7 +2250,12 @@ async def _stream_websocket_events( total_timeout_seconds: float | None, max_event_bytes: int, enforce_openai_sdk_contract: bool = True, -) -> AsyncIterator[str]: +) -> AsyncIterator[tuple[str, str | None]]: + """Yield ``(sse_block, event_type)`` pairs. + + The event type is extracted from the payload parsed once here so that + downstream consumers never re-decode the formatted block. + """ deadline = None if total_timeout_seconds is None else time.monotonic() + total_timeout_seconds while True: @@ -1849,9 +2298,10 @@ async def _stream_websocket_events( if not isinstance(payload, dict): continue normalized = payload if not enforce_openai_sdk_contract else _normalize_stream_event_payload(payload) - event_type = normalized.get("type") - yield format_sse_event(normalized) - if isinstance(event_type, str) and _is_response_stream_terminal_event_type( + raw_event_type = normalized.get("type") + event_type = raw_event_type if isinstance(raw_event_type, str) else None + yield format_sse_event(normalized), event_type + if event_type is not None and _is_response_stream_terminal_event_type( event_type, enforce_openai_sdk_contract=enforce_openai_sdk_contract, ): @@ -1865,7 +2315,8 @@ async def _stream_codex_websocket_events( total_timeout_seconds: float | None, max_event_bytes: int, enforce_openai_sdk_contract: bool = True, -) -> AsyncIterator[str]: +) -> AsyncIterator[tuple[str, str | None]]: + """Yield ``(sse_block, event_type)`` pairs; see ``_stream_websocket_events``.""" deadline = None if total_timeout_seconds is None else time.monotonic() + total_timeout_seconds while True: @@ -1915,9 +2366,10 @@ async def _stream_codex_websocket_events( if not isinstance(payload, dict): continue normalized = payload if not enforce_openai_sdk_contract else _normalize_stream_event_payload(payload) - event_type = normalized.get("type") - yield format_sse_event(normalized) - if isinstance(event_type, str) and _is_response_stream_terminal_event_type( + raw_event_type = normalized.get("type") + event_type = raw_event_type if isinstance(raw_event_type, str) else None + yield format_sse_event(normalized), event_type + if event_type is not None and _is_response_stream_terminal_event_type( event_type, enforce_openai_sdk_contract=enforce_openai_sdk_contract, ): @@ -1952,7 +2404,8 @@ async def _stream_responses_via_websocket( route_trace: UpstreamProxyRouteTrace | None = None, allow_direct_egress: bool = True, enforce_openai_sdk_contract: bool = True, -) -> AsyncIterator[str]: +) -> AsyncIterator[tuple[str, str | None]]: + """Yield ``(sse_block, event_type)`` pairs from the upstream websocket.""" websocket_url = _to_websocket_upstream_url(url) request_started_at = time.monotonic() request_payload = _prepare_websocket_response_create_payload(payload_dict) @@ -2131,7 +2584,7 @@ async def _record_lifecycle_failure(exc: Exception) -> None: enforce_openai_sdk_contract=enforce_openai_sdk_contract, ) ) - async for event in event_iter: + async for event, event_type in event_iter: archive_text( direction="server_to_codex", kind="responses", @@ -2143,14 +2596,13 @@ async def _record_lifecycle_failure(exc: Exception) -> None: headers=headers, extra={"event_format": "sse"}, ) - parsed_event = parse_sse_event(event) - if parsed_event and _is_response_stream_terminal_event_type( - parsed_event.type, + if event_type is not None and _is_response_stream_terminal_event_type( + event_type, enforce_openai_sdk_contract=enforce_openai_sdk_contract, ): seen_terminal = True await _record_lifecycle_success() - yield event + yield event, event_type if not seen_terminal: await _record_lifecycle_failure(aiohttp.ClientError("Upstream websocket closed without terminal event")) except Exception as exc: @@ -2688,7 +3140,7 @@ async def stream_responses( publish_live_usage( parse_rate_limit_event_text(event_block), account_id=codex_lb_account_id, - chatgpt_account_id=None if codex_lb_account_id else account_id, + chatgpt_account_id=account_id, ) yield event_block @@ -2859,7 +3311,7 @@ async def _stream_via_http_attempt( publish_live_usage( parse_rate_limit_headers(getattr(raw_resp, "headers", None)), account_id=codex_lb_account_id, - chatgpt_account_id=None if codex_lb_account_id else account_id, + chatgpt_account_id=account_id, ) if resp.status >= 400: if raise_for_status: @@ -2906,13 +3358,7 @@ async def _stream_via_http_attempt( event_block, enforce_openai_sdk_contract=enforce_openai_sdk_contract, ) - event = parse_sse_event(event_block) - if event: - if event.type in _RESPONSE_STREAM_TERMINAL_EVENT_TYPES or ( - event.type == "error" and not enforce_openai_sdk_contract - ): - seen_terminal = True - elif isinstance(normalized_event_type, str) and ( + if isinstance(normalized_event_type, str) and ( normalized_event_type in _RESPONSE_STREAM_TERMINAL_EVENT_TYPES or (normalized_event_type == "error" and not enforce_openai_sdk_contract) ): @@ -2953,7 +3399,7 @@ async def _stream_via_http_attempt( publish_live_usage( parse_rate_limit_headers(getattr(resp, "headers", None)), account_id=codex_lb_account_id, - chatgpt_account_id=None if codex_lb_account_id else account_id, + chatgpt_account_id=account_id, ) if resp.status >= 400: if raise_for_status: @@ -3000,13 +3446,7 @@ async def _stream_via_http_attempt( event_block, enforce_openai_sdk_contract=enforce_openai_sdk_contract, ) - event = parse_sse_event(event_block) - if event: - if event.type in _RESPONSE_STREAM_TERMINAL_EVENT_TYPES or ( - event.type == "error" and not enforce_openai_sdk_contract - ): - seen_terminal = True - elif isinstance(normalized_event_type, str) and ( + if isinstance(normalized_event_type, str) and ( normalized_event_type in _RESPONSE_STREAM_TERMINAL_EVENT_TYPES or (normalized_event_type == "error" and not enforce_openai_sdk_contract) ): @@ -3114,7 +3554,7 @@ async def _stream_via_http_after_websocket_rejection( try: if transport == "websocket": try: - async for event_block in _stream_responses_via_websocket( + async for event_block, event_type in _stream_responses_via_websocket( payload_dict=payload_dict, url=url, headers=upstream_headers, @@ -3132,14 +3572,11 @@ async def _stream_via_http_after_websocket_rejection( ): if status_code is None: status_code = 101 - event = parse_sse_event(event_block) - if event: - event_type = event.type - if _is_response_stream_terminal_event_type( - event_type, - enforce_openai_sdk_contract=enforce_openai_sdk_contract, - ): - seen_terminal = True + if event_type is not None and _is_response_stream_terminal_event_type( + event_type, + enforce_openai_sdk_contract=enforce_openai_sdk_contract, + ): + seen_terminal = True yield event_block except aiohttp.WSServerHandshakeError as exc: if not _should_fallback_to_http_after_websocket_handshake_error(transport_mode, exc): @@ -3599,7 +4036,7 @@ class _CompactCommandTransport: async def execute(self) -> CompactResponsePayload: settings = get_settings() upstream_base = settings.upstream_base_url.rstrip("/") - url = f"{upstream_base}/codex/responses/compact" + url = f"{upstream_base}/codex/responses" require_route_or_direct_egress_opt_in( route=self.route, allow_direct_egress=self.allow_direct_egress, @@ -3612,12 +4049,14 @@ async def execute(self) -> CompactResponsePayload: self.headers, self.access_token, upstream_account_id, - accept="application/json", + accept="text/event-stream", ) pre_request_started_at = time.monotonic() compact_timeout_seconds = _effective_compact_total_timeout(settings.upstream_compact_timeout_seconds) effective_connect_timeout = _effective_compact_connect_timeout(settings.upstream_connect_timeout_seconds) - payload_dict = dict(self.payload.to_payload()) + payload_dict = _responses_compact_payload_for_responses_endpoint(self.payload) + payload_dict["store"] = False + payload_dict["stream"] = True if settings.image_inline_fetch_enabled: payload_dict = await _inline_input_image_urls( payload_dict, @@ -3737,7 +4176,18 @@ async def execute(self) -> CompactResponsePayload: upstream_status_code=status_code, ) try: - data = await _codex_response_json(resp) + data = await _compact_response_payload_from_success_response( + _CodexSSEResponse(resp), + idle_timeout_seconds=compact_timeout_seconds or settings.stream_idle_timeout_seconds, + max_event_bytes=settings.max_sse_event_bytes, + ) + except (StreamIdleTimeoutError, StreamEventTooLargeError) as exc: + raise _proxy_response_error_from_compact_sse_stream_exception( + exc, + upstream_status_code=status_code, + ) from exc + except ProxyResponseError: + raise except Exception as exc: error_code = "upstream_error" error_message = "Invalid JSON from upstream" @@ -3752,12 +4202,14 @@ async def execute(self) -> CompactResponsePayload: failure_exception_type=failure_exception_type, upstream_status_code=status_code, ) from exc + raw_data = data + data = _normalize_compact_response_payload_shape(data) parsed = parse_compact_response_payload(data) archive_json( direction="server_to_codex", kind="compact", transport="http", - payload=data, + payload=raw_data, account_id=self.account_id, method="POST", url=url, @@ -3815,7 +4267,11 @@ async def execute(self) -> CompactResponsePayload: upstream_status_code=resp.status, ) try: - data = await resp.json(content_type=None) + data = await _compact_response_payload_from_success_response( + resp, + idle_timeout_seconds=compact_timeout_seconds or settings.stream_idle_timeout_seconds, + max_event_bytes=settings.max_sse_event_bytes, + ) except (aiohttp.ClientError, asyncio.TimeoutError, OSError) as exc: message = str(exc) or "Request to upstream timed out" error_code = process_network_error_code( @@ -3838,6 +4294,13 @@ async def execute(self) -> CompactResponsePayload: upstream_status_code=resp.status, failed_session=_failed_shared_session_for_process_network_error(error_code, self.session), ) from exc + except (StreamIdleTimeoutError, StreamEventTooLargeError) as exc: + raise _proxy_response_error_from_compact_sse_stream_exception( + exc, + upstream_status_code=resp.status, + ) from exc + except ProxyResponseError: + raise except Exception as exc: error_code = "upstream_error" error_message = "Invalid JSON from upstream" @@ -3852,12 +4315,14 @@ async def execute(self) -> CompactResponsePayload: failure_exception_type=failure_exception_type, upstream_status_code=resp.status, ) from exc + raw_data = data + data = _normalize_compact_response_payload_shape(data) parsed = parse_compact_response_payload(data) archive_json( direction="server_to_codex", kind="compact", transport="http", - payload=data, + payload=raw_data, account_id=self.account_id, method="POST", url=url, diff --git a/app/core/clients/proxy_websocket.py b/app/core/clients/proxy_websocket.py index c13e25e83b..c14ee69c65 100644 --- a/app/core/clients/proxy_websocket.py +++ b/app/core/clients/proxy_websocket.py @@ -5,7 +5,6 @@ import logging import os import re -import ssl from dataclasses import dataclass from enum import StrEnum from typing import Any, Mapping, NoReturn, Protocol, Sequence, cast @@ -20,7 +19,6 @@ ConnectionClosedOK, InvalidHandshake, InvalidProxy, - InvalidProxyMessage, InvalidStatus, ) from websockets.typing import Origin, Subprotocol @@ -81,7 +79,6 @@ ) _LIVE_CALL_ID_PATTERN = re.compile(rf"{REALTIME_LIVE_CALL_ID_ROUTE_REGEX}\Z") UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE = "upstream_websocket_liveness_timeout" -UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE = "upstream_websocket_closed_before_send" _WEBSOCKETS_KEEPALIVE_TIMEOUT_REASON = "keepalive ping timeout" _AIOHTTP_HEARTBEAT_TIMEOUT_PREFIX = "No PONG received after " @@ -136,31 +133,6 @@ class _UpstreamWebSocketPolicy: logger = logging.getLogger(__name__) -class _ProxySetupSafeClientConnection(ClientConnection): - """Close a proxied connection safely before ``connection_made``. - - ``websockets`` transfers the proxy transport to this protocol before a - TLS upgrade, but initializes ``recv_messages`` only after that upgrade - succeeds. If the transport closes during the upgrade, asyncio calls - ``connection_lost`` first and the dependency dereferences the missing - receive assembler. Complete only the state that exists at this point; - established connections continue through the dependency's full close - path unchanged. - """ - - def connection_lost(self, exc: Exception | None) -> None: - if hasattr(self, "recv_messages"): - super().connection_lost(exc) - return - - self.protocol.receive_eof() - self.set_recv_exc(exc) - if self.keepalive_task is not None: - self.keepalive_task.cancel() - if not self.connection_lost_waiter.done(): - self.connection_lost_waiter.set_result(None) - - def normalize_realtime_call_id(value: str) -> str | None: normalized = value.strip() if not normalized or len(normalized) > _LIVE_CALL_ID_MAX_LENGTH: @@ -190,32 +162,6 @@ def _consume_connection_lost_exception(done: asyncio.Future[Any]) -> None: return -async def _connect_websockets_transport( - url: str, - *, - proxy_url: str | None, - connect_kwargs: Mapping[str, Any], -) -> ClientConnection: - """Open once, or retry one fresh tunnel for a shared-proxy setup close. - - Awaiting ``websocket_connect`` proves no application frame was dispatched - when a timeout or transport error escapes. A fresh tunnel on the same - account is therefore safe. The process-wide environment proxy isn't an - account-owned route, so exhaustion must not penalize or rotate accounts. - """ - - try: - return await websocket_connect(url, **connect_kwargs) - except (asyncio.TimeoutError, OSError, InvalidProxyMessage) as exc: - if proxy_url is None or isinstance(exc, ssl.SSLCertVerificationError): - raise - logger.warning( - "Retrying upstream websocket after shared proxy setup failure exception_type=%s", - type(exc).__name__, - ) - return await websocket_connect(url, **connect_kwargs) - - @dataclass(slots=True) class UpstreamWebSocketMessage: kind: str @@ -254,21 +200,11 @@ def is_account_neutral_websocket_error_code(error_code: str | None) -> bool: # the compatibility keepalive code here as long as adapters can emit it. return error_code in { PROCESS_NETWORK_UNAVAILABLE_CODE, - UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE, "upstream_keepalive_timeout", } -def _raise_websocket_closed_before_send() -> NoReturn: - """Raise only when the adapter has not invoked the transport send primitive.""" - - raise UpstreamWebSocketTransportError( - "Upstream websocket was already closed before send", - error_code=UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - ) - - def _is_websocket_liveness_timeout(exc: BaseException) -> bool: if isinstance(exc, ConnectionClosedError): # websockets emits this locally-sent 1011 when its own ping watchdog @@ -372,18 +308,12 @@ def __init__( connection_lost_waiter.add_done_callback(_consume_connection_lost_exception) async def send_text(self, text: str) -> None: - state = getattr(self._connection, "state", None) - if state is not None and getattr(state, "name", None) != "OPEN": - _raise_websocket_closed_before_send() try: await self._connection.send(text) except Exception as exc: await _raise_websocket_send_error(exc, uses_proxy=self._uses_proxy) async def send_bytes(self, data: bytes) -> None: - state = getattr(self._connection, "state", None) - if state is not None and getattr(state, "name", None) != "OPEN": - _raise_websocket_closed_before_send() try: await self._connection.send(data) except Exception as exc: @@ -478,8 +408,6 @@ def __init__( self._response_headers = _normalize_response_headers(response_headers) async def send_text(self, text: str) -> None: - if bool(getattr(self._websocket, "closed", False)): - _raise_websocket_closed_before_send() try: result = self._websocket.send_str(text) if asyncio.iscoroutine(result): @@ -489,8 +417,6 @@ async def send_text(self, text: str) -> None: await _raise_websocket_send_error(classification_exc, endpoint_id=self._endpoint_id, uses_proxy=True) async def send_bytes(self, data: bytes) -> None: - if bool(getattr(self._websocket, "closed", False)): - _raise_websocket_closed_before_send() try: result = self._websocket.send_bytes(data) if asyncio.iscoroutine(result): @@ -999,31 +925,28 @@ async def _connect_upstream_websocket( ) try: subprotocol_kwargs = {"subprotocols": cast(Sequence[Subprotocol], subprotocols)} if subprotocols else {} - proxy_connection_kwargs = ( - {"create_connection": _ProxySetupSafeClientConnection} if proxy_url is not None else {} - ) - response = await _connect_websockets_transport( + response = await websocket_connect( url, - proxy_url=proxy_url, - connect_kwargs={ - "origin": origin, - "additional_headers": upstream_headers or None, - "user_agent_header": user_agent, - "open_timeout": settings.upstream_connect_timeout_seconds, - "ping_timeout": ping_timeout, - "max_size": settings.max_sse_event_bytes, - "proxy": proxy_url, - **proxy_connection_kwargs, - **subprotocol_kwargs, - }, + origin=origin, + additional_headers=upstream_headers or None, + user_agent_header=user_agent, + open_timeout=settings.upstream_connect_timeout_seconds, + ping_timeout=ping_timeout, + max_size=settings.max_sse_event_bytes, + proxy=proxy_url, + # Do not offer permessage-deflate upstream: the websockets library + # enables it by default, but the sibling upstream transports (the + # routed aiohttp path and the raw-handshake transport) already run + # uncompressed, and per-frame zlib decode on high-rate event + # streams burns CPU on the proxy host. The client-facing socket + # keeps negotiating permessage-deflate per responses-api-compat. + compression=None, + **subprotocol_kwargs, ) except asyncio.TimeoutError as exc: raise ProxyResponseError( 502, openai_error("upstream_unavailable", "Request to upstream timed out"), - failure_phase="connect", - failure_detail="shared_proxy_connect_pre_dispatch_exhausted" if proxy_url is not None else None, - failure_exception_type=type(exc).__name__, ) from exc except InvalidStatus as exc: response = exc.response @@ -1062,13 +985,6 @@ async def _connect_upstream_websocket( raise ProxyResponseError( 502, openai_error("upstream_unavailable", message), - failure_phase="connect", - failure_detail=( - "shared_proxy_connect_pre_dispatch_exhausted" - if proxy_url is not None and isinstance(exc, InvalidProxyMessage) - else None - ), - failure_exception_type=type(exc).__name__, ) from exc except OSError as exc: error_code = process_network_error_code( @@ -1082,12 +998,6 @@ async def _connect_upstream_websocket( openai_error(error_code, message), failure_phase="connect", retryable_same_contract=error_code == PROCESS_NETWORK_UNAVAILABLE_CODE, - failure_detail=( - "shared_proxy_connect_pre_dispatch_exhausted" - if proxy_url is not None and not isinstance(exc, ssl.SSLCertVerificationError) - else None - ), - failure_exception_type=type(exc).__name__, ) from exc return ArchivingUpstreamWebSocket( diff --git a/app/core/config/settings.py b/app/core/config/settings.py index b05907572f..a710f441d0 100644 --- a/app/core/config/settings.py +++ b/app/core/config/settings.py @@ -242,9 +242,12 @@ class Settings(BaseSettings): database_url: str = DEFAULT_DATABASE_URL # Pool timeout and recycle are fixed constants in ``app/db/session.py``; # the background-task engine always derives its pool sizing from the two - # settings below. - database_pool_size: int = Field(default=15, gt=0) - database_max_overflow: int = Field(default=10, ge=0) + # settings below. Defaults are sized so one replica's two pooled engines + # cap at (25 + 15) * 2 = 80 PostgreSQL connections, preserving >= 20 raw + # server slots on PostgreSQL's default max_connections=100 for reserved + # connections, the migration path's two-connection peak, and operations. + database_pool_size: int = Field(default=25, gt=0) + database_max_overflow: int = Field(default=15, ge=0) database_migrate_on_startup: bool = True database_sqlite_pre_migrate_backup_enabled: bool = True database_sqlite_pre_migrate_backup_max_files: int = Field(default=5, ge=1) @@ -288,6 +291,7 @@ class Settings(BaseSettings): usage_refresh_enabled: bool = True usage_refresh_interval_seconds: int = Field(default=60, gt=0) live_usage_ingestion_enabled: bool = True + rate_limit_reset_credits_refresh_enabled: bool = True rate_limit_reset_credits_refresh_interval_seconds: int = Field(default=60, gt=0) openai_cache_affinity_max_age_seconds: int = Field(default=1800, gt=0) warmup_model: str = "gpt-5.4-mini" @@ -307,6 +311,39 @@ class Settings(BaseSettings): le=30.0, ) http_responses_session_bridge_gateway_safe_mode: bool = False + # Attach the durable operation identity to response.create client metadata. + # The upstream must explicitly support/deduplicate this value before any + # automatic replay is enabled; metadata-only propagation is safe by default. + http_responses_session_bridge_operation_ledger_enabled: bool = True + # Bound durable replay storage per operation so a long response cannot + # exhaust the database. An incomplete spool is never replayed. + http_responses_session_bridge_operation_event_spool_max_bytes: int = Field(default=2 * 1024 * 1024, gt=0) + http_responses_session_bridge_operation_event_spool_batch_size: int = Field(default=32, gt=0, le=256) + http_responses_session_bridge_operation_event_spool_flush_interval_seconds: float = Field( + default=0.1, + ge=0.01, + le=5.0, + ) + http_responses_session_bridge_operation_event_spool_max_pending_events: int = Field(default=2048, gt=0) + http_responses_session_bridge_operation_event_spool_max_pending_bytes: int = Field( + default=32 * 1024 * 1024, + gt=0, + ) + # Keep durable transcript material short-lived by default. The transcript + # is sensitive prompt/output data and is only a recovery aid. + http_responses_session_bridge_operation_spool_retention_seconds: float = Field( + default=7 * 24 * 60 * 60, + gt=0, + ) + # Recovery-first mode can either ask the client to drop an ambiguous anchor + # or let the bridge retry that anchored request once on a fresh upstream + # socket. Both are at-least-once strategies; fail-closed remains default. + http_responses_session_bridge_ambiguous_continuation_recovery_mode: Literal[ + "fail_closed", + "client_full_history_once", + "server_anchored_replay_once", + "server_indefinite_recovery", + ] = "fail_closed" http_responses_session_bridge_instance_id: str = Field(default_factory=_default_http_bridge_instance_id) http_responses_session_bridge_instance_ring: Annotated[list[str], NoDecode] = Field(default_factory=list) http_responses_session_bridge_advertise_base_url: str | None = None @@ -326,6 +363,8 @@ class Settings(BaseSettings): usage_history_retention_days: int = Field(default=0, ge=0, le=3650) quota_planner_scheduler_enabled: bool = True automations_scheduler_enabled: bool = True + telemetry_enabled: bool | None = None + telemetry_endpoint: str = "https://telemetry.tokmaxxing.com" encryption_key_file: Path = DEFAULT_ENCRYPTION_KEY_FILE # Startup cross-replica encryption-key consistency check against the shared # database sentinel: "enforce" refuses startup on mismatch, "warn" logs an @@ -439,12 +478,20 @@ def upstream_websocket_proxy_env(self) -> Mapping[str, str | None]: workers_per_instance: int = Field(default=1, ge=1) proxy_refresh_failure_cooldown_seconds: float = Field(default=5.0, ge=0.0) usage_refresh_auth_failure_cooldown_seconds: float = Field(default=300.0, ge=0.0) + timeout_invariant_validation_strict: bool = False # Local memory-pressure guard (0 = disabled). Requests are rejected with # 503 once RSS reaches the threshold; a warning is logged from 80% of it # (``app/core/resilience/memory_monitor.py`` derives the warning level). memory_reject_threshold_mb: int = 0 + # Event-loop lag watchdog (0 = disabled). Samples asyncio.sleep drift once + # per second; lag at or above the threshold emits a rate-limited warning + # and Prometheus signals (``app/core/resilience/loop_lag_monitor.py``). + # Default 0.5s: an order of magnitude above healthy scheduling jitter, + # well below the lag that fails 10s-budget health checks. + event_loop_lag_warn_threshold_seconds: float = Field(default=0.5, ge=0.0) + # OpenTelemetry otel_enabled: bool = False otel_exporter_endpoint: str = "" diff --git a/app/core/errors.py b/app/core/errors.py index 735b6d7a09..75dfafab38 100644 --- a/app/core/errors.py +++ b/app/core/errors.py @@ -13,7 +13,6 @@ class OpenAIErrorDetail(TypedDict, total=False): plan_type: str resets_at: int | float resets_in_seconds: int | float - action: str class OpenAIErrorEnvelope(TypedDict): @@ -46,10 +45,6 @@ class ResponseFailedEvent(TypedDict): PREVIOUS_RESPONSE_STREAM_INCOMPLETE_MESSAGE = "Upstream websocket closed before response.completed" PREVIOUS_RESPONSE_NOT_FOUND_CODE = "previous_response_not_found" PREVIOUS_RESPONSE_NOT_FOUND_MESSAGE = "Previous response was not found; retry without previous_response_id." -_INVALID_PREVIOUS_RESPONSE_ID_MESSAGE_RE = re.compile( - r"^invalid\s+(?:previous_response_id|'previous_response_id'|\"previous_response_id\"|`previous_response_id`)\.?$", - re.IGNORECASE, -) def openai_error( @@ -81,16 +76,16 @@ def is_previous_response_not_found_message(message: str | None) -> bool: if message is None: return False normalized = " ".join(message.lower().split()) - return ("previous response" in normalized and "not found" in normalized) or ( - _is_canonical_invalid_previous_response_id_message(message) - ) + return "previous response" in normalized and "not found" in normalized -def _is_canonical_invalid_previous_response_id_message(message: str | None) -> bool: +def _is_invalid_previous_response_id_message(message: str | None) -> bool: if message is None: return False normalized = " ".join(message.lower().split()) - return _INVALID_PREVIOUS_RESPONSE_ID_MESSAGE_RE.fullmatch(normalized) is not None + if normalized.endswith("."): + normalized = normalized[:-1] + return normalized == "invalid `previous_response_id`" def previous_response_id_from_not_found_message(message: str | None) -> str | None: @@ -116,11 +111,13 @@ def is_previous_response_not_found_error( ) -> bool: if code == PREVIOUS_RESPONSE_NOT_FOUND_CODE: return True - if code != "invalid_request_error" or param not in {None, "previous_response_id"}: + if code != "invalid_request_error": + return False + if param is None: + return _is_invalid_previous_response_id_message(message) + if param != "previous_response_id": return False - if param == "previous_response_id": - return is_previous_response_not_found_message(message) - return _is_canonical_invalid_previous_response_id_message(message) + return is_previous_response_not_found_message(message) or _is_invalid_previous_response_id_message(message) def response_failed_event( diff --git a/app/core/exceptions.py b/app/core/exceptions.py index 54fadc5ca7..777ec2dd1b 100644 --- a/app/core/exceptions.py +++ b/app/core/exceptions.py @@ -8,8 +8,15 @@ class AppError(Exception): code: str = "internal_error" message: str = "Unexpected error" - def __init__(self, message: str | None = None, *, code: str | None = None) -> None: + def __init__( + self, + message: str | None = None, + *, + code: str | None = None, + param: str | None = None, + ) -> None: self.message = message or self.__class__.message + self.param = param if code is not None: self.code = code super().__init__(self.message) @@ -30,6 +37,12 @@ class ProxyModelNotAllowed(AppError): error_type = "permission_error" +class ProxyReasoningEffortNotAllowed(AppError): + status_code = 403 + code = "reasoning_effort_not_allowed" + error_type = "permission_error" + + class ProxyRateLimitError(AppError): status_code = 429 code = "rate_limit_exceeded" @@ -42,6 +55,13 @@ class ProxyUpstreamError(AppError): error_type = "server_error" +class ProxyRequiredCapabilityTransportError(AppError): + status_code = 400 + code = "required_capability_transport_unsupported" + error_type = "invalid_request_error" + message = "Required capability routing is only supported over the Responses WebSocket transport." + + # --- Dashboard-envelope errors --- diff --git a/app/core/handlers/exceptions.py b/app/core/handlers/exceptions.py index 37b89c2c6c..7caef49013 100644 --- a/app/core/handlers/exceptions.py +++ b/app/core/handlers/exceptions.py @@ -12,7 +12,6 @@ ) from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse, Response -from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError from starlette._utils import get_route_path from starlette.exceptions import HTTPException as StarletteHTTPException @@ -31,6 +30,8 @@ ProxyAuthError, ProxyModelNotAllowed, ProxyRateLimitError, + ProxyReasoningEffortNotAllowed, + ProxyRequiredCapabilityTransportError, ProxyUpstreamError, ) from app.core.middleware.multipart_content_encoding import ( @@ -44,11 +45,6 @@ ) from app.core.multipart import MultipartPayloadTooLarge from app.core.runtime_logging import log_error_response -from app.db.session import ( - DATABASE_POOL_RETRY_AFTER_SECONDS, - DATABASE_POOL_UNAVAILABLE_CODE, - DATABASE_POOL_UNAVAILABLE_MESSAGE, -) from app.modules.proxy.images_observability import ( IMAGE_ROUTE_MODEL_STATE, IMAGE_ROUTE_STARTED_AT_STATE, @@ -62,7 +58,9 @@ _OPENAI_EXCEPTION_TYPES: tuple[type[AppError], ...] = ( ProxyAuthError, ProxyModelNotAllowed, + ProxyReasoningEffortNotAllowed, ProxyRateLimitError, + ProxyRequiredCapabilityTransportError, ProxyUpstreamError, ) @@ -240,10 +238,16 @@ async def _openai_domain_handler(request: Request, exc: AppError) -> JSONRespons status=exc.status_code, outcome="auth_error", ) - return JSONResponse( - status_code=exc.status_code, - content=openai_error(exc.code, exc.message, error_type=error_type), - ) + elif isinstance(exc, ProxyRequiredCapabilityTransportError): + await _record_image_route_exception_observability( + request, + status=exc.status_code, + outcome="invalid_request", + ) + error = openai_error(exc.code, exc.message, error_type=error_type) + if exc.param is not None: + error["error"]["param"] = exc.param + return JSONResponse(status_code=exc.status_code, content=error) # --- Domain exceptions: Dashboard envelope --- @@ -268,47 +272,6 @@ async def _dashboard_domain_handler(request: Request, exc: AppError) -> JSONResp headers=headers, ) - @app.exception_handler(SQLAlchemyTimeoutError) - async def database_pool_timeout_handler( - request: Request, - exc: SQLAlchemyTimeoutError, - ) -> JSONResponse: - del exc - fmt = _error_format(request) - log_error_response( - logger, - request, - 503, - DATABASE_POOL_UNAVAILABLE_CODE, - DATABASE_POOL_UNAVAILABLE_MESSAGE, - category="database_pool_unavailable_response", - ) - headers = {"Retry-After": str(DATABASE_POOL_RETRY_AFTER_SECONDS)} - if fmt == "dashboard": - return JSONResponse( - status_code=503, - content=dashboard_error( - DATABASE_POOL_UNAVAILABLE_CODE, - DATABASE_POOL_UNAVAILABLE_MESSAGE, - ), - headers=headers, - ) - if fmt == "openai": - return JSONResponse( - status_code=503, - content=openai_error( - DATABASE_POOL_UNAVAILABLE_CODE, - DATABASE_POOL_UNAVAILABLE_MESSAGE, - error_type="server_error", - ), - headers=headers, - ) - return JSONResponse( - status_code=503, - content={"detail": DATABASE_POOL_UNAVAILABLE_MESSAGE}, - headers=headers, - ) - # --- Framework exceptions: format based on router marker --- @app.exception_handler(RequestValidationError) diff --git a/app/core/http_protocol.py b/app/core/http_protocol.py new file mode 100644 index 0000000000..93fcfa1d3b --- /dev/null +++ b/app/core/http_protocol.py @@ -0,0 +1,117 @@ +"""Uvicorn HTTP protocol selection tolerant of opportunistic upgrade offers. + +JetBrains/Ktor clients attach cleartext HTTP/2 upgrade headers +(``Connection: Upgrade, HTTP2-Settings`` + ``Upgrade: h2c`` + +``HTTP2-Settings``) to ordinary HTTP/1.1 Responses API POSTs. RFC 9110 +section 7.8 lets a server ignore such an offer and answer over HTTP/1.1 — +upstream OpenAI endpoints do exactly that — but uvicorn's stock protocol +implementations either wedge on the offer (httptools) or leak the declined +offer's hop-by-hop headers into the ASGI scope (h11). See +https://github.com/Soju06/codex-lb/issues/1757 and the module docstring of +``app.core.http_protocol_httptools`` for the full failure analysis. + +This module exposes :func:`load_http_protocol_class`, which returns the +tolerant httptools subclass when httptools is importable (matching uvicorn's +``auto`` preference) and an h11 subclass with the same header hygiene +otherwise. +""" + +from __future__ import annotations + +import asyncio + +from uvicorn.protocols.http.h11_impl import H11Protocol + +# Hop-by-hop headers that only exist to carry the declined protocol switch. +# ``HTTP2-Settings`` is defined exclusively for the h2c upgrade (RFC 9113 +# section 3.1) and MUST NOT be forwarded once the offer is declined. +UPGRADE_HOP_BY_HOP_HEADERS = frozenset({b"upgrade", b"http2-settings"}) + + +def combined_upgrade_offer(headers: list[tuple[bytes, bytes]]) -> bytes | None: + """Return the accepted ``Upgrade`` token, honoring repeated/list-valued fields. + + Unlike uvicorn's ``_get_upgrade`` — which keeps only the tokens of the + *last* ``Connection`` field (so ``Connection: Upgrade`` followed by + ``Connection: keep-alive`` hides the offer) and the last ``Upgrade`` + field's raw value (so ``Upgrade: websocket, h2c`` matches nothing) — + repeated fields are combined per RFC 9110 section 5.3 and the ``Upgrade`` + protocol list is tokenized per section 7.8. ``websocket`` is returned + whenever it is among the offered protocols (the server may pick any + offered protocol it supports); otherwise the client's first preference is + returned. Header names must already be lowercased (both uvicorn + implementations store them that way). + """ + connection_tokens: list[bytes] = [] + upgrade_tokens: list[bytes] = [] + for name, value in headers: + if name == b"connection": + connection_tokens.extend(token.lower().strip() for token in value.split(b",")) + elif name == b"upgrade": + upgrade_tokens.extend(token for token in (token.lower().strip() for token in value.split(b",")) if token) + if b"upgrade" not in connection_tokens or not upgrade_tokens: + return None + if b"websocket" in upgrade_tokens: + return b"websocket" + return upgrade_tokens[0] + + +def offers_ignorable_upgrade(headers: list[tuple[bytes, bytes]]) -> bool: + """True when the request offers a non-WebSocket protocol switch (e.g. h2c).""" + upgrade = combined_upgrade_offer(headers) + return upgrade is not None and upgrade != b"websocket" + + +def without_upgrade_headers(headers: list[tuple[bytes, bytes]]) -> list[tuple[bytes, bytes]]: + """Drop the declined offer's hop-by-hop headers and ``Connection`` tokens.""" + sanitized: list[tuple[bytes, bytes]] = [] + for name, value in headers: + if name in UPGRADE_HOP_BY_HOP_HEADERS: + continue + if name == b"connection": + tokens = [token.strip() for token in value.split(b",")] + kept = [token for token in tokens if token and token.lower() not in UPGRADE_HOP_BY_HOP_HEADERS] + if not kept: + continue + value = b", ".join(kept) + sanitized.append((name, value)) + return sanitized + + +class UpgradeTolerantH11Protocol(H11Protocol): + """h11 protocol that hides declined non-WebSocket upgrade offers from the app. + + The stock h11 implementation already serves such requests as plain + HTTP/1.1 with the full body, but it exposes the declined offer's + hop-by-hop headers in the ASGI scope and logs a spurious + "Unsupported upgrade request." warning. ``_should_upgrade`` is the seam: + it runs right after ``self.headers`` (the same list object referenced by + ``scope["headers"]``) is populated, so sanitizing in place here is enough. + """ + + def _should_upgrade(self) -> bool: + # Reimplements the stock decision on top of combined Connection fields + # (RFC 9110 section 5.3): the stock ``_get_upgrade`` keeps only the + # last field's tokens, so ``Connection: Upgrade`` followed by + # ``Connection: keep-alive`` would hide the offer entirely. + upgrade = combined_upgrade_offer(self.headers) + if upgrade is None: + return False + if upgrade == b"websocket": + if self._should_upgrade_to_ws(): + return True + self._unsupported_upgrade_warning() + return False + self.headers[:] = without_upgrade_headers(self.headers) + return False + + +def load_http_protocol_class() -> type[asyncio.Protocol]: + """Return the HTTP protocol implementation for ``uvicorn.Config(http=...)``.""" + try: + from app.core.http_protocol_httptools import UpgradeTolerantHttpToolsProtocol + except ImportError: + # httptools is an optional (transitive) dependency; uvicorn's "auto" + # selection would fall back to h11 as well. + return UpgradeTolerantH11Protocol + return UpgradeTolerantHttpToolsProtocol diff --git a/app/core/http_protocol_httptools.py b/app/core/http_protocol_httptools.py new file mode 100644 index 0000000000..7c1bfc82c3 --- /dev/null +++ b/app/core/http_protocol_httptools.py @@ -0,0 +1,146 @@ +"""Uvicorn httptools protocol that tolerates non-WebSocket upgrade offers. + +Uvicorn's ``auto`` HTTP implementation picks the ``httptools`` parser whenever +the ``httptools`` package is importable (it is, transitively via +``fastapi[standard]``). That parser treats *any* HTTP/1.1 request carrying +``Connection: Upgrade`` as a protocol switch: httptools raises +``HttpParserUpgrade`` at the end of the headers, never delivers the body, and +uvicorn only handles the WebSocket case. For every other ``Upgrade`` offer — +most notably the cleartext HTTP/2 (``h2c``) offer JetBrains/Ktor clients attach +to ordinary Responses API POSTs — uvicorn logs "Unsupported upgrade request." +and stops feeding the parser. Two failure shapes follow: + +- body coalesced with the headers: the body is silently dropped, so the + application sees an empty body (422 from request validation); +- headers and body written as separate segments (Ktor's write pattern): the + next bytes hit the wedged parser, ``HttpParserError`` follows, and the client + receives ``400 Bad Request / Invalid HTTP request received.`` + +RFC 9110 section 7.8 lets a server ignore an upgrade offer and answer over +HTTP/1.1 — upstream OpenAI endpoints do exactly that. This subclass neutralizes +non-WebSocket upgrade offers: the request head is replayed through a fresh +parser with the declined offer's hop-by-hop headers removed, and the request is +served as plain HTTP/1.1. Legitimate WebSocket upgrades keep the stock path. + +See https://github.com/Soju06/codex-lb/issues/1757. +""" + +from __future__ import annotations + +import httptools +from uvicorn.protocols.http.httptools_impl import HttpToolsProtocol + +from app.core.http_protocol import combined_upgrade_offer, offers_ignorable_upgrade, without_upgrade_headers + + +class UpgradeTolerantHttpToolsProtocol(HttpToolsProtocol): + """httptools protocol that serves non-WebSocket upgrade offers as HTTP/1.1.""" + + def _active_parser(self) -> httptools.HttpRequestParser: + # The base class only clears ``self.parser`` in connection_lost, after + # which no parser callback or data_received can run. + parser = self.parser + assert parser is not None + return parser + + def _should_upgrade(self) -> bool: + # Combine repeated Connection fields (RFC 9110 section 5.3) so a + # trailing ``Connection: keep-alive`` field cannot hide a WebSocket + # handshake from the protocol switch (the stock ``_get_upgrade`` keeps + # only the last field's tokens). Also used by the stock parser + # callbacks to defer body handling until the handoff. + return combined_upgrade_offer(self.headers) == b"websocket" and self._should_upgrade_to_ws() + + def _paused_on_ignorable_upgrade(self) -> bool: + return self._active_parser().should_upgrade() and offers_ignorable_upgrade(self.headers) + + # -- Parser callbacks -------------------------------------------------- + # For an upgrade-offering request httptools fires on_headers_complete and + # on_message_complete *before* feed_data raises HttpParserUpgrade, and it + # never delivers the body. The stock callbacks would therefore start the + # ASGI cycle with an empty-but-complete body. Defer instead: data_received + # replays the sanitized request through a fresh parser, and these callbacks + # then run with ``should_upgrade()`` false. + + def on_headers_complete(self) -> None: + if self._paused_on_ignorable_upgrade(): + return + super().on_headers_complete() + + def on_body(self, body: bytes) -> None: + if self._paused_on_ignorable_upgrade(): + return + super().on_body(body) + + def on_message_complete(self) -> None: + if self._paused_on_ignorable_upgrade(): + return + super().on_message_complete() + + def data_received(self, data: bytes) -> None: + # Mirrors HttpToolsProtocol.data_received; the upgrade branch cannot be + # intercepted from outside because the stock method swallows the + # HttpParserUpgrade exception itself. + self._unset_keepalive_if_required() + + # Replay declined offers iteratively, not recursively: a single + # segment can pipeline many upgrade-offering requests (one replay + # each), so recursion depth would be attacker-controlled — ~66KB of + # minimal h2c GETs already exceeds Python's default 1000-frame limit, + # and the RecursionError would escape into the event loop and abort + # the connection. Each replay strips at least one declined offer from + # ``data``, so the loop terminates. + while True: + try: + self._active_parser().feed_data(data) + except httptools.HttpParserError: + msg = "Invalid HTTP request received." + self.logger.warning(msg) + self.send_400_response(msg) + except httptools.HttpParserUpgrade as exc: + if self._should_upgrade(): + self.handle_websocket_upgrade() + elif offers_ignorable_upgrade(self.headers): + data = self._continue_as_plain_http(data, exc) + continue + else: + self._unsupported_upgrade_warning() + return + + def _continue_as_plain_http(self, data: bytes, exc: httptools.HttpParserUpgrade) -> bytes: + """Decline the offered protocol switch; return the bytes to re-feed as HTTP/1.1.""" + self.logger.debug( + "Ignoring unsupported upgrade offer; serving the request as plain HTTP/1.1.", + ) + # httptools pauses at the end of the headers; the exception argument is + # the offset of the first unparsed byte in this segment (the body when + # it arrived coalesced with the headers). + offset = exc.args[0] if exc.args else len(data) + head = self._sanitized_request_head() + self.parser = httptools.HttpRequestParser(self) + try: + self.parser.set_dangerous_leniencies(lenient_data_after_close=True) + except AttributeError: # pragma: no cover - httptools < 0.6.3 + pass + # The sanitized head no longer carries upgrade headers, so re-feeding + # it cannot pause the fresh parser on the same offer (a *pipelined* + # follow-up offer pauses again and takes another loop iteration in + # data_received); malformed leftover bytes keep the stock 400 + # handling. Later segments of a split request feed the fresh parser + # through the normal data_received path. + return head + data[offset:] + + def _sanitized_request_head(self) -> bytes: + """Rebuild the parsed request head without the declined upgrade offer. + + ``self.url`` and ``self.headers`` were accumulated by the parser + callbacks of the aborted parse (header names already lowercased), so + the head is complete even when the client split it across segments. + """ + parser = self._active_parser() + method = parser.get_method() + http_version = parser.get_http_version().encode("ascii") + lines = [b"%s %s HTTP/%s\r\n" % (method, self.url, http_version)] + lines.extend(b"%s: %s\r\n" % (name, value) for name, value in without_upgrade_headers(self.headers)) + lines.append(b"\r\n") + return b"".join(lines) diff --git a/app/core/metrics/prometheus.py b/app/core/metrics/prometheus.py index 055bbbc985..55dda64dc7 100644 --- a/app/core/metrics/prometheus.py +++ b/app/core/metrics/prometheus.py @@ -302,6 +302,17 @@ def labels(self, *args: str, **kwargs: str) -> "HistogramLike": ... ["outcome"], registry=REGISTRY, ) + event_loop_lag_seconds = Gauge( + "codex_lb_event_loop_lag_seconds", + "Sampled event-loop scheduling lag (asyncio.sleep drift) in seconds", + registry=REGISTRY, + **({"multiprocess_mode": "livemax"} if MULTIPROCESS_MODE else {}), + ) + event_loop_lag_warnings_total = Counter( + "codex_lb_event_loop_lag_warnings_total", + "Total event-loop lag samples at or above the warning threshold", + registry=REGISTRY, + ) stream_keepalive_sent_total = Counter( "codex_lb_stream_keepalive_sent_total", "Total downstream SSE keepalive frames emitted by surface", @@ -385,6 +396,8 @@ def mark_process_dead() -> None: http_bridge_prewarm_total: CounterLike | None = None http_bridge_stuck_retire_total: CounterLike | None = None http_bridge_retry_circuit_total: CounterLike | None = None + event_loop_lag_seconds: GaugeLike | None = None + event_loop_lag_warnings_total: CounterLike | None = None stream_keepalive_sent_total: CounterLike | None = None stream_idle_timeout_total: CounterLike | None = None cache_invalidation_bump_failures_total: CounterLike | None = None @@ -429,6 +442,8 @@ def mark_process_dead() -> None: "cap_partition_replicas", "circuit_breaker_state", "continuity_fail_closed_total", + "event_loop_lag_seconds", + "event_loop_lag_warnings_total", "continuity_owner_resolution_total", "http_bridge_prewarm_total", "http_bridge_retry_circuit_total", diff --git a/app/core/middleware/__init__.py b/app/core/middleware/__init__.py index c2f09bb1e8..2589eea1d3 100644 --- a/app/core/middleware/__init__.py +++ b/app/core/middleware/__init__.py @@ -6,6 +6,7 @@ from app.core.middleware.request_body_limit import add_request_body_limit_middleware from app.core.middleware.request_decompression import add_request_decompression_middleware from app.core.middleware.request_id import add_request_id_middleware +from app.core.middleware.required_capability_http import add_required_capability_http_middleware from app.core.middleware.trusted_proxy_headers import add_trusted_proxy_headers_middleware __all__ = [ @@ -17,5 +18,6 @@ "add_request_body_limit_middleware", "add_request_decompression_middleware", "add_request_id_middleware", + "add_required_capability_http_middleware", "add_trusted_proxy_headers_middleware", ] diff --git a/app/core/middleware/api_firewall.py b/app/core/middleware/api_firewall.py index dccdd3a8a9..6e57d1a6df 100644 --- a/app/core/middleware/api_firewall.py +++ b/app/core/middleware/api_firewall.py @@ -1,86 +1,92 @@ from __future__ import annotations -import logging -from collections.abc import Awaitable, Callable from ipaddress import IPv4Network, IPv6Network from typing import cast from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse, Response -from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError +from fastapi.responses import JSONResponse +from starlette.datastructures import Headers +from starlette.types import ASGIApp, Receive, Scope, Send from app.core.config.settings import get_settings from app.core.errors import openai_error -from app.core.middleware.firewall_cache import get_firewall_ip_cache +from app.core.middleware.firewall_cache import FirewallIPCache, get_firewall_ip_cache from app.core.request_locality import ( FORWARDED_CHAIN_HEADER_NAMES, parse_trusted_proxy_networks, resolve_connection_client_ip, ) -from app.db.session import ( - DATABASE_POOL_RETRY_AFTER_SECONDS, - DATABASE_POOL_UNAVAILABLE_CODE, - DATABASE_POOL_UNAVAILABLE_MESSAGE, - get_request_session, -) +from app.db.session import get_background_session from app.modules.firewall.repository import FirewallRepository from app.modules.firewall.service import FirewallRepositoryPort, FirewallService -logger = logging.getLogger(__name__) +class ApiFirewallMiddleware: + """IP allowlist for ``/v1/*`` and ``/backend-api/codex/*`` HTTP requests. -def add_api_firewall_middleware(app: FastAPI) -> None: - settings = get_settings() - trusted_proxy_networks = parse_trusted_proxy_networks(settings.firewall_trusted_proxy_cidrs) - firewall_cache = get_firewall_ip_cache() - - @app.middleware("http") - async def api_firewall_middleware( - request: Request, - call_next: Callable[[Request], Awaitable[Response]], - ) -> Response: - path = request.url.path - if not _is_protected_api_path(path): - return await call_next(request) + Pure ASGI: unprotected paths pass through with a single prefix check, and + allow decisions are answered from the in-memory TTL cache; the database is + only consulted on a cache miss. + """ + + def __init__( + self, + app: ASGIApp, + *, + trust_proxy_headers: bool, + trusted_proxy_networks: tuple[IPv4Network | IPv6Network, ...], + firewall_cache: FirewallIPCache, + ) -> None: + self.app = app + self._trust_proxy_headers = trust_proxy_headers + self._trusted_proxy_networks = trusted_proxy_networks + self._firewall_cache = firewall_cache + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http" or not _is_protected_api_path(scope["path"]): + await self.app(scope, receive, send) + return + client = scope.get("client") client_ip = resolve_connection_client_ip( - request.headers, - request.client.host if request.client else None, - trust_proxy_headers=settings.firewall_trust_proxy_headers, - trusted_proxy_networks=trusted_proxy_networks, + Headers(scope=scope), + client[0] if client else None, + trust_proxy_headers=self._trust_proxy_headers, + trusted_proxy_networks=self._trusted_proxy_networks, allowed_proxy_header_names=FORWARDED_CHAIN_HEADER_NAMES, ) + firewall_cache = self._firewall_cache cached_decision = await firewall_cache.is_allowed(client_ip) if client_ip is not None else None if cached_decision is not None: is_allowed = cached_decision else: version_before_read = firewall_cache.version - try: - async with get_request_session() as session: - repository = cast(FirewallRepositoryPort, FirewallRepository(session)) - service = FirewallService(repository) - is_allowed = await service.is_ip_allowed(client_ip) - except SQLAlchemyTimeoutError: - logger.warning("database_pool_checkout_timeout surface=api_firewall") - return JSONResponse( - status_code=503, - content=openai_error( - DATABASE_POOL_UNAVAILABLE_CODE, - DATABASE_POOL_UNAVAILABLE_MESSAGE, - error_type="server_error", - ), - headers={"Retry-After": str(DATABASE_POOL_RETRY_AFTER_SECONDS)}, - ) + async with get_background_session() as session: + repository = cast(FirewallRepositoryPort, FirewallRepository(session)) + service = FirewallService(repository) + is_allowed = await service.is_ip_allowed(client_ip) if client_ip is not None: await firewall_cache.set(client_ip, is_allowed, if_version=version_before_read) if is_allowed: - return await call_next(request) + await self.app(scope, receive, send) + return - return JSONResponse( + response = JSONResponse( status_code=403, content=openai_error("ip_forbidden", "Access denied for client IP", error_type="access_error"), ) + await response(scope, receive, send) + + +def add_api_firewall_middleware(app: FastAPI) -> None: + settings = get_settings() + app.add_middleware( + ApiFirewallMiddleware, + trust_proxy_headers=settings.firewall_trust_proxy_headers, + trusted_proxy_networks=parse_trusted_proxy_networks(settings.firewall_trusted_proxy_cidrs), + firewall_cache=get_firewall_ip_cache(), + ) def _is_protected_api_path(path: str) -> bool: diff --git a/app/core/middleware/app_version.py b/app/core/middleware/app_version.py index bffd195703..a20b96afbe 100644 --- a/app/core/middleware/app_version.py +++ b/app/core/middleware/app_version.py @@ -1,20 +1,37 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable - -from fastapi import FastAPI, Request -from fastapi.responses import Response +from fastapi import FastAPI +from starlette.datastructures import MutableHeaders +from starlette.types import ASGIApp, Message, Receive, Scope, Send from app import __version__ +class AppVersionMiddleware: + """Append ``X-App-Version`` to every 200-499 HTTP response. + + Pure ASGI (no ``BaseHTTPMiddleware``) so streaming responses relay chunks + without an extra task and memory-stream hop. 5xx responses and WebSocket + scopes never carry the header, and a route-owned ``X-App-Version`` value is + preserved (see ``openspec/specs/api-response-metadata/spec.md``). + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + async def send_with_app_version(message: Message) -> None: + if message["type"] == "http.response.start" and 200 <= message["status"] < 500: + headers = MutableHeaders(raw=message.setdefault("headers", [])) + headers.setdefault("X-App-Version", __version__) + await send(message) + + await self.app(scope, receive, send_with_app_version) + + def add_app_version_middleware(app: FastAPI) -> None: - @app.middleware("http") - async def app_version_middleware( - request: Request, - call_next: Callable[[Request], Awaitable[Response]], - ) -> Response: - response = await call_next(request) - if 200 <= response.status_code < 500: - response.headers.setdefault("X-App-Version", __version__) - return response + app.add_middleware(AppVersionMiddleware) diff --git a/app/core/middleware/inflight.py b/app/core/middleware/inflight.py index c324ac7797..6f8faa3cdd 100644 --- a/app/core/middleware/inflight.py +++ b/app/core/middleware/inflight.py @@ -1,6 +1,7 @@ from __future__ import annotations from importlib import import_module +from types import ModuleType from starlette.responses import JSONResponse from starlette.types import ASGIApp, Receive, Scope, Send @@ -37,6 +38,9 @@ class InFlightMiddleware: def __init__(self, app: ASGIApp) -> None: self.app = app + # Resolved lazily on the first request (import-cycle safety) and cached + # so the hot path skips the per-request sys.modules lookup. + self._shutdown_state: ModuleType | None = None async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: scope_type = scope["type"] @@ -44,7 +48,9 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: await self.app(scope, receive, send) return - shutdown_state = import_module("app.core.shutdown") + shutdown_state = self._shutdown_state + if shutdown_state is None: + shutdown_state = self._shutdown_state = import_module("app.core.shutdown") if scope_type == "websocket": # Register before checking the drain barrier. A synchronous signal diff --git a/app/core/middleware/request_decompression.py b/app/core/middleware/request_decompression.py index 99b63aa503..7e8c4472b5 100644 --- a/app/core/middleware/request_decompression.py +++ b/app/core/middleware/request_decompression.py @@ -3,14 +3,14 @@ import gzip import io import zlib -from collections.abc import Awaitable, Callable from typing import Protocol import zstandard as zstd -from fastapi import FastAPI, Request -from fastapi.responses import Response +from fastapi import FastAPI from starlette._utils import get_route_path -from starlette.requests import ClientDisconnect +from starlette.datastructures import Headers +from starlette.requests import ClientDisconnect, Request +from starlette.types import ASGIApp, Message, Receive, Scope, Send from app.core.middleware.request_body_limit import ( REQUEST_BODY_TOO_LARGE_MESSAGE, @@ -108,58 +108,110 @@ def _decompress_body(data: bytes, encodings: list[str], max_size: int) -> bytes: return result -def _replace_request_body(request: Request, body: bytes) -> None: - request._body = body +def _rewrite_scope_headers_for_body(scope: Scope, body_length: int) -> None: + """Drop content-encoding/content-length and declare the decompressed length.""" headers: list[tuple[bytes, bytes]] = [] - for key, value in request.scope.get("headers", []): + for key, value in scope.get("headers", []): if key.lower() in (b"content-encoding", b"content-length"): continue headers.append((key, value)) - headers.append((b"content-length", str(len(body)).encode("ascii"))) - request.scope["headers"] = headers - # Ensure subsequent request.headers reflects the updated scope headers. - request.__dict__.pop("_headers", None) + headers.append((b"content-length", str(body_length).encode("ascii"))) + scope["headers"] = headers -def add_request_decompression_middleware(app: FastAPI) -> None: - @app.middleware("http") - async def request_decompression_middleware( - request: Request, - call_next: Callable[[Request], Awaitable[Response]], - ) -> Response: - content_encoding = request.headers.get("content-encoding") +async def _drain_request_body(receive: Receive) -> bytes: + """Read the full request body from ``receive``. + + Mirrors ``Request.body()`` semantics: a mid-body ``http.disconnect`` raises + ``ClientDisconnect``, and receive failures propagate. The caller's + ``receive`` is the body-limit middleware's limited receive, so the wire-size + cap (``_RequestBodyTooLarge``) propagates through this drain unchanged. + """ + chunks = bytearray() + while True: + message = await receive() + if message["type"] == "http.disconnect": + raise ClientDisconnect() + chunks.extend(message.get("body", b"")) + if not message.get("more_body", False): + return bytes(chunks) + + +class RequestDecompressionMiddleware: + """Decompress zstd/gzip/deflate request bodies with per-layer decode budgets. + + Pure ASGI replacement for the previous ``BaseHTTPMiddleware`` dispatch: + requests without ``Content-Encoding`` (the common case) pass straight + through. Encoded requests are drained through the upstream receive (the + body-limit middleware's limited receive, so the wire-size cap still applies + to the compressed bytes), decompressed under the per-path budget, and the + downstream app is given a replay receive that yields the decompressed body + once and then delegates to the original receive so ``http.disconnect`` is + still observed (this replaces the ``_CachedRequest`` body replay the + BaseHTTP wrapper used to provide). + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + content_encoding = Headers(scope=scope).get("content-encoding") if not content_encoding: - return await call_next(request) + await self.app(scope, receive, send) + return encodings = [enc.strip().lower() for enc in content_encoding.split(",") if enc.strip()] if not encodings: - return await call_next(request) - max_size = request_body_limit_for_path(get_route_path(request.scope)) - try: - body = await request.body() - except ClientDisconnect: - raise + await self.app(scope, receive, send) + return + + max_size = request_body_limit_for_path(get_route_path(scope)) + body = await _drain_request_body(receive) try: decompressed = _decompress_body(body, encodings, max_size) except _DecompressedBodyTooLarge: - return request_ingress_error_response( - request, + response = request_ingress_error_response( + Request(scope), status_code=413, code="payload_too_large", message=REQUEST_BODY_TOO_LARGE_MESSAGE, ) + await response(scope, receive, send) + return except ValueError: - return request_ingress_error_response( - request, + response = request_ingress_error_response( + Request(scope), status_code=400, code="invalid_request", message="Unsupported Content-Encoding", ) + await response(scope, receive, send) + return except Exception: - return request_ingress_error_response( - request, + response = request_ingress_error_response( + Request(scope), status_code=400, code="invalid_request", message="Request body is compressed but could not be decompressed", ) - _replace_request_body(request, decompressed) - return await call_next(request) + await response(scope, receive, send) + return + + _rewrite_scope_headers_for_body(scope, len(decompressed)) + body_replayed = False + + async def replay_receive() -> Message: + nonlocal body_replayed + if not body_replayed: + body_replayed = True + return {"type": "http.request", "body": decompressed, "more_body": False} + return await receive() + + await self.app(scope, replay_receive, send) + + +def add_request_decompression_middleware(app: FastAPI) -> None: + app.add_middleware(RequestDecompressionMiddleware) diff --git a/app/core/middleware/request_id.py b/app/core/middleware/request_id.py index 1ee1af006a..d7ba92820a 100644 --- a/app/core/middleware/request_id.py +++ b/app/core/middleware/request_id.py @@ -1,10 +1,10 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable from uuid import uuid4 -from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse +from fastapi import FastAPI +from starlette.datastructures import MutableHeaders +from starlette.types import ASGIApp, Message, Receive, Scope, Send from app.core.utils.request_id import ( clear_request_id, @@ -16,22 +16,57 @@ ) -def add_request_id_middleware(app: FastAPI) -> None: - @app.middleware("http") - async def request_id_middleware( - request: Request, - call_next: Callable[[Request], Awaitable[JSONResponse]], - ) -> JSONResponse: - inbound_request_id = request.headers.get("x-request-id") or request.headers.get("request-id") - request_id = inbound_request_id or str(uuid4()) +def _inbound_request_id(scope: Scope) -> str | None: + """Return the first ``x-request-id`` value, else the first ``request-id`` value.""" + x_request_id: bytes | None = None + request_id: bytes | None = None + for name, value in scope.get("headers", []): + lowered = name.lower() + if x_request_id is None and lowered == b"x-request-id": + x_request_id = value + elif request_id is None and lowered == b"request-id": + request_id = value + inbound = x_request_id or request_id + if not inbound: + return None + return inbound.decode("latin-1") + + +class RequestIdMiddleware: + """Bind request-id contextvars around the request and echo ``x-request-id``. + + Pure ASGI: the contextvars are set and reset in the same task that runs the + downstream app, so they stay visible for the whole response stream (with + ``BaseHTTPMiddleware`` they were reset when the dispatch returned, before + the body finished streaming). + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + request_id = _inbound_request_id(scope) or str(uuid4()) request_id_token = set_request_id(request_id) request_scope_token = set_request_scope_id(str(uuid4())) + + async def send_with_request_id(message: Message) -> None: + if message["type"] == "http.response.start": + headers = MutableHeaders(raw=message.setdefault("headers", [])) + headers.setdefault("x-request-id", request_id) + await send(message) + try: - response = await call_next(request) - response.headers.setdefault("x-request-id", request_id) - return response + await self.app(scope, receive, send_with_request_id) finally: reset_request_scope_id(request_scope_token) reset_request_id(request_id_token) clear_request_scope_id() clear_request_id() + + +def add_request_id_middleware(app: FastAPI) -> None: + app.add_middleware(RequestIdMiddleware) diff --git a/app/core/middleware/required_capability_http.py b/app/core/middleware/required_capability_http.py new file mode 100644 index 0000000000..ab7229a034 --- /dev/null +++ b/app/core/middleware/required_capability_http.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import logging +import time + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse +from starlette._utils import get_route_path +from starlette.types import ASGIApp, Receive, Scope, Send + +from app.core.auth.dependencies import validate_required_proxy_api_key_authorization +from app.core.clients.proxy import CODEX_LB_REQUIRED_CAPABILITY_HEADER +from app.core.errors import openai_error +from app.core.exceptions import ProxyAuthError, ProxyRequiredCapabilityTransportError +from app.core.runtime_logging import log_error_response +from app.modules.proxy.images_observability import ( + IMAGE_ROUTE_STARTED_AT_STATE, + record_images_route_observability, +) + +logger = logging.getLogger(__name__) + +_REQUIRED_CAPABILITY_HEADER_BYTES = CODEX_LB_REQUIRED_CAPABILITY_HEADER.lower().encode("latin-1") + +_JSON_BODY_DENY_PATHS = frozenset( + { + "/backend-api/codex/responses", + "/backend-api/codex/responses/compact", + "/backend-api/codex/images/generations", + "/v1/responses", + "/v1/responses/compact", + "/v1/chat/completions", + "/v1/embeddings", + "/v1/images/generations", + "/v1/reset-credit", + "/v1/warmup", + "/api/codex/rate-limit-reset-credits/consume", + } +) + + +def _is_pre_body_deny_path(path: str) -> bool: + normalized = path.rstrip("/") + return normalized in _JSON_BODY_DENY_PATHS or normalized.startswith("/v1/warmup/") + + +def _has_required_capability_header(scope: Scope) -> bool: + for name, _value in scope.get("headers", []): + if name.lower() == _REQUIRED_CAPABILITY_HEADER_BYTES: + return True + return False + + +class RequiredCapabilityHttpMiddleware: + """Deny capability-marked POSTs on JSON-body proxy paths before the body is read. + + Pure ASGI with cheap synchronous guards first; the ``Request`` object is + only constructed on the cold deny path. + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if ( + scope["type"] != "http" + or scope["method"] != "POST" + or not _has_required_capability_header(scope) + or not _is_pre_body_deny_path(get_route_path(scope)) + ): + await self.app(scope, receive, send) + return + + request = Request(scope) + try: + await validate_required_proxy_api_key_authorization(request.headers.get("authorization")) + except ProxyAuthError as exc: + response = _capability_error_response(request, exc) + else: + response = _capability_error_response(request, ProxyRequiredCapabilityTransportError()) + await response(scope, receive, send) + + +def add_required_capability_http_middleware(app: FastAPI) -> None: + app.add_middleware(RequiredCapabilityHttpMiddleware) + + +def _capability_error_response( + request: Request, + exc: ProxyAuthError | ProxyRequiredCapabilityTransportError, +) -> JSONResponse: + log_error_response( + logger, + request, + exc.status_code, + exc.code, + exc.message, + category="openai_error_response", + ) + path = get_route_path(request.scope) + if path.rstrip("/").endswith("/images/generations"): + started_at = getattr(request.state, IMAGE_ROUTE_STARTED_AT_STATE, None) + if not isinstance(started_at, float): + started_at = time.perf_counter() + record_images_route_observability( + route="generations", + model=None, + stream=False, + status=exc.status_code, + outcome="auth_error" if isinstance(exc, ProxyAuthError) else "invalid_request", + started_at=started_at, + ) + return JSONResponse( + status_code=exc.status_code, + content=openai_error(exc.code, exc.message, error_type=exc.error_type), + ) diff --git a/app/core/openai/chat_requests.py b/app/core/openai/chat_requests.py index a11506fd3f..3485a18c92 100644 --- a/app/core/openai/chat_requests.py +++ b/app/core/openai/chat_requests.py @@ -124,14 +124,17 @@ def _validate_messages(self) -> "ChatCompletionsRequest": @model_validator(mode="after") def _validate_tools(self) -> "ChatCompletionsRequest": responses_shaped_payload = not self.messages and self.input is not None - self.tools = validate_tool_types( + validated = validate_tool_types( self.tools, allow_builtin_tools=responses_shaped_payload, ) + if validated != self.tools: + self.tools = validated return self def to_responses_request(self) -> ResponsesRequest: data = self.model_dump(mode="json", exclude_none=True) + tools_were_set = "tools" in self.model_fields_set messages = data.pop("messages", None) data.pop("store", None) data.pop("n", None) @@ -141,10 +144,7 @@ def to_responses_request(self) -> ResponsesRequest: stream_options = data.pop("stream_options", None) raw_tools = data.pop("tools", []) raw_tool_choice = data.pop("tool_choice", None) - reasoning_effort = data.pop("reasoning_effort", None) preserve_instruction_roles = _is_json_object_response_format(response_format) - if reasoning_effort is not None and "reasoning" not in data: - data["reasoning"] = {"effort": reasoning_effort} normalize_reasoning_aliases(data) if response_format is not None: _apply_response_format(data, response_format) @@ -160,7 +160,8 @@ def to_responses_request(self) -> ResponsesRequest: # a missing or explicitly empty `messages` field. if not isinstance(data.get("instructions"), str): data["instructions"] = "" - data["tools"] = raw_tools + if tools_were_set: + data["tools"] = raw_tools if raw_tool_choice is not None: data["tool_choice"] = raw_tool_choice return ResponsesRequest.model_validate(data) @@ -177,7 +178,8 @@ def to_responses_request(self) -> ResponsesRequest: ) data["instructions"] = instructions data["input"] = input_items - data["tools"] = tools + if tools_were_set: + data["tools"] = tools if tool_choice is not None: data["tool_choice"] = tool_choice return ResponsesRequest.model_validate(data) diff --git a/app/core/openai/chat_responses.py b/app/core/openai/chat_responses.py index d334bffae5..78ad429dff 100644 --- a/app/core/openai/chat_responses.py +++ b/app/core/openai/chat_responses.py @@ -353,17 +353,19 @@ def iter_chat_chunks( response = payload.get("response") if isinstance(response, dict): maybe_error = response.get("error") - if isinstance(maybe_error, dict): + if isinstance(maybe_error, dict) and maybe_error: error = maybe_error else: maybe_error = payload.get("error") - if isinstance(maybe_error, dict): + if isinstance(maybe_error, dict) and maybe_error: error = maybe_error if error is not None: error_payload: dict[str, JsonValue] = {"error": error} - yield _dump_sse(error_payload) - yield "data: [DONE]\n\n" - return + else: + error_payload = _default_error_envelope().model_dump(mode="json", exclude_none=True) + yield _dump_sse(error_payload) + yield "data: [DONE]\n\n" + return if event_type in ("response.completed", "response.incomplete"): for tool_state in state.tool_calls: stream_delta = tool_state.build_stream_delta() @@ -448,6 +450,9 @@ async def stream_chat_chunks( if chunk.strip() == "data: [DONE]": terminal_chunk_sent = True break + if not terminal_chunk_sent: + yield _dump_sse(_upstream_stream_truncated_error_payload()) + yield "data: [DONE]\n\n" async def collect_chat_completion(stream: AsyncIterator[str], model: str) -> ChatCompletionResult: @@ -459,6 +464,8 @@ async def collect_chat_completion(stream: AsyncIterator[str], model: str) -> Cha incomplete_reason: str | None = None tool_index = ToolCallIndex() tool_calls: list[ToolCallState] = [] + terminal_error: ChatCompletionResult | None = None + terminal_event_seen = False async for line in stream: payload = _parse_data(line) @@ -477,6 +484,8 @@ async def collect_chat_completion(stream: AsyncIterator[str], model: str) -> Cha if tool_delta is not None: _merge_tool_call_delta(tool_calls, tool_delta) if event_type in ("response.failed", "error"): + if terminal_error is not None: + continue error = None if event_type == "response.failed": response = payload.get("response") @@ -488,10 +497,12 @@ async def collect_chat_completion(stream: AsyncIterator[str], model: str) -> Cha maybe_error = payload.get("error") if isinstance(maybe_error, dict): error = maybe_error - if error is not None: - return _error_envelope_from_payload(error) - return _default_error_envelope() + terminal_error = _error_envelope_from_payload(error) if error is not None else _default_error_envelope() + continue + if terminal_error is not None: + continue if event_type in ("response.completed", "response.incomplete"): + terminal_event_seen = True response = payload.get("response") if isinstance(response, dict): response_id_value = response.get("id") @@ -501,6 +512,11 @@ async def collect_chat_completion(stream: AsyncIterator[str], model: str) -> Cha if event_type == "response.incomplete": incomplete_reason = _finish_reason_from_incomplete(response) + if terminal_error is not None: + return terminal_error + if not terminal_event_seen: + return _upstream_stream_truncated_error() + message_content: str | None = "".join(content_parts) message_refusal = "".join(refusal_parts) or None message_tool_calls = _compact_tool_calls(tool_calls) @@ -575,6 +591,20 @@ def _dump_sse(payload: dict[str, JsonValue]) -> str: return format_sse_data(payload) +def _upstream_stream_truncated_error_payload() -> dict[str, JsonValue]: + return { + "error": { + "message": "Responses stream ended before a terminal event", + "type": "server_error", + "code": "upstream_stream_truncated", + } + } + + +def _upstream_stream_truncated_error() -> OpenAIErrorEnvelope: + return OpenAIErrorEnvelope.model_validate(_upstream_stream_truncated_error_payload()) + + def _finish_reason_from_incomplete(response: JsonValue | None) -> str: response_mapping = _as_mapping(response) if response_mapping is None: diff --git a/app/core/openai/model_registry.py b/app/core/openai/model_registry.py index eaea3c16db..3a0c3d91f7 100644 --- a/app/core/openai/model_registry.py +++ b/app/core/openai/model_registry.py @@ -142,7 +142,7 @@ class ModelRegistryExport: ) # GPT-5.6 ships to four additional plan tiers upstream -# (codex-rs/models-manager/models.json at rust-v0.144.1). +# (codex-rs/models-manager/models.json at rust-v0.145.0). _BOOTSTRAP_GPT56_AVAILABLE_IN_PLANS = frozenset( { *_BOOTSTRAP_AVAILABLE_IN_PLANS, @@ -219,8 +219,10 @@ def _gpt56_raw( availability_nux: dict[str, JsonValue] | None = None, ) -> dict[str, JsonValue]: """Raw catalog fields for the GPT-5.6 family, mirroring the upstream - bundled catalog (codex-rs/models-manager/models.json at rust-v0.144.1) - field-for-field. The ~16.5 KB ``base_instructions`` string and the + bundled catalog (codex-rs/models-manager/models.json at rust-v0.145.0) + field-for-field, with one tracked exception: ``max_context_window``, which + upstream later raised from 272,000 to 872,000 (see the field comment + below). The ~16.5 KB ``base_instructions`` string and the personality-templated ``model_messages`` object are deliberately not bundled; the live upstream registry supplies them on the first refresh. """ @@ -237,6 +239,16 @@ def _gpt56_raw( "use_responses_lite": True, "include_skills_usage_instructions": False, "auto_review_model_override": None, + # Upstream raised only the ceiling: ``max_context_window`` 272000 -> + # 872000 with ``context_window`` unchanged at 272000. ``_bootstrap_model`` + # synthesizes ``max_context_window == context_window``, so the family + # ceiling has to override it here, the same decoupling ``gpt-5.4`` uses. + # Pinned evidence: openai/codex commit + # 2eee483e49f88b868f67364134a658b3298e6c14 -- "Raise the GPT-5.6 maximum + # context window" (openai/codex#39102). Not yet in a ``rust-v*`` release + # tag; ``rust-v0.148.0-alpha.21`` still ships 272000. Re-pin to the tag + # once one carries it. + "max_context_window": 872_000, "auto_compact_token_limit": None, "comp_hash": "3000", "reasoning_summary_format": "experimental", @@ -270,7 +282,7 @@ def _gpt56_raw( prefer_websockets=True, minimal_client_version="0.144.0", reasoning_levels=_REASONING_LEVELS_ULTRA, - context_window=372_000, + context_window=272_000, default_reasoning_level="low", priority=1, available_in_plans=_BOOTSTRAP_GPT56_AVAILABLE_IN_PLANS, @@ -283,7 +295,7 @@ def _gpt56_raw( prefer_websockets=True, minimal_client_version="0.144.0", reasoning_levels=_REASONING_LEVELS_ULTRA, - context_window=372_000, + context_window=272_000, default_reasoning_level="medium", priority=2, available_in_plans=_BOOTSTRAP_GPT56_AVAILABLE_IN_PLANS, @@ -296,7 +308,7 @@ def _gpt56_raw( prefer_websockets=True, minimal_client_version="0.144.0", reasoning_levels=_REASONING_LEVELS_MAX, - context_window=372_000, + context_window=272_000, default_reasoning_level="medium", priority=3, available_in_plans=_BOOTSTRAP_GPT56_AVAILABLE_IN_PLANS, diff --git a/app/core/openai/models.py b/app/core/openai/models.py index 4bf025cc25..b6d73b9c66 100644 --- a/app/core/openai/models.py +++ b/app/core/openai/models.py @@ -142,5 +142,14 @@ def _normalize_usage(cls, value: ModelLikeInput | None) -> ResponseUsage | None: return _normalize_model_value(ResponseUsage, value) +def normalize_compaction_item_id(item_id: object) -> str | None: + """Return a valid compaction ID without changing opaque upstream identity.""" + if not isinstance(item_id, str): + return None + if item_id.startswith("cmp_"): + return item_id + return None + + OpenAIResponseResult: TypeAlias = OpenAIResponsePayload | OpenAIErrorEnvelope CompactResponseResult: TypeAlias = CompactResponsePayload | OpenAIErrorEnvelope diff --git a/app/core/openai/parsing.py b/app/core/openai/parsing.py index 9b6f614352..295ffd71be 100644 --- a/app/core/openai/parsing.py +++ b/app/core/openai/parsing.py @@ -17,6 +17,37 @@ _RESPONSE_ADAPTER = TypeAdapter(OpenAIResponsePayload) _COMPACT_RESPONSE_ADAPTER = TypeAdapter(CompactResponsePayload) +# Stream lifecycle frames are the only events whose validated model fields the +# proxy consumes (usage settlement, error normalization, response-id capture). +# Hot streaming paths validate only these frames and classify everything else +# from the already-parsed payload dict via ``classify_event_type``. +_LIFECYCLE_EVENT_TYPES = frozenset( + { + "response.created", + "response.completed", + "response.incomplete", + "response.failed", + "error", + } +) + + +def classify_event_type(payload: JsonValue | None) -> str | None: + """Classify an SSE event type from an already-parsed payload dict. + + Mirrors the dict branch of the proxy's ``_event_type_from_payload``: + a string ``type`` field wins; a typeless payload carrying a dict + ``error`` classifies as ``"error"``. No pydantic validation is run. + """ + if not isinstance(payload, dict): + return None + payload_type = payload.get("type") + if isinstance(payload_type, str): + return payload_type + if isinstance(payload.get("error"), dict): + return "error" + return None + def parse_sse_event(line: str) -> OpenAIEvent | None: return parse_sse_event_payload(parse_sse_data_json(line)) diff --git a/app/core/openai/public_output.py b/app/core/openai/public_output.py deleted file mode 100644 index 8dc0ab9888..0000000000 --- a/app/core/openai/public_output.py +++ /dev/null @@ -1,176 +0,0 @@ -"""Shared normalization for output items exposed by the public Responses API.""" - -from __future__ import annotations - -import re -from collections.abc import Mapping - -from app.core.types import JsonValue -from app.core.utils.json_guards import is_json_mapping - -_PUBLIC_RESPONSE_OUTPUT_ITEM_TYPES = frozenset( - { - "message", - "compaction", - "function_call", - "function_call_output", - "reasoning", - "web_search_call", - "file_search_call", - "computer_call", - "code_interpreter_call", - "mcp_approval_request", - "mcp_list_tools", - "output_image", - } -) -PUBLIC_RESPONSE_TEXT_PART_TYPES = frozenset({"output_text", "input_text", "text", "refusal"}) -_REASONING_SUMMARY_BLANK_HTML_COMMENT_RE = re.compile(r"(?m)^[ \t]*[ \t]*(?:\r?\n|\Z)") -MAX_PUBLIC_RESPONSE_OUTPUT_ITEMS = 4096 - - -def collect_public_output_item_event( - payload: Mapping[str, JsonValue], - output_items: dict[int, dict[str, JsonValue]], -) -> bool: - """Collect one bounded streamed output item for terminal backfill. - - Returns False only when an output-item lifecycle event is malformed or - exceeds the bound. Callers that persist replay proofs use that result to - fail closed; public response rendering may simply ignore the bad item. - """ - - event_type = payload.get("type") - if event_type not in ("response.output_item.added", "response.output_item.done"): - return True - output_index = payload.get("output_index") - item = payload.get("item") - if ( - not isinstance(output_index, int) - or isinstance(output_index, bool) - or output_index < 0 - or output_index >= MAX_PUBLIC_RESPONSE_OUTPUT_ITEMS - or not is_json_mapping(item) - ): - return False - if output_index not in output_items and len(output_items) >= MAX_PUBLIC_RESPONSE_OUTPUT_ITEMS: - return False - output_items[output_index] = dict(item) - return True - - -def merge_public_response_output_items( - response: Mapping[str, JsonValue], - output_items: Mapping[int, dict[str, JsonValue]], -) -> dict[str, JsonValue]: - """Backfill an empty terminal response from streamed output items.""" - - merged = dict(response) - if not output_items: - return merged - existing_output = response.get("output") - if isinstance(existing_output, list) and existing_output: - return merged - merged["output"] = [item for _, item in sorted(output_items.items())] - return merged - - -def strip_blank_html_comment_lines(text: str) -> str: - terminal_match = None - for match in _REASONING_SUMMARY_BLANK_HTML_COMMENT_RE.finditer(text): - if match.end() == len(text): - terminal_match = match - cleaned, count = _REASONING_SUMMARY_BLANK_HTML_COMMENT_RE.subn("", text) - if count == 0: - return text - if terminal_match is not None: - return cleaned.rstrip("\r\n") - return cleaned - - -def normalize_public_output_item(item: Mapping[str, JsonValue]) -> dict[str, JsonValue] | None: - """Return the exact item representation exposed on the public API.""" - - item_type = item.get("type") - if item_type == "reasoning": - return _normalize_reasoning_output_item(item) - if isinstance(item_type, str) and is_public_passthrough_output_item_type(item_type): - return dict(item) - text_value = extract_public_output_item_text(item) - if text_value is None: - return None - normalized: dict[str, JsonValue] = { - "type": "message", - "role": "assistant", - "status": item.get("status") if isinstance(item.get("status"), str) else "completed", - "content": [{"type": "output_text", "text": text_value}], - } - item_id = item.get("id") - if isinstance(item_id, str) and item_id: - normalized["id"] = item_id - return normalized - - -def extract_public_output_item_text(item: Mapping[str, JsonValue]) -> str | None: - direct_text = item.get("text") - if isinstance(direct_text, str) and direct_text: - return direct_text - content = item.get("content") - if is_json_mapping(content): - content_parts: list[Mapping[str, JsonValue]] = [content] - elif isinstance(content, list): - content_parts = [part for part in content if is_json_mapping(part)] - else: - content_parts = [] - parts: list[str] = [] - for part in content_parts: - part_type = part.get("type") - if isinstance(part_type, str) and part_type in PUBLIC_RESPONSE_TEXT_PART_TYPES: - text = part.get("text") - if isinstance(text, str) and text: - parts.append(text) - continue - text = part.get("text") - if isinstance(text, str) and text: - parts.append(text) - if parts: - return "".join(parts) - summary = item.get("summary") - if isinstance(summary, str) and summary: - return summary - return None - - -def is_public_passthrough_output_item_type(item_type: str) -> bool: - return ( - item_type in _PUBLIC_RESPONSE_OUTPUT_ITEM_TYPES - or item_type.endswith("_call") - or item_type.endswith("_call_output") - ) - - -def _normalize_reasoning_output_item(item: Mapping[str, JsonValue]) -> dict[str, JsonValue]: - normalized = dict(item) - summary = item.get("summary") - if not isinstance(summary, list): - return normalized - - normalized_summary: list[JsonValue] = [] - changed = False - for part in summary: - if not is_json_mapping(part): - normalized_summary.append(part) - continue - text = part.get("text") - if part.get("type") != "summary_text" or not isinstance(text, str): - normalized_summary.append(dict(part)) - continue - cleaned = strip_blank_html_comment_lines(text) - normalized_part = dict(part) - normalized_part["text"] = cleaned - normalized_summary.append(normalized_part) - changed = changed or cleaned != text - - if changed: - normalized["summary"] = normalized_summary - return normalized diff --git a/app/core/openai/requests.py b/app/core/openai/requests.py index 658830f6e4..2bb8451662 100644 --- a/app/core/openai/requests.py +++ b/app/core/openai/requests.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import cast -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator, model_validator from app.core.openai.exceptions import ClientPayloadError from app.core.openai.tool_call_safety import is_downstream_side_effect_tool_call_item @@ -613,6 +613,8 @@ class ResponsesTextControls(BaseModel): class ResponsesRequest(BaseModel): model_config = ConfigDict(extra="allow") + _codex_lb_client_reasoning_effort: str | None = PrivateAttr(default=None) + _codex_lb_provider_reasoning_effort_materialized: bool = PrivateAttr(default=False) @model_validator(mode="before") @classmethod @@ -732,7 +734,9 @@ def model_dump_for_forwarding(self) -> MutableJsonObject: return payload def to_payload(self) -> JsonObject: - return _strip_unsupported_fields(self.model_dump_for_forwarding()) + payload = _strip_unsupported_fields(self.model_dump_for_forwarding()) + _normalize_compaction_trigger_singleton(payload) + return payload def to_replay_safety_payload(self) -> JsonObject: return _strip_unsupported_fields(self.model_dump_for_forwarding(), strip_replayed_tool_call_namespaces=False) @@ -740,6 +744,7 @@ def to_replay_safety_payload(self) -> JsonObject: class ResponsesCompactRequest(BaseModel): model_config = ConfigDict(extra="allow") + _codex_lb_client_reasoning_effort: str | None = PrivateAttr(default=None) @model_validator(mode="before") @classmethod @@ -984,6 +989,7 @@ def _strip_compact_unsupported_fields(payload: MutableJsonObject) -> MutableJson if is_json_mapping(normalized_payload): payload = dict(normalized_payload) _trim_compact_input_for_upstream(payload) + _normalize_compaction_trigger_singleton(payload) payload.pop("store", None) payload.pop("text", None) payload.pop("tools", None) @@ -993,6 +999,21 @@ def _strip_compact_unsupported_fields(payload: MutableJsonObject) -> MutableJson return payload +def _normalize_compaction_trigger_singleton(payload: MutableJsonObject) -> None: + input_value = payload.get("input") + if not is_json_list(input_value) or not input_value: + return + + trigger_items = [item for item in input_value if is_json_mapping(item) and item.get("type") == "compaction_trigger"] + if not trigger_items: + return + + payload["input"] = [ + item for item in input_value if not (is_json_mapping(item) and item.get("type") == "compaction_trigger") + ] + cast(list[JsonValue], payload["input"]).append({"type": "compaction_trigger"}) + + def _trim_compact_input_for_upstream(payload: MutableJsonObject) -> None: input_value = payload.get("input") if not is_json_list(input_value): @@ -1198,6 +1219,12 @@ def _compact_discard_consumed_continuity_output_pairs( latest_index = len(input_value) - 1 latest_mapping = _json_mapping_or_none(input_value[latest_index]) latest_type = latest_mapping.get("type") if latest_mapping is not None else None + if latest_type == "compaction_trigger": + if latest_index == 0: + return + latest_index -= 1 + latest_mapping = _json_mapping_or_none(input_value[latest_index]) + latest_type = latest_mapping.get("type") if latest_mapping is not None else None if ( latest_mapping is None or not isinstance(latest_type, str) @@ -1231,33 +1258,55 @@ def _compact_terminal_required_indices( latest_index = len(input_value) - 1 latest_mapping = _json_mapping_or_none(input_value[latest_index]) latest_type = latest_mapping.get("type") if latest_mapping is not None else None + terminal_trigger_indices: set[int] = set() + if latest_type == "compaction_trigger": + terminal_trigger_indices.add(latest_index) + if latest_index == 0: + return terminal_trigger_indices, True, False + latest_index -= 1 + latest_mapping = _json_mapping_or_none(input_value[latest_index]) + latest_type = latest_mapping.get("type") if latest_mapping is not None else None + + def with_terminal_trigger(indices: set[int]) -> set[int]: + return indices | terminal_trigger_indices + if latest_type not in _COMPACT_TOOL_CALL_ITEM_TYPES | _COMPACT_TOOL_CALL_OUTPUT_ITEM_TYPES: - return {latest_index}, True, False + return with_terminal_trigger({latest_index}), True, False if latest_mapping is not None and _compact_item_is_state_anchor(latest_mapping): - return _compact_required_terminal_indices(input_value, latest_index, token_counts), True, True + terminal_indices = _compact_required_terminal_indices(input_value, latest_index, token_counts) + return with_terminal_trigger(terminal_indices), True, True if latest_mapping is not None and _compact_item_has_elidable_inline_image(latest_mapping): - return _compact_required_terminal_indices(input_value, latest_index, token_counts), True, True + terminal_indices = _compact_required_terminal_indices(input_value, latest_index, token_counts) + return with_terminal_trigger(terminal_indices), True, True matching_call_index = _compact_matching_tool_call_index(input_value, latest_index) if latest_mapping is not None and _compact_terminal_item_is_side_effect( input_value, latest_index, matching_call_index=matching_call_index, ): - return _compact_required_terminal_indices(input_value, latest_index, token_counts), True, True + terminal_indices = _compact_required_terminal_indices(input_value, latest_index, token_counts) + return with_terminal_trigger(terminal_indices), True, True if latest_type in _COMPACT_TOOL_CALL_OUTPUT_ITEM_TYPES and has_continuity_anchor and matching_call_index is None: - return {latest_index}, True, False + return with_terminal_trigger({latest_index}), True, False if latest_type in _COMPACT_TOOL_CALL_ITEM_TYPES: - return _compact_required_terminal_indices(input_value, latest_index, token_counts), True, True + terminal_indices = _compact_required_terminal_indices(input_value, latest_index, token_counts) + return with_terminal_trigger(terminal_indices), True, True + + if terminal_trigger_indices: + # The trigger is the only mandatory suffix sentinel. An ordinary + # matched tool pair immediately before it remains best-effort context + # and may be dropped when other anchors consume the wire budget. + return terminal_trigger_indices, True, False paired_tail = _compact_reconciled_tool_call_indices( input_value, - {latest_index}, + with_terminal_trigger({latest_index}), token_counts=token_counts, token_budget=_MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS, ) if latest_index in paired_tail: return paired_tail, False, False - return set(), False, False + return terminal_trigger_indices, bool(terminal_trigger_indices), False def _compact_item_has_elidable_inline_image(item: JsonValue) -> bool: @@ -1643,6 +1692,36 @@ def _compact_item_texts(item: Mapping[str, JsonValue]) -> list[str]: return texts +def responses_input_contains_goal_continuation_context(input_value: JsonValue) -> bool: + """Return whether Responses input carries Codex's goal-continuation marker.""" + + if not is_json_list(input_value): + return False + for item in input_value: + if not is_json_mapping(item): + continue + for text in _compact_item_texts(item): + if text.lstrip().startswith(_GOAL_CONTINUATION_CONTEXT_PREFIX): + return True + return False + + +def responses_request_contains_goal_continuation_context(payload: ResponsesRequest) -> bool: + """Return whether a normalized request carries Codex's goal restart marker.""" + + # ResponsesRequest normalization lifts developer/system input messages into + # ``instructions``. The marker can therefore disappear from ``input`` and + # follow pre-existing instruction text by the time affinity is classified. + # Keep both locations in this check or a harmless parser refactor can + # silently break restart recovery while marker-preservation tests still pass. + instructions = payload.instructions + if isinstance(instructions, str) and any( + line.lstrip().startswith(_GOAL_CONTINUATION_CONTEXT_PREFIX) for line in instructions.splitlines() + ): + return True + return responses_input_contains_goal_continuation_context(payload.input) + + def _compact_trimmed_input_with_markers( input_value: list[JsonValue], token_counts: list[int], selected_indices: set[int] ) -> list[JsonValue]: @@ -1740,6 +1819,7 @@ def _sanitize_interleaved_reasoning_input(payload: MutableJsonObject) -> None: def normalize_reasoning_aliases(payload: MutableJsonObject) -> None: reasoning_effort = payload.pop("reasoningEffort", None) + snake_case_reasoning_effort = payload.pop("reasoning_effort", None) reasoning_summary = payload.pop("reasoningSummary", None) provider_thinking = payload.pop("thinking", None) provider_enable_thinking = payload.pop("enable_thinking", None) @@ -1750,8 +1830,20 @@ def normalize_reasoning_aliases(payload: MutableJsonObject) -> None: else: reasoning_map = {} - if isinstance(reasoning_effort, str) and "effort" not in reasoning_map: - reasoning_map["effort"] = reasoning_effort + existing_effort = reasoning_map.get("effort") + if isinstance(existing_effort, str) and not existing_effort.strip(): + reasoning_map.pop("effort") + + alias_effort = next( + ( + candidate.strip() + for candidate in (reasoning_effort, snake_case_reasoning_effort) + if isinstance(candidate, str) and candidate.strip() + ), + None, + ) + if alias_effort is not None and "effort" not in reasoning_map: + reasoning_map["effort"] = alias_effort if isinstance(reasoning_summary, str) and "summary" not in reasoning_map: reasoning_map["summary"] = reasoning_summary @@ -1775,15 +1867,14 @@ def _normalize_thinking_alias( enable_thinking: JsonValue, ) -> MutableJsonObject | None: if isinstance(thinking, bool): - return {"effort": "medium"} if thinking else None + if thinking: + return {"effort": "medium"} if isinstance(thinking, str): normalized = thinking.strip().lower() - if normalized in {"low", "medium", "high", "xhigh", "max", "ultra"}: + if normalized in {"minimal", "low", "medium", "high", "xhigh", "max", "ultra"}: return {"effort": normalized} if normalized in {"enabled", "true", "on"}: return {"effort": "medium"} - if normalized in {"disabled", "false", "off"}: - return None thinking_mapping = _json_mapping_or_none(thinking) if thinking_mapping is not None: normalized: MutableJsonObject = {} @@ -1793,19 +1884,19 @@ def _normalize_thinking_alias( normalized["effort"] = effort.strip().lower() if isinstance(summary, str) and summary.strip(): normalized["summary"] = summary.strip() - if normalized: - return normalized thinking_type = thinking_mapping.get("type") - if isinstance(thinking_type, str): - normalized_type = thinking_type.strip().lower() - if normalized_type == "enabled": - return {"effort": "medium"} - if normalized_type == "disabled": - return None + if "effort" not in normalized and isinstance(thinking_type, str) and thinking_type.strip().lower() == "enabled": + normalized["effort"] = "medium" enabled = thinking_mapping.get("enabled") - if isinstance(enabled, bool): - return {"effort": "medium"} if enabled else None + if "effort" not in normalized and enabled is True: + normalized["effort"] = "medium" + if "effort" not in normalized and enable_thinking is True: + normalized["effort"] = "medium" + if normalized: + return normalized + # Disabled `thinking` spellings are inactive, not authoritative: a + # separate enabled alias must still participate in policy evaluation. if isinstance(enable_thinking, bool): return {"effort": "medium"} if enable_thinking else None return None diff --git a/app/core/resilience/loop_lag_monitor.py b/app/core/resilience/loop_lag_monitor.py new file mode 100644 index 0000000000..dbe8a69b72 --- /dev/null +++ b/app/core/resilience/loop_lag_monitor.py @@ -0,0 +1,60 @@ +"""Event-loop lag watchdog. + +Samples scheduling delay by measuring ``asyncio.sleep`` drift. When the loop +is starved (a callback storm, synchronous work on the loop, CPU saturation), +every request and health check degrades at once while per-request logs stay +quiet: nothing says "the loop itself is busy". The 2026-08-20 incident — an +``asyncio.shield`` callback storm pinning one core for hours — surfaced only +as mysterious global slowness and health-check flapping. This monitor turns +that state into an explicit, rate-limited warning log plus Prometheus +signals (``codex_lb_event_loop_lag_seconds`` gauge and +``codex_lb_event_loop_lag_warnings_total`` counter) so operators and alerts +can distinguish "loop starved" from "upstream slow". +""" + +from __future__ import annotations + +import asyncio +import logging +import time + +from app.core.metrics import prometheus as prometheus_metrics + +logger = logging.getLogger(__name__) + +_SAMPLE_INTERVAL_SECONDS = 1.0 +# One warning line per window at most; the gauge/counter stay per-sample. The +# worst lag seen inside a suppressed window is carried into the next line so +# suppression never hides the magnitude of a spike. +_WARN_LOG_INTERVAL_SECONDS = 60.0 + + +async def run_event_loop_lag_monitor(*, warn_threshold_seconds: float) -> None: + """Sample loop lag forever; the caller owns and cancels the task.""" + last_warn_monotonic = float("-inf") + worst_suppressed_lag = 0.0 + while True: + started = time.monotonic() + await asyncio.sleep(_SAMPLE_INTERVAL_SECONDS) + lag = max(0.0, time.monotonic() - started - _SAMPLE_INTERVAL_SECONDS) + gauge = prometheus_metrics.event_loop_lag_seconds + if gauge is not None: + gauge.set(lag) + if lag < warn_threshold_seconds: + continue + counter = prometheus_metrics.event_loop_lag_warnings_total + if counter is not None: + counter.inc() + now = time.monotonic() + if now - last_warn_monotonic < _WARN_LOG_INTERVAL_SECONDS: + worst_suppressed_lag = max(worst_suppressed_lag, lag) + continue + logger.warning( + "event_loop_lag lag_seconds=%.3f worst_suppressed_seconds=%.3f threshold_seconds=%.3f " + "(event loop starved: callback storm, sync work on the loop, or CPU saturation)", + lag, + worst_suppressed_lag, + warn_threshold_seconds, + ) + last_warn_monotonic = now + worst_suppressed_lag = 0.0 diff --git a/app/core/runtime_logging.py b/app/core/runtime_logging.py index f5911e1b65..732cc9707a 100644 --- a/app/core/runtime_logging.py +++ b/app/core/runtime_logging.py @@ -206,14 +206,15 @@ def log_error_response( level = logging.ERROR if status_code >= 500 else logging.WARNING logger.log( level, - "%s request_id=%s method=%s path=%s status=%s code=%s message=%s", - category, - get_request_id(), - request.method, - request.url.path, + "%s request_id=%s method=%s path=%s status=%s code=%s message_present=%s message_length=%s", + safe_log_field(category), + safe_log_field(get_request_id()), + safe_log_field(request.method), + safe_log_field(request.url.path), status_code, _error_log_field(code), - _error_log_field(message), + bool(message), + len(message) if message else 0, exc_info=exc_info, ) @@ -225,6 +226,15 @@ def _error_log_field(value: str | None) -> str: return json.dumps(redacted) +def safe_log_field(value: object | None) -> str: + """Return a redacted, single-line representation for a log field.""" + if value is None: + return "-" + single_line = str(value).replace("\r", " ").replace("\n", " ") + redacted = _redact_log_value(single_line) + return redacted or "-" + + def _collapse_log_value(value: str | None) -> str | None: if value is None: return None diff --git a/app/core/timeout_invariants.py b/app/core/timeout_invariants.py new file mode 100644 index 0000000000..e23d14b9a1 --- /dev/null +++ b/app/core/timeout_invariants.py @@ -0,0 +1,306 @@ +"""Startup timeout-invariant validation over raw ``Settings`` values. + +This module intentionally validates only startup ``Settings`` fields and a +small set of code constants whose relations are fixed at import/runtime. It +does not validate per-request ContextVar overrides +(``app/core/clients/proxy.py:3450-3467``, +``app/modules/proxy/_service/streaming/helpers.py:861-868``, +``app/modules/proxy/_service/compact.py:727-738``, +``app/modules/proxy/_service/transcribe.py:230-232``, +``app/core/clients/files.py:77-90``, and +``app/modules/proxy/service.py:1464-1478``), runtime clamps/derived effective +values (``app/core/clients/proxy.py:1049-1088``, +``app/core/auth/refresh.py:391-395``, and +``app/modules/proxy/load_balancer.py:1846-1856``), or DB/API-key/model-source +runtime settings (``app/core/config/settings_cache.py:22-36``, +``app/modules/settings/api.py:547-710``, +``app/modules/proxy/_service/streaming/retry.py:153-165``, and +``app/modules/model_sources/forwarding.py:112-221``). +""" + +from __future__ import annotations + +import argparse +import logging +import operator +import sys +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Protocol + +logger = logging.getLogger(__name__) + + +class TimeoutSettings(Protocol): + upstream_connect_timeout_seconds: float + proxy_request_budget_seconds: float + http_responses_stream_request_budget_seconds: float + compact_request_budget_seconds: float + sse_keepalive_interval_seconds: float + http_responses_session_bridge_request_budget_seconds: float + http_responses_session_bridge_stuck_gate_retire_after_seconds: float + http_responses_session_bridge_clean_close_retry_jitter_max_seconds: float + proxy_admission_wait_timeout_seconds: float + proxy_account_lease_ttl_seconds: float + model_registry_enabled: bool + + @property + def model_registry_snapshot_max_age_seconds(self) -> int | float: ... + + timeout_invariant_validation_strict: bool + + +@dataclass(frozen=True, slots=True) +class TimeoutOperand: + label: str + evaluate: Callable[[TimeoutSettings], float] + code_anchor: str + + +@dataclass(frozen=True, slots=True) +class TimeoutInvariantRule: + id: str + lhs: TimeoutOperand + relation: str + rhs: TimeoutOperand + rationale: str + + +@dataclass(frozen=True, slots=True) +class TimeoutInvariantViolation: + rule: TimeoutInvariantRule + lhs_value: float + rhs_value: float + + def format(self) -> str: + return ( + f"{self.rule.id}: {self.rule.lhs.label}={self.lhs_value:g} " + f"{self.rule.relation} {self.rule.rhs.label}={self.rhs_value:g} violated; " + f"{self.rule.rationale} " + f"(lhs: {self.rule.lhs.code_anchor}; rhs: {self.rule.rhs.code_anchor})" + ) + + +class TimeoutInvariantError(RuntimeError): + def __init__(self, violations: Sequence[TimeoutInvariantViolation]) -> None: + self.violations = tuple(violations) + super().__init__("\n".join(violation.format() for violation in self.violations)) + + +def _field(name: str, anchor: str) -> TimeoutOperand: + return TimeoutOperand(name, lambda settings: float(getattr(settings, name)), anchor) + + +def _expr(label: str, anchor: str, evaluate: Callable[[TimeoutSettings], float]) -> TimeoutOperand: + return TimeoutOperand(label, evaluate, anchor) + + +UPSTREAM_CONNECT = _field("upstream_connect_timeout_seconds", "app/core/clients/proxy.py:2720") +PROXY_BUDGET = _field("proxy_request_budget_seconds", "app/core/config/settings.py:260") +STREAM_BUDGET = _field( + "http_responses_stream_request_budget_seconds", + "app/modules/proxy/_service/streaming/helpers.py:724", +) +COMPACT_BUDGET = _field("compact_request_budget_seconds", "app/modules/proxy/_service/compact.py:585") +SSE_KEEPALIVE = _field("sse_keepalive_interval_seconds", "app/modules/proxy/api.py:3930") +TOKEN_REFRESH = _field("token_refresh_timeout_seconds", "app/modules/accounts/auth_manager.py:1123") +BRIDGE_BUDGET = _field( + "http_responses_session_bridge_request_budget_seconds", + "app/modules/proxy/_service/http_bridge/helpers.py:2469", +) +BRIDGE_CLEAN_CLOSE_JITTER = _field( + "http_responses_session_bridge_clean_close_retry_jitter_max_seconds", + "app/modules/proxy/_service/http_bridge/request_submit.py:294", +) +ADMISSION_WAIT = _field("proxy_admission_wait_timeout_seconds", "app/modules/proxy/service.py:768") +ACCOUNT_LEASE_TTL = _field("proxy_account_lease_ttl_seconds", "app/modules/proxy/load_balancer.py:1993") +BRIDGE_STUCK_GATE_HARD_ANCHOR_RETIRE = _expr( + "2 * http_responses_session_bridge_stuck_gate_retire_after_seconds", + "app/modules/proxy/_service/http_bridge/helpers.py:686", + lambda settings: 2.0 * settings.http_responses_session_bridge_stuck_gate_retire_after_seconds, +) +MODEL_REGISTRY_SNAPSHOT_MAX_AGE = _field( + "model_registry_snapshot_max_age_seconds", + "app/core/openai/model_registry_store.py:367", +) +MODEL_REGISTRY_REFRESH_INTERVAL = _expr( + "_REFRESH_INTERVAL_SECONDS", + "app/core/openai/model_refresh_scheduler.py:37", + lambda settings: _model_registry_refresh_interval_seconds(), +) +DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL = _expr( + "DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS", + "app/modules/proxy/durable_bridge_repository.py:42", + lambda settings: _durable_bridge_retry_circuit_state_ttl_seconds(), +) +DURABLE_BRIDGE_RETRY_CIRCUIT_MIN_TTL = _expr( + "_HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS + _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS", + "app/modules/proxy/_service/http_bridge/retry_circuit.py:19-21", + lambda settings: _durable_bridge_retry_circuit_min_ttl_seconds(), +) + + +def _model_registry_refresh_interval_seconds() -> float: + from app.core.openai.model_refresh_scheduler import _REFRESH_INTERVAL_SECONDS + + return float(_REFRESH_INTERVAL_SECONDS) + + +def _durable_bridge_retry_circuit_state_ttl_seconds() -> float: + from app.modules.proxy.durable_bridge_repository import DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS + + return float(DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS) + + +def _durable_bridge_retry_circuit_min_ttl_seconds() -> float: + from app.modules.proxy._service.http_bridge.retry_circuit import ( + _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS, + _HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS, + ) + + return float(_HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS + _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS) + + +TIMEOUT_INVARIANT_RULES: tuple[TimeoutInvariantRule, ...] = ( + TimeoutInvariantRule( + "admission-wait-within-proxy-budget", + ADMISSION_WAIT, + "<=", + PROXY_BUDGET, + "Global admission waits must not consume more than the request budget they protect.", + ), + TimeoutInvariantRule( + "admission-wait-within-stream-budget", + ADMISSION_WAIT, + "<=", + STREAM_BUDGET, + "Streaming retries wait for capacity inside the stream budget and must leave room for the stream attempt.", + ), + TimeoutInvariantRule( + "admission-wait-within-compact-budget", + ADMISSION_WAIT, + "<=", + COMPACT_BUDGET, + "Compact response-create admission must not outlive the compact request budget.", + ), + TimeoutInvariantRule( + "bridge-stuck-gate-retire-within-bridge-budget", + BRIDGE_STUCK_GATE_HARD_ANCHOR_RETIRE, + "<", + BRIDGE_BUDGET, + "Hard-continuity stuck gate retirement waits up to 2x the configured threshold and must happen before " + "the bridge request budget is exhausted.", + ), + TimeoutInvariantRule( + "account-lease-ttl-covers-proxy-budget", + ACCOUNT_LEASE_TTL, + ">=", + PROXY_BUDGET, + "Response-create leases use the raw lease TTL, so stale reclaim must not precede a healthy non-stream " + "request deadline.", + ), + TimeoutInvariantRule( + "account-lease-ttl-covers-compact-budget", + ACCOUNT_LEASE_TTL, + ">=", + COMPACT_BUDGET, + "Compact response-create leases must not be stale-reclaimed before the compact request budget expires.", + ), + TimeoutInvariantRule( + "model-registry-snapshot-outlives-refresh-interval", + MODEL_REGISTRY_SNAPSHOT_MAX_AGE, + ">", + MODEL_REGISTRY_REFRESH_INTERVAL, + "Persisted model-registry snapshots must remain loadable for at least one fixed refresh cadence.", + ), + TimeoutInvariantRule( + "durable-bridge-retry-circuit-ttl-covers-backoff-and-half-open", + DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL, + ">", + DURABLE_BRIDGE_RETRY_CIRCUIT_MIN_TTL, + "Durable HTTP bridge retry-circuit state must outlive the longest cooldown and half-open lease.", + ), +) + +# TODO(timeout_sem_001): database_migration_lock_timeout_seconds is independent startup DB migration policy. +# TODO(timeout_sem_008): proxy_downstream_websocket_idle_timeout_seconds has no verified ordering with bridge TTL. +# TODO(timeout_sem_009): oauth_timeout_seconds is used in OAuth/client flows, not a verified proxy-path deadline. +# TODO(timeout_sem_015): openai_cache_affinity_max_age_seconds participates with dashboard prompt-cache TTL in +# cleanup retention. +# TODO(timeout_sem_021): upstream_route_cache_ttl_seconds is invalidation freshness policy; no timeout inequality +# verified. +# timeout_sem_022 is enforced by model-registry-snapshot-outlives-refresh-interval. +# TODO(timeout_sem_023): firewall_ip_cache_ttl_seconds has no verified timeout owner beyond trust-cache freshness. +# TODO(timeout_sem_024): leader_election_ttl_seconds renewal is derived internally as ttl//3, not a cross-setting +# inequality. +# TODO(timeout_sem_027): proxy_account_cap_partition_scale_down_seconds is a stability window; exact heartbeat relation +# is internal. +# TODO(timeout_sem_029): usage_refresh_auth_failure_cooldown_seconds is policy cooldown, not a verified scheduler +# inequality. +# TODO(timeout_sem_030): shutdown_drain_timeout_seconds depends on deployment termination grace outside Settings. +# timeout_sem_031 is enforced by durable-bridge-retry-circuit-ttl-covers-backoff-and-half-open. +# TODO(timeout_sem_032/033): SQLite busy retry constants are module-local and not Settings-field rules. +# TODO(timeout_sem_034/035): account-selection recovery caps are module constants clamped by request deadlines at +# runtime. + +_RELATIONS: dict[str, Callable[[float, float], bool]] = { + "<": operator.lt, + "<=": operator.le, + ">": operator.gt, + ">=": operator.ge, +} + + +def find_timeout_invariant_violations(settings: TimeoutSettings) -> list[TimeoutInvariantViolation]: + violations: list[TimeoutInvariantViolation] = [] + for rule in TIMEOUT_INVARIANT_RULES: + if rule.id == "model-registry-snapshot-outlives-refresh-interval" and not settings.model_registry_enabled: + continue + lhs_value = rule.lhs.evaluate(settings) + rhs_value = rule.rhs.evaluate(settings) + if not _RELATIONS[rule.relation](lhs_value, rhs_value): + violations.append(TimeoutInvariantViolation(rule, lhs_value, rhs_value)) + return violations + + +def validate_timeout_invariants( + settings: TimeoutSettings, + *, + strict: bool = False, + log: bool = True, +) -> list[TimeoutInvariantViolation]: + violations = find_timeout_invariant_violations(settings) + if violations and log: + for violation in violations: + logger.critical("timeout invariant violation: %s", violation.format()) + if strict and violations: + raise TimeoutInvariantError(violations) + return violations + + +def validate_runtime_timeout_invariants(settings: TimeoutSettings) -> list[TimeoutInvariantViolation]: + return validate_timeout_invariants( + settings, + strict=settings.timeout_invariant_validation_strict, + log=True, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + from app.core.config.settings import get_settings + + parser = argparse.ArgumentParser(description="Validate codex-lb timeout invariants.") + parser.add_argument("--strict", action="store_true", help="exit nonzero when any invariant is violated") + args = parser.parse_args(argv) + + violations = validate_timeout_invariants(get_settings(), strict=False, log=True) + if not violations: + print(f"OK: {len(TIMEOUT_INVARIANT_RULES)} timeout invariant rules satisfied") + return 0 + for violation in violations: + print(violation.format(), file=sys.stderr) + return 1 if args.strict else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/core/usage/refresh_scheduler.py b/app/core/usage/refresh_scheduler.py index 19410599d4..a6133d836f 100644 --- a/app/core/usage/refresh_scheduler.py +++ b/app/core/usage/refresh_scheduler.py @@ -7,17 +7,24 @@ import time from collections.abc import Awaitable, Callable, Collection from dataclasses import dataclass, field -from datetime import datetime +from datetime import datetime, timezone from typing import Any, AsyncIterator, Protocol, TypeVar, cast +from app.core.balancer.logic import RATE_LIMITED_MIN_COOLDOWN_SECONDS from app.core.config.settings import get_settings +from app.core.plan_types import normalize_account_plan_type from app.core.usage import capacity_for_plan +from app.core.utils.time import naive_utc_to_epoch from app.db.models import Account, AccountLimitWarmup, AccountStatus, UsageHistory from app.db.session import detach_session_objects, get_background_session from app.modules.accounts.background_repository import BackgroundAccountsRepository from app.modules.accounts.repository import AccountsRepository from app.modules.limit_warmup.repository import LimitWarmupRepository -from app.modules.limit_warmup.service import LimitWarmupService, StreamingLimitWarmupSender +from app.modules.limit_warmup.service import ( + LimitWarmupService, + StreamingLimitWarmupSender, + usage_reset_confirmed, +) from app.modules.proxy.account_cache import get_account_selection_cache from app.modules.proxy.load_balancer import background_recovery_state_from_account from app.modules.proxy.rate_limit_cache import get_rate_limit_headers_cache @@ -30,11 +37,19 @@ logger = logging.getLogger(__name__) _RECOVERABLE_ACCOUNT_STATUSES = frozenset({AccountStatus.RATE_LIMITED, AccountStatus.QUOTA_EXCEEDED}) +_BLOCK_RESET_MATCH_TOLERANCE_SECONDS = 5 _T = TypeVar("_T") +@dataclass(frozen=True, slots=True) +class _MonthlyResetEvidence: + baseline: UsageHistory + before: UsageHistory + after: UsageHistory + + class _LeaderElectionLike(Protocol): async def run_if_leader(self, fn: Callable[[], Awaitable[_T]]) -> _T | None: ... @@ -182,6 +197,9 @@ async def _refresh_as_leader(self) -> float: selected_account, cycle_complete = self._select_next_account(accounts) if selected_account is not None: selected_account_ids = [selected_account.id] + previous_plan_types = { + selected_account.id: normalize_account_plan_type(selected_account.plan_type) + } before_primary = await usage_repo.latest_by_account( window="primary", account_ids=selected_account_ids, @@ -225,7 +243,25 @@ async def _refresh_as_leader(self) -> float: refreshed_selected_accounts = [ account for account in refreshed_accounts if account.id == selected_account.id ] + monthly_reset_evidence = await _resolve_monthly_reset_evidence( + accounts=refreshed_selected_accounts, + usage_repo=usage_repo, + before_monthly=before_monthly, + after_monthly=after_monthly, + ) detach_session_objects(session) + warmup_before_monthly = dict(before_monthly) + warmup_after_monthly = dict(after_monthly) + for account_id, reset_evidence in monthly_reset_evidence.items(): + warmup_before_monthly[account_id] = reset_evidence.before + warmup_after_monthly[account_id] = reset_evidence.after + async with get_background_session() as session: + await reconcile_recoverable_account_statuses( + accounts_repo=AccountsRepository(session), + usage_repo=UsageRepository(session), + accounts=refreshed_selected_accounts, + monthly_reset_evidence=monthly_reset_evidence, + ) warmup_service = LimitWarmupService( cast(Any, _BackgroundLimitWarmupRepository()), cast(Any, _BackgroundRequestLogsRepository()), @@ -241,26 +277,19 @@ async def _refresh_as_leader(self) -> float: before_primary=before_primary, before_secondary=_select_long_window_entries( accounts=refreshed_selected_accounts, - monthly_entries=before_monthly, + monthly_entries=warmup_before_monthly, secondary_entries=before_secondary, ), after_primary=after_primary, after_secondary=_select_long_window_entries( accounts=refreshed_selected_accounts, - monthly_entries=after_monthly, + monthly_entries=warmup_after_monthly, secondary_entries=after_secondary, ), + previous_plan_types=previous_plan_types, refresh_started_at=refresh_started_at, usage_refresh_interval_seconds=self.interval_seconds, ) - async with get_background_session() as session: - usage_repo = UsageRepository(session) - accounts_repo = AccountsRepository(session) - await reconcile_recoverable_account_statuses( - accounts_repo=accounts_repo, - usage_repo=usage_repo, - accounts=refreshed_selected_accounts, - ) if cycle_complete: await _invalidate_usage_refresh_caches() except Exception: @@ -311,7 +340,10 @@ async def _invalidate_usage_refresh_caches() -> None: @contextlib.asynccontextmanager async def _background_accounts_repo() -> AsyncIterator[AccountsRepository]: async with get_background_session() as session: - yield AccountsRepository(session) + try: + yield AccountsRepository(session) + finally: + detach_session_objects(session) async def reconcile_recoverable_account_statuses( @@ -319,6 +351,7 @@ async def reconcile_recoverable_account_statuses( accounts_repo: _RecoverableAccountsRepository, usage_repo: _LatestUsageRepository, accounts: list[Account], + monthly_reset_evidence: dict[str, _MonthlyResetEvidence] | None = None, ) -> int: candidates = [account for account in accounts if account.status in _RECOVERABLE_ACCOUNT_STATUSES] if not candidates: @@ -331,22 +364,33 @@ async def reconcile_recoverable_account_statuses( recovered = 0 for account in candidates: - state = background_recovery_state_from_account( + monthly_entry = latest_monthly.get(account.id) + if _confirmed_free_monthly_reset_recovery( account=account, - primary_entry=latest_primary.get(account.id), - secondary_entry=_select_long_window_entry( + reset_evidence=(monthly_reset_evidence or {}).get(account.id), + latest=monthly_entry, + ): + status = AccountStatus.ACTIVE + reset_at = None + blocked_at = None + else: + state = background_recovery_state_from_account( account=account, - monthly_entry=latest_monthly.get(account.id), - secondary_entry=latest_secondary.get(account.id), - ), - ) - if state.status != AccountStatus.ACTIVE: - continue - reset_at = int(state.reset_at) if state.reset_at else None - blocked_at = int(state.blocked_at) if state.blocked_at else None + primary_entry=latest_primary.get(account.id), + secondary_entry=_select_long_window_entry( + account=account, + monthly_entry=monthly_entry, + secondary_entry=latest_secondary.get(account.id), + ), + ) + if state.status != AccountStatus.ACTIVE: + continue + status = state.status + reset_at = int(state.reset_at) if state.reset_at else None + blocked_at = int(state.blocked_at) if state.blocked_at else None deactivation_reason = None if ( - state.status == account.status + status == account.status and deactivation_reason == account.deactivation_reason and reset_at == account.reset_at and blocked_at == account.blocked_at @@ -354,7 +398,7 @@ async def reconcile_recoverable_account_statuses( continue updated = await accounts_repo.update_status_if_current( account.id, - state.status, + status, deactivation_reason, reset_at, blocked_at=blocked_at, @@ -365,7 +409,7 @@ async def reconcile_recoverable_account_statuses( ) if not updated: continue - account.status = state.status + account.status = status account.deactivation_reason = deactivation_reason account.reset_at = reset_at account.blocked_at = blocked_at @@ -373,6 +417,119 @@ async def reconcile_recoverable_account_statuses( return recovered +def _confirmed_free_monthly_reset_recovery( + *, + account: Account, + reset_evidence: _MonthlyResetEvidence | None, + latest: UsageHistory | None, +) -> bool: + if account.status != AccountStatus.RATE_LIMITED: + return False + if normalize_account_plan_type(account.plan_type) != "free": + return False + if account.reset_at is None or account.blocked_at is None: + return False + now = time.time() + if now >= account.reset_at: + return False + if now < account.blocked_at + RATE_LIMITED_MIN_COOLDOWN_SECONDS: + return False + if reset_evidence is None or latest is None: + return False + baseline = reset_evidence.baseline + before = reset_evidence.before + after = reset_evidence.after + if ( + baseline.window != "monthly" + or before.window != "monthly" + or after.window != "monthly" + or latest.window != "monthly" + ): + return False + if baseline.reset_at is None: + return False + if abs(baseline.reset_at - account.reset_at) > _BLOCK_RESET_MATCH_TOLERANCE_SECONDS: + return False + if naive_utc_to_epoch(baseline.recorded_at) <= account.blocked_at: + return False + if not usage_reset_confirmed(before=before, after=after): + return False + if after.used_percent >= 100.0 or latest.used_percent >= 100.0: + return False + return ( + naive_utc_to_epoch(after.recorded_at) > account.blocked_at + and naive_utc_to_epoch(latest.recorded_at) > account.blocked_at + ) + + +async def _resolve_monthly_reset_evidence( + *, + accounts: list[Account], + usage_repo: UsageRepository, + before_monthly: dict[str, UsageHistory], + after_monthly: dict[str, UsageHistory], +) -> dict[str, _MonthlyResetEvidence]: + evidence: dict[str, _MonthlyResetEvidence] = {} + for account in accounts: + before = before_monthly.get(account.id) + after = after_monthly.get(account.id) + if usage_reset_confirmed(before=before, after=after): + assert before is not None and after is not None + evidence[account.id] = _MonthlyResetEvidence( + baseline=before, + before=before, + after=after, + ) + if ( + account.status != AccountStatus.RATE_LIMITED + or normalize_account_plan_type(account.plan_type) != "free" + or account.reset_at is None + or account.blocked_at is None + ): + continue + since = datetime.fromtimestamp(account.blocked_at, timezone.utc).replace(tzinfo=None) + history = await usage_repo.history_since(account.id, "monthly", since) + persisted = _latest_confirmed_reset_transition_after_baseline( + [entry for entry in history if entry.recorded_at > since], + expected_reset_at=account.reset_at, + reset_at_tolerance_seconds=_BLOCK_RESET_MATCH_TOLERANCE_SECONDS, + ) + if persisted is not None: + evidence[account.id] = persisted + return evidence + + +def _latest_confirmed_reset_transition_after_baseline( + history: list[UsageHistory], + *, + expected_reset_at: int, + reset_at_tolerance_seconds: int, +) -> _MonthlyResetEvidence | None: + baseline = next( + ( + (index, entry) + for index, entry in enumerate(history) + if entry.reset_at is not None and abs(entry.reset_at - expected_reset_at) <= reset_at_tolerance_seconds + ), + None, + ) + if baseline is None: + return None + baseline_index, baseline_entry = baseline + + latest_transition: _MonthlyResetEvidence | None = None + for index in range(baseline_index, len(history) - 1): + before = history[index] + after = history[index + 1] + if usage_reset_confirmed(before=before, after=after): + latest_transition = _MonthlyResetEvidence( + baseline=baseline_entry, + before=before, + after=after, + ) + return latest_transition + + def _select_long_window_entry( *, account: Account, diff --git a/app/core/usage/reset_credits_refresh_scheduler.py b/app/core/usage/reset_credits_refresh_scheduler.py index a107e504f2..564923d7be 100644 --- a/app/core/usage/reset_credits_refresh_scheduler.py +++ b/app/core/usage/reset_credits_refresh_scheduler.py @@ -64,16 +64,37 @@ class RateLimitResetCreditsRefreshScheduler: interval_seconds: int rng: random.Random = field(default_factory=random.Random) + enabled: bool = True _task: asyncio.Task[None] | None = None _stop: asyncio.Event = field(default_factory=asyncio.Event) _lock: asyncio.Lock = field(default_factory=asyncio.Lock) async def start(self) -> None: + if not self.enabled: + await self._warn_if_auto_redeem_conflicts() + return if self._task and not self._task.done(): return self._stop.clear() self._task = asyncio.create_task(self._run_loop()) + async def _warn_if_auto_redeem_conflicts(self) -> None: + # The refresh loop is the only driver of automatic redemption, so a + # disabled scheduler silently starves a persisted auto-redeem opt-in. + try: + async with get_background_session() as session: + dashboard_settings = await SettingsRepository(session).get_or_create() + auto_redeem_enabled = dashboard_settings.auto_redeem_reset_credits_before_expiry + except Exception: + logger.exception("Reset credits auto-redeem conflict check failed") + return + if auto_redeem_enabled: + logger.warning( + "rate_limit_reset_credits_refresh_enabled=false disables automatic reset-credit " + "redemption, but dashboard setting auto_redeem_reset_credits_before_expiry is " + "enabled; credits will expire without redemption until polling is re-enabled" + ) + async def stop(self) -> None: if not self._task: return @@ -386,4 +407,5 @@ def build_rate_limit_reset_credits_scheduler() -> RateLimitResetCreditsRefreshSc settings = get_settings() return RateLimitResetCreditsRefreshScheduler( interval_seconds=settings.rate_limit_reset_credits_refresh_interval_seconds, + enabled=settings.rate_limit_reset_credits_refresh_enabled, ) diff --git a/app/core/utils/shared_future.py b/app/core/utils/shared_future.py new file mode 100644 index 0000000000..5e959442c5 --- /dev/null +++ b/app/core/utils/shared_future.py @@ -0,0 +1,80 @@ +"""Await shared futures without per-waiter callbacks on the shared object. + +``asyncio.wait_for(asyncio.shield(shared), timeout)`` attaches done callbacks +to ``shared`` for every waiter and removes them with O(n) list scans when a +waiter is cancelled or times out. With many waiters piled onto one long-lived +future (the http-bridge inflight/capacity registries, refresh singleflight), +a mass timeout turns the event loop into an O(N^2) callback grinder. Python +3.14's ``shield`` additionally leaks one ``_clear_awaited_by_callback`` per +attempt onto the still-pending future, so each retry cycle makes every later +scan more expensive. In the 2026-08-20 production incident this starved the +event loop for hours (98% of GIL samples inside ``Future.remove_done_callback``) +with zero client sessions attached. + +``wait_on_shared_future`` keeps exactly one fan-out callback on the shared +future regardless of waiter count. Each waiter awaits its own single-use proxy +future, so waiter timeout and cancellation are O(1) set operations that never +touch the shared future's callback list. +""" + +from __future__ import annotations + +import asyncio +from typing import TypeVar + +_T = TypeVar("_T") + +_WAITERS_ATTR = "_shared_future_fanout_waiters" + + +def _fan_out(shared: "asyncio.Future[_T]", waiters: "set[asyncio.Future[_T]]") -> None: + for waiter in waiters: + if waiter.done(): + continue + if shared.cancelled(): + waiter.cancel() + continue + exc = shared.exception() + if exc is not None: + waiter.set_exception(exc) + # Consume eagerly: a waiter whose task was cancelled between this + # fan-out and its resumption would otherwise log + # "exception was never retrieved" from the proxy destructor. + waiter.exception() + else: + waiter.set_result(shared.result()) + waiters.clear() + + +async def wait_on_shared_future( + shared: "asyncio.Future[_T]", + *, + timeout: float | None = None, +) -> _T: + """Drop-in equivalent of ``wait_for(shield(shared), timeout)`` for futures + awaited by many concurrent waiters. + + - ``shared``'s result, exception, or cancellation propagates to every + waiter exactly as with ``shield``. + - ``timeout`` raises ``TimeoutError``; ``shared`` is never cancelled or + otherwise mutated by a waiter timing out or being cancelled. + - Cancelling the awaiting task detaches its proxy in O(1) and leaves + ``shared`` (and the work it represents) running. + """ + if shared.done(): + return shared.result() + waiters: set[asyncio.Future[_T]] | None = getattr(shared, _WAITERS_ATTR, None) + if waiters is None: + # No await between the ``done()`` check above and this registration, + # so the fan-out callback cannot have fired with an empty set. + waiters = set() + setattr(shared, _WAITERS_ATTR, waiters) + shared.add_done_callback(lambda done, _waiters=waiters: _fan_out(done, _waiters)) + proxy: asyncio.Future[_T] = asyncio.get_running_loop().create_future() + waiters.add(proxy) + try: + if timeout is None: + return await proxy + return await asyncio.wait_for(proxy, timeout) + finally: + waiters.discard(proxy) diff --git a/app/core/utils/sse.py b/app/core/utils/sse.py index b25d185d7f..f757856c83 100644 --- a/app/core/utils/sse.py +++ b/app/core/utils/sse.py @@ -20,6 +20,29 @@ SSE_KEEPALIVE_FRAME = ": keepalive\n\n" CODEX_KEEPALIVE_FRAME = 'event: codex.keepalive\ndata: {"type":"codex.keepalive"}\n\n' +# The exact single-event shape ``format_sse_event`` emits (and the upstream +# Codex backend sends): a leading ``event: `` line, one JSON-object +# ``data:`` line, LF-only framing, and a blank-line terminator. Blocks that +# match can expose their event type without a JSON parse and are safe to +# relay downstream byte-for-byte. +_CANONICAL_SSE_BLOCK = re.compile(r"\Aevent: ([^\r\n]+)\ndata: \{[^\r\n]*\n\n\Z") + + +def sse_event_type_from_block(event_block: str) -> str | None: + """Cheaply extract the event type from a canonically framed SSE block. + + Returns the ``event:`` line's value only when the block matches the exact + shape ``format_sse_event`` produces (see ``_CANONICAL_SSE_BLOCK``). + Anything else — data-only blocks, multi-line data, CR/CRLF framing, + comment or ``id:`` lines, non-object data payloads, or an ``event:`` field + that appears after ``data:`` (legal SSE, but not canonical here) — returns + ``None`` so callers fall back to a full parse. + """ + match = _CANONICAL_SSE_BLOCK.match(event_block) + if match is None: + return None + return match.group(1) + async def inject_sse_keepalives( source: AsyncIterator[str], diff --git a/app/db/account_identity_lock.py b/app/db/account_identity_lock.py new file mode 100644 index 0000000000..a8fd57aedc --- /dev/null +++ b/app/db/account_identity_lock.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from collections.abc import Collection +from hashlib import sha256 + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +_POSTGRES_ACCOUNT_IDENTITY_LOCK_TIMEOUT_MS = 30_000 + + +def advisory_lock_key(scope: str, value: str) -> int: + digest = sha256(f"{scope}:{value}".encode("utf-8")).digest() + return int.from_bytes(digest[:8], byteorder="big", signed=True) + + +def account_identity_lock_key(chatgpt_account_id: str) -> int: + """Return the existing PostgreSQL lock namespace for one upstream identity.""" + return advisory_lock_key("account-id", f"chatgpt:{chatgpt_account_id}") + + +async def lock_postgresql_account_identities( + session: AsyncSession, + chatgpt_account_ids: Collection[str | None], +) -> tuple[int, ...]: + """Lock upstream identity membership in canonical transaction-scoped order.""" + bind = session.get_bind() + if bind is None or bind.dialect.name != "postgresql": + return () + lock_keys = tuple( + sorted( + { + account_identity_lock_key(chatgpt_account_id) + for chatgpt_account_id in chatgpt_account_ids + if chatgpt_account_id + } + ) + ) + try: + if lock_keys: + # Match the database layer's existing 30-second contention budget. + # Transaction-local scope bounds every subsequent lock in this unit + # of work and PostgreSQL restores it at transaction end. + await session.execute( + text("SELECT set_config('lock_timeout', :timeout, true)"), + {"timeout": f"{_POSTGRES_ACCOUNT_IDENTITY_LOCK_TIMEOUT_MS}ms"}, + ) + for lock_key in lock_keys: + await session.execute( + text("SELECT pg_advisory_xact_lock(:lock_key)"), + {"lock_key": lock_key}, + ) + except BaseException: + await session.rollback() + raise + return lock_keys diff --git a/app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py b/app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py new file mode 100644 index 0000000000..2022dd0c08 --- /dev/null +++ b/app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py @@ -0,0 +1,63 @@ +"""add durable HTTP bridge operation identities and outcomes + +Revision ID: 20260804_000000_add_http_bridge_operations +Revises: 20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads +Create Date: 2026-08-04 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260804_000000_add_http_bridge_operations" +down_revision = "20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads" +branch_labels = None +depends_on = None + +_TABLE = "http_bridge_operations" + + +def _has_table(connection: Connection) -> bool: + return sa.inspect(connection).has_table(_TABLE) + + +def upgrade() -> None: + bind = op.get_bind() + if _has_table(bind): + return + op.create_table( + _TABLE, + sa.Column("operation_id", sa.String(80), primary_key=True), + sa.Column("session_id", sa.String(36), nullable=False), + sa.Column("request_fingerprint", sa.String(64), nullable=False), + sa.Column("account_id", sa.String(), nullable=True), + sa.Column("model", sa.String(), nullable=True), + sa.Column("parent_response_id", sa.Text(), nullable=True), + sa.Column("state", sa.String(32), nullable=False, server_default="submitted"), + sa.Column("response_id", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.ForeignKeyConstraint(["session_id"], ["http_bridge_sessions.id"], ondelete="CASCADE"), + sa.UniqueConstraint( + "session_id", + "request_fingerprint", + name="uq_http_bridge_operations_session_fingerprint", + ), + ) + op.create_index( + "idx_http_bridge_operations_session_parent_state", + _TABLE, + ["session_id", "parent_response_id", "state"], + ) + op.create_index("idx_http_bridge_operations_state_updated", _TABLE, ["state", "updated_at"]) + + +def downgrade() -> None: + bind = op.get_bind() + if not _has_table(bind): + return + op.drop_index("idx_http_bridge_operations_state_updated", table_name=_TABLE) + op.drop_index("idx_http_bridge_operations_session_parent_state", table_name=_TABLE) + op.drop_table(_TABLE) diff --git a/app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py b/app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py new file mode 100644 index 0000000000..016641815d --- /dev/null +++ b/app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py @@ -0,0 +1,53 @@ +"""fence HTTP bridge operations across durable sessions + +Revision ID: 20260804_000001_add_global_http_bridge_operation_fingerprint +Revises: 20260804_000000_add_http_bridge_operations +Create Date: 2026-08-04 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260804_000001_add_global_http_bridge_operation_fingerprint" +down_revision = "20260804_000000_add_http_bridge_operations" +branch_labels = None +depends_on = None + +_TABLE = "http_bridge_operations" +_FINGERPRINT_INDEX = "uq_http_bridge_operations_request_fingerprint" +_PARENT_INDEX = "idx_http_bridge_operations_parent_state" + + +def _has_table(connection: Connection) -> bool: + return sa.inspect(connection).has_table(_TABLE) + + +def _has_index(connection: Connection, name: str) -> bool: + return any(index.get("name") == name for index in sa.inspect(connection).get_indexes(_TABLE)) + + +def upgrade() -> None: + bind = op.get_bind() + if not _has_table(bind): + return + # A request fingerprint includes the parent response anchor, so it is a + # global operation identity even when a client reconnects to another + # durable bridge session. The unique index also closes the race where two + # workers observe a miss and try to dispatch the same continuation. + if not _has_index(bind, _FINGERPRINT_INDEX): + op.create_index(_FINGERPRINT_INDEX, _TABLE, ["request_fingerprint"], unique=True) + if not _has_index(bind, _PARENT_INDEX): + op.create_index(_PARENT_INDEX, _TABLE, ["parent_response_id", "state", "updated_at"]) + + +def downgrade() -> None: + bind = op.get_bind() + if not _has_table(bind): + return + if _has_index(bind, _PARENT_INDEX): + op.drop_index(_PARENT_INDEX, table_name=_TABLE) + if _has_index(bind, _FINGERPRINT_INDEX): + op.drop_index(_FINGERPRINT_INDEX, table_name=_TABLE) diff --git a/app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py b/app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py new file mode 100644 index 0000000000..a9f23893c2 --- /dev/null +++ b/app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py @@ -0,0 +1,75 @@ +"""add durable HTTP bridge operation request and event spool + +Revision ID: 20260805_000000_add_http_bridge_operation_spool +Revises: 20260804_000001_add_global_http_bridge_operation_fingerprint +Create Date: 2026-08-05 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260805_000000_add_http_bridge_operation_spool" +down_revision = "20260804_000001_add_global_http_bridge_operation_fingerprint" +branch_labels = None +depends_on = None + +_TABLE = "http_bridge_operations" +_EVENTS_TABLE = "http_bridge_operation_events" + + +def _has_table(connection: Connection, table: str) -> bool: + return sa.inspect(connection).has_table(table) + + +def _has_column(connection: Connection, table: str, column: str) -> bool: + return any(item["name"] == column for item in sa.inspect(connection).get_columns(table)) + + +def upgrade() -> None: + bind = op.get_bind() + if _has_table(bind, _TABLE): + if not _has_column(bind, _TABLE, "request_text"): + op.add_column(_TABLE, sa.Column("request_text", sa.Text(), nullable=True)) + if not _has_column(bind, _TABLE, "event_bytes"): + op.add_column(_TABLE, sa.Column("event_bytes", sa.Integer(), nullable=False, server_default="0")) + if not _has_column(bind, _TABLE, "event_spool_complete"): + op.add_column( + _TABLE, + sa.Column("event_spool_complete", sa.Boolean(), nullable=False, server_default=sa.text("true")), + ) + if _has_table(bind, _EVENTS_TABLE): + return + op.create_table( + _EVENTS_TABLE, + sa.Column("event_id", sa.String(36), primary_key=True), + sa.Column("operation_id", sa.String(80), nullable=False), + sa.Column("sequence_number", sa.Integer(), nullable=False), + sa.Column("event_fingerprint", sa.String(64), nullable=False), + sa.Column("event_text", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.ForeignKeyConstraint(["operation_id"], [f"{_TABLE}.operation_id"], ondelete="CASCADE"), + sa.UniqueConstraint( + "operation_id", + "event_fingerprint", + name="uq_http_bridge_operation_events_operation_fingerprint", + ), + ) + op.create_index( + "idx_http_bridge_operation_events_operation_sequence", + _EVENTS_TABLE, + ["operation_id", "sequence_number"], + ) + + +def downgrade() -> None: + bind = op.get_bind() + if _has_table(bind, _EVENTS_TABLE): + op.drop_index("idx_http_bridge_operation_events_operation_sequence", table_name=_EVENTS_TABLE) + op.drop_table(_EVENTS_TABLE) + if _has_table(bind, _TABLE): + for column in ("event_spool_complete", "event_bytes", "request_text"): + if _has_column(bind, _TABLE, column): + op.drop_column(_TABLE, column) diff --git a/app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py b/app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py new file mode 100644 index 0000000000..2194bd9029 --- /dev/null +++ b/app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py @@ -0,0 +1,65 @@ +"""make operation event spool completion conservative + +Revision ID: 20260805_000001_finalize_http_bridge_operation_spool +Revises: 20260805_000000_add_http_bridge_operation_spool +Create Date: 2026-08-05 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260805_000001_finalize_http_bridge_operation_spool" +down_revision = "20260805_000000_add_http_bridge_operation_spool" +branch_labels = None +depends_on = None + +_TABLE = "http_bridge_operations" + + +def _has_table(connection: Connection) -> bool: + return sa.inspect(connection).has_table(_TABLE) + + +def _has_column(connection: Connection, column: str) -> bool: + return any(item["name"] == column for item in sa.inspect(connection).get_columns(_TABLE)) + + +def upgrade() -> None: + bind = op.get_bind() + if not _has_table(bind) or not _has_column(bind, "event_spool_complete"): + return + # Rows written by older releases used true as the implicit value. They + # cannot be replayed safely unless their event queue is drained again. + op.execute(sa.text(f"UPDATE {_TABLE} SET event_spool_complete = false")) + # SQLite has no direct ALTER COLUMN syntax. Alembic's batch operation + # rebuilds the table and preserves the false default for future inserts; + # merely changing the ORM declaration would leave the old true default in + # sqlite_master. + if bind.dialect.name == "sqlite": + with op.batch_alter_table(_TABLE) as batch_op: + batch_op.alter_column( + "event_spool_complete", + existing_type=sa.Boolean(), + existing_nullable=False, + server_default=sa.text("false"), + ) + else: + op.alter_column(_TABLE, "event_spool_complete", server_default=sa.text("false")) + + +def downgrade() -> None: + bind = op.get_bind() + if _has_table(bind) and _has_column(bind, "event_spool_complete"): + if bind.dialect.name == "sqlite": + with op.batch_alter_table(_TABLE) as batch_op: + batch_op.alter_column( + "event_spool_complete", + existing_type=sa.Boolean(), + existing_nullable=False, + server_default=sa.text("true"), + ) + else: + op.alter_column(_TABLE, "event_spool_complete", server_default=sa.text("true")) diff --git a/app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py b/app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py new file mode 100644 index 0000000000..a769d9dfbc --- /dev/null +++ b/app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py @@ -0,0 +1,49 @@ +"""add anonymous telemetry identity and consent + +Revision ID: 20260806_000000_add_anonymous_telemetry +Revises: 20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads +Create Date: 2026-08-06 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "20260806_000000_add_anonymous_telemetry" +down_revision = "20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads" +branch_labels = None +depends_on = None + + +def _columns() -> set[str]: + return {column["name"] for column in sa.inspect(op.get_bind()).get_columns("dashboard_settings")} + + +def upgrade() -> None: + columns = _columns() + with op.batch_alter_table("dashboard_settings") as batch_op: + if "telemetry_consent" not in columns: + batch_op.add_column( + sa.Column( + "telemetry_consent", + sa.String(length=16), + server_default=sa.text("'undecided'"), + nullable=False, + ) + ) + if "telemetry_instance_id" not in columns: + batch_op.add_column(sa.Column("telemetry_instance_id", sa.String(length=36), nullable=True)) + if "telemetry_private_key_encrypted" not in columns: + batch_op.add_column(sa.Column("telemetry_private_key_encrypted", sa.LargeBinary(), nullable=True)) + + +def downgrade() -> None: + columns = _columns() + with op.batch_alter_table("dashboard_settings") as batch_op: + if "telemetry_private_key_encrypted" in columns: + batch_op.drop_column("telemetry_private_key_encrypted") + if "telemetry_instance_id" in columns: + batch_op.drop_column("telemetry_instance_id") + if "telemetry_consent" in columns: + batch_op.drop_column("telemetry_consent") diff --git a/app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py b/app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py new file mode 100644 index 0000000000..12e14906ae --- /dev/null +++ b/app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py @@ -0,0 +1,65 @@ +"""add API-key reasoning effort allowlists + +The nullable column preserves the existing unrestricted policy for every +existing API key. New writes serialize a non-empty canonical JSON list. + +Revision ID: 20260806_030000_add_api_key_allowed_reasoning_efforts +Revises: 20260816_000000_add_account_pending_deletion +Create Date: 2026-08-06 03:00:00.000000 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260806_030000_add_api_key_allowed_reasoning_efforts" +down_revision = "20260816_000000_add_account_pending_deletion" +branch_labels = None +depends_on = None + +_TABLE = "api_keys" +_COLUMN = "allowed_reasoning_efforts" +_POLICY_CHECK = "ck_api_keys_reasoning_policy_exclusive" + + +def _columns(connection: Connection) -> set[str]: + inspector = sa.inspect(connection) + if not inspector.has_table(_TABLE): + return set() + return {str(column["name"]) for column in inspector.get_columns(_TABLE) if column.get("name") is not None} + + +def _check_constraints(connection: Connection) -> set[str]: + inspector = sa.inspect(connection) + if not inspector.has_table(_TABLE): + return set() + return {str(constraint["name"]) for constraint in inspector.get_check_constraints(_TABLE) if constraint.get("name")} + + +def upgrade() -> None: + connection = op.get_bind() + columns = _columns(connection) + if _COLUMN not in columns: + with op.batch_alter_table(_TABLE) as batch_op: + batch_op.add_column(sa.Column(_COLUMN, sa.Text(), nullable=True)) + + constraints = _check_constraints(connection) + if _POLICY_CHECK not in constraints: + with op.batch_alter_table(_TABLE) as batch_op: + batch_op.create_check_constraint( + _POLICY_CHECK, + f"{_COLUMN} IS NULL OR enforced_reasoning_effort IS NULL", + ) + + +def downgrade() -> None: + connection = op.get_bind() + columns = _columns(connection) + if _COLUMN not in columns: + return + with op.batch_alter_table(_TABLE) as batch_op: + if _POLICY_CHECK in _check_constraints(connection): + batch_op.drop_constraint(_POLICY_CHECK, type_="check") + batch_op.drop_column(_COLUMN) diff --git a/app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py b/app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py new file mode 100644 index 0000000000..1a19cc4299 --- /dev/null +++ b/app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py @@ -0,0 +1,24 @@ +"""merge the durable HTTP bridge ledger with the current release head. + +The operation-ledger revisions were authored on the recovery branch while +main continued to receive additive schema revisions. This no-op merge keeps +Alembic at one head without rewriting either already-applied lineage. +""" + +from __future__ import annotations + +revision = "20260807_000000_merge_http_bridge_operation_spool_and_latest_main" +down_revision = ( + "20260806_120000_add_http_bridge_owner_process_epoch", + "20260805_000001_finalize_http_bridge_operation_spool", +) +branch_labels = None +depends_on = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py b/app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py new file mode 100644 index 0000000000..4b08cf493e --- /dev/null +++ b/app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py @@ -0,0 +1,44 @@ +"""persist the HTTP bridge ambiguous recovery dispatch budget + +Revision ID: 20260810_000000_add_http_bridge_recovery_dispatch_count +Revises: 20260807_000000_merge_http_bridge_operation_spool_and_latest_main +Create Date: 2026-08-10 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260810_000000_add_http_bridge_recovery_dispatch_count" +down_revision = "20260807_000000_merge_http_bridge_operation_spool_and_latest_main" +branch_labels = None +depends_on = None + +_TABLE = "http_bridge_operations" +_COLUMN = "recovery_dispatch_count" + + +def _has_table(connection: Connection) -> bool: + return sa.inspect(connection).has_table(_TABLE) + + +def _has_column(connection: Connection) -> bool: + return any(item["name"] == _COLUMN for item in sa.inspect(connection).get_columns(_TABLE)) + + +def upgrade() -> None: + bind = op.get_bind() + if not _has_table(bind) or _has_column(bind): + return + op.add_column( + _TABLE, + sa.Column(_COLUMN, sa.Integer(), nullable=False, server_default=sa.text("0")), + ) + + +def downgrade() -> None: + bind = op.get_bind() + if _has_table(bind) and _has_column(bind): + op.drop_column(_TABLE, _COLUMN) diff --git a/app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py b/app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py new file mode 100644 index 0000000000..78c4105b29 --- /dev/null +++ b/app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py @@ -0,0 +1,25 @@ +"""merge the recovery-dispatch and hourly-rollup migration heads. + +The durable HTTP bridge recovery branch and upstream's cancelled-count rollup +landed as independent additive revisions. This no-op merge keeps startup and +CI migration checks at one canonical Alembic head without rewriting either +already-applied lineage. +""" + +from __future__ import annotations + +revision = "20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads" +down_revision = ( + "20260810_000000_add_http_bridge_recovery_dispatch_count", + "20260811_000000_add_hourly_rollup_cancelled_count", +) +branch_labels = None +depends_on = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py b/app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py new file mode 100644 index 0000000000..100635eb9b --- /dev/null +++ b/app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py @@ -0,0 +1,43 @@ +"""add source scope to sticky continuity abandonment + +Revision ID: 20260812_120000_add_sticky_abandonment_scope +Revises: 20260813_000000_add_file_account_pins +Create Date: 2026-08-12 12:00:00.000000 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260812_120000_add_sticky_abandonment_scope" +down_revision = "20260813_000000_add_file_account_pins" +branch_labels = None +depends_on = None + +_TABLE = "sticky_sessions" +_COLUMN = "continuity_abandonment_scope" + + +def _columns(connection: Connection) -> set[str]: + inspector = sa.inspect(connection) + if not inspector.has_table(_TABLE): + return set() + return {str(column["name"]) for column in inspector.get_columns(_TABLE) if column.get("name") is not None} + + +def upgrade() -> None: + bind = op.get_bind() + if _COLUMN in _columns(bind): + return + with op.batch_alter_table(_TABLE) as batch_op: + batch_op.add_column(sa.Column(_COLUMN, sa.String(length=32), nullable=True)) + + +def downgrade() -> None: + bind = op.get_bind() + if _COLUMN not in _columns(bind): + return + with op.batch_alter_table(_TABLE) as batch_op: + batch_op.drop_column(_COLUMN) diff --git a/app/db/alembic/versions/20260813_000000_add_file_account_pins.py b/app/db/alembic/versions/20260813_000000_add_file_account_pins.py new file mode 100644 index 0000000000..dbfd6f4080 --- /dev/null +++ b/app/db/alembic/versions/20260813_000000_add_file_account_pins.py @@ -0,0 +1,40 @@ +"""add durable file account pins + +Revision ID: 20260813_000000_add_file_account_pins +Revises: 20260806_000000_add_anonymous_telemetry +Create Date: 2026-08-13 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "20260813_000000_add_file_account_pins" +down_revision = "20260806_000000_add_anonymous_telemetry" +branch_labels = None +depends_on = None + +_TABLE = "file_account_pins" + + +def upgrade() -> None: + bind = op.get_bind() + if sa.inspect(bind).has_table(_TABLE): + return + op.create_table( + _TABLE, + sa.Column("file_id", sa.String(), nullable=False), + sa.Column("account_id", sa.String(), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("file_id"), + ) + op.create_index("ix_file_account_pins_expires_at", _TABLE, ["expires_at"], unique=False) + + +def downgrade() -> None: + bind = op.get_bind() + if not sa.inspect(bind).has_table(_TABLE): + return + op.drop_index("ix_file_account_pins_expires_at", table_name=_TABLE) + op.drop_table(_TABLE) diff --git a/app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py b/app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py new file mode 100644 index 0000000000..8b9938181c --- /dev/null +++ b/app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py @@ -0,0 +1,81 @@ +"""add account pending-deletion marker columns + +Revision ID: 20260816_000000_add_account_pending_deletion +Revises: 20260812_120000_add_sticky_abandonment_scope +Create Date: 2026-08-16 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "20260816_000000_add_account_pending_deletion" +down_revision = "20260812_120000_add_sticky_abandonment_scope" +branch_labels = None +depends_on = None + +_TABLE = "accounts" +_INDEX = "idx_accounts_delete_requested_at" + + +def _columns(bind) -> set[str]: + return {column["name"] for column in sa.inspect(bind).get_columns(_TABLE)} + + +def _indexes(bind) -> set[str]: + return {index["name"] for index in sa.inspect(bind).get_indexes(_TABLE)} + + +def upgrade() -> None: + bind = op.get_bind() + columns = _columns(bind) + if "delete_requested_at" not in columns: + op.add_column(_TABLE, sa.Column("delete_requested_at", sa.DateTime(), nullable=True)) + if "delete_history_requested" not in columns: + op.add_column( + _TABLE, + sa.Column( + "delete_history_requested", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + ) + if _INDEX not in _indexes(bind): + # Pending-deletion queue probe/order support; partial so it is empty + # (and free) in the steady state with no pending deletions. + op.create_index( + _INDEX, + _TABLE, + ["delete_requested_at", "id"], + postgresql_where=sa.text("delete_requested_at IS NOT NULL"), + sqlite_where=sa.text("delete_requested_at IS NOT NULL"), + ) + + +def downgrade() -> None: + bind = op.get_bind() + columns = _columns(bind) + if "delete_requested_at" in columns: + # The marker columns are the deletion queue's only durable state: + # dropping them while deletions are queued would silently abandon + # acknowledged deletions and hand the parent build unusable + # (credential-wiped, partially drained) account rows it would list + # again. Refuse instead — let the worker finish (or supersede the + # deletions via re-import/reauth) before downgrading. + pending = bind.execute( + sa.text(f"SELECT COUNT(*) FROM {_TABLE} WHERE delete_requested_at IS NOT NULL") # noqa: S608 + ).scalar() + if pending: + raise RuntimeError( + f"cannot downgrade {revision}: {pending} account(s) are still queued for " + "background deletion; wait for the deletion worker to finish (or supersede " + "the deletions with a credential re-import) before downgrading" + ) + if _INDEX in _indexes(bind): + op.drop_index(_INDEX, table_name=_TABLE) + if "delete_history_requested" in columns: + op.drop_column(_TABLE, "delete_history_requested") + if "delete_requested_at" in columns: + op.drop_column(_TABLE, "delete_requested_at") diff --git a/app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py b/app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py new file mode 100644 index 0000000000..3619b2d0f1 --- /dev/null +++ b/app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py @@ -0,0 +1,50 @@ +"""add model source embeddings capability + +Revision ID: 20260816_000000_add_model_source_embeddings +Revises: 20260806_030000_add_api_key_allowed_reasoning_efforts +Create Date: 2026-08-16 00:00:00.000000 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260816_000000_add_model_source_embeddings" +down_revision = "20260806_030000_add_api_key_allowed_reasoning_efforts" +branch_labels = None +depends_on = None + + +def _has_table(connection: Connection, table_name: str) -> bool: + return sa.inspect(connection).has_table(table_name) + + +def _columns(connection: Connection, table_name: str) -> set[str]: + if not _has_table(connection, table_name): + return set() + return {column["name"] for column in sa.inspect(connection).get_columns(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + model_source_columns = _columns(bind, "model_sources") + if model_source_columns and "supports_embeddings" not in model_source_columns: + with op.batch_alter_table("model_sources") as batch_op: + batch_op.add_column( + sa.Column( + "supports_embeddings", + sa.Boolean(), + server_default=sa.false(), + nullable=False, + ) + ) + + +def downgrade() -> None: + bind = op.get_bind() + model_source_columns = _columns(bind, "model_sources") + if "supports_embeddings" in model_source_columns: + with op.batch_alter_table("model_sources") as batch_op: + batch_op.drop_column("supports_embeddings") diff --git a/app/db/alembic/versions/20260826_010000_merge_v1240_and_chek_heads.py b/app/db/alembic/versions/20260826_010000_merge_v1240_and_chek_heads.py new file mode 100644 index 0000000000..a55a7adae2 --- /dev/null +++ b/app/db/alembic/versions/20260826_010000_merge_v1240_and_chek_heads.py @@ -0,0 +1,24 @@ +"""merge upstream v1.24.0 and CHEK compatibility heads + +Revision ID: 20260826_010000_merge_v1240_and_chek_heads +Revises: 20260816_000000_add_model_source_embeddings, 20260826_000000_add_response_transition_manifest +Create Date: 2026-08-26 +""" + +from __future__ import annotations + +revision = "20260826_010000_merge_v1240_and_chek_heads" +down_revision = ( + "20260816_000000_add_model_source_embeddings", + "20260826_000000_add_response_transition_manifest", +) +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Join the two already-applied migration lineages.""" + + +def downgrade() -> None: + """Split back to the two parent migration heads.""" diff --git a/app/db/models.py b/app/db/models.py index 14c8aceec4..ff3de73a51 100644 --- a/app/db/models.py +++ b/app/db/models.py @@ -7,6 +7,7 @@ from sqlalchemy import ( BigInteger, Boolean, + CheckConstraint, DateTime, Float, ForeignKey, @@ -66,6 +67,16 @@ class RequestKind(str, Enum): WARMUP = "warmup" +class FileAccountPin(Base): + __tablename__ = "file_account_pins" + + file_id: Mapped[str] = mapped_column(String, primary_key=True) + account_id: Mapped[str] = mapped_column(String, nullable=False) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + __table_args__ = (Index("ix_file_account_pins_expires_at", "expires_at"),) + + class Account(Base): __tablename__ = "accounts" @@ -127,6 +138,20 @@ class Account(Base): server_default=false(), nullable=False, ) + # Pending-deletion marker: set by the fast DELETE path, consumed by the + # background deletion worker, cleared only by a credential replacement + # (re-import/reauth) that supersedes the deletion. Non-NULL rows are + # hidden from account listings and are already unroutable (the fast path + # also sets status=DEACTIVATED). + delete_requested_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + # Frozen at the first delete request (repeat requests do not escalate): + # True selects the history-deleting variant in the background worker. + delete_history_requested: Mapped[bool] = mapped_column( + Boolean, + default=False, + server_default=false(), + nullable=False, + ) api_key_assignments: Mapped[list["ApiKeyAccountAssignment"]] = relationship( "ApiKeyAccountAssignment", @@ -777,17 +802,15 @@ class StickySession(Base): onupdate=func.now(), nullable=False, ) - # Set only by purge_stale_hard_codex_session_mappings's first pass. A hard - # codex_session row normally proves ownership for `conversation`-continuity - # requests (see affinity.py's require_unambiguous_account), which have no - # other owner index. Once the durably-unavailable owner's proof is this - # stale, we stop treating the row as a live pin (so a fresh account can be - # selected) but keep it around with this marker set instead of deleting it - # outright, so selection can tell "this key was deliberately abandoned, - # picking a new owner is authorized" apart from "this key was never seen, - # ambiguity must fail closed." The row is only ever hard-deleted once it - # has sat abandoned past a further grace window with nobody claiming it. + # A non-null timestamp with NULL scope is the historical global tombstone. + # Source-scoped abandonment instead leaves this timestamp NULL and stores + # the typed scope below. That asymmetry is intentional: binaries predating + # the scope column see a live hard owner during rollout/rollback, while new + # binaries can let a process-session restart ignore the ambiguous raw row + # without erasing its explicit-turn-state ownership. Stale-hard cleanup + # may later promote the scoped marker to a timestamped global tombstone. continuity_abandoned_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None) + continuity_abandonment_scope: Mapped[str | None] = mapped_column(String(32), nullable=True, default=None) class CapabilityLineageMarker(Base): @@ -938,6 +961,14 @@ class DashboardSettings(Base): ) totp_secret_encrypted: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True) totp_last_verified_step: Mapped[int | None] = mapped_column(Integer, nullable=True) + telemetry_consent: Mapped[str] = mapped_column( + String(16), + default="undecided", + server_default=text("'undecided'"), + nullable=False, + ) + telemetry_instance_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + telemetry_private_key_encrypted: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True) http_responses_session_bridge_prompt_cache_idle_ttl_seconds: Mapped[int] = mapped_column( Integer, default=3600, @@ -1115,6 +1146,12 @@ class ApiFirewallAllowlist(Base): class ApiKey(Base): __tablename__ = "api_keys" + __table_args__ = ( + CheckConstraint( + "allowed_reasoning_efforts IS NULL OR enforced_reasoning_effort IS NULL", + name="ck_api_keys_reasoning_policy_exclusive", + ), + ) id: Mapped[str] = mapped_column(String, primary_key=True) name: Mapped[str] = mapped_column(String, nullable=False) @@ -1129,6 +1166,7 @@ class ApiKey(Base): ) enforced_model: Mapped[str | None] = mapped_column(String, nullable=True) enforced_reasoning_effort: Mapped[str | None] = mapped_column(String, nullable=True) + allowed_reasoning_efforts: Mapped[str | None] = mapped_column(Text, nullable=True) enforced_service_tier: Mapped[str | None] = mapped_column(String, nullable=True) traffic_class: Mapped[str] = mapped_column( String, @@ -1237,6 +1275,12 @@ class ModelSource(Base): server_default=false(), nullable=False, ) + supports_embeddings: Mapped[bool] = mapped_column( + Boolean, + default=False, + server_default=false(), + nullable=False, + ) timeout_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) max_concurrency: Mapped[int | None] = mapped_column(Integer, nullable=True) created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), nullable=False) @@ -1805,12 +1849,22 @@ class HttpBridgeRecoveryAttemptState(str, Enum): class HttpBridgeRowlessRecoveryState(str, Enum): + """Legacy recovery authority states retained for schema compatibility.""" + CAPTURED = "captured" APPROVED = "approved" UNKNOWN = "unknown" CONSUMED = "consumed" +class HttpBridgeOperationState(str, Enum): + SUBMITTED = "submitted" + UNKNOWN = "unknown" + ACKNOWLEDGED = "acknowledged" + COMPLETED = "completed" + FAILED = "failed" + + class HttpBridgeSessionRecord(Base): __tablename__ = "http_bridge_sessions" @@ -1844,6 +1898,9 @@ class HttpBridgeSessionRecord(Base): latest_input_item_count: Mapped[int | None] = mapped_column(Integer, nullable=True) latest_input_full_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True) latest_pending_tool_calls_json: Mapped[str | None] = mapped_column(Text, nullable=True) + # Legacy CHEK recovery columns remain mapped so migration drift checks and + # rollback-compatible deployments recognize the production schema. The + # upstream runtime does not consume these fields. latest_response_transition_manifest_json: Mapped[str | None] = mapped_column(Text, nullable=True) recovery_required_anchor_hash: Mapped[str | None] = mapped_column(String(64), nullable=True) recovery_required_account_id: Mapped[str | None] = mapped_column(String, nullable=True) @@ -1931,7 +1988,7 @@ class HttpBridgeRecoveryAttemptRecord(Base): class HttpBridgeRowlessRecoveryAuthority(Base): - """Content-free operator authority that outlives bridge-session cleanup.""" + """Legacy CHEK authority rows retained as an inert schema contract.""" __tablename__ = "http_bridge_rowless_recovery_authorities" @@ -2015,6 +2072,79 @@ class HttpBridgeRowlessRecoveryAuthority(Base): ) +class HttpBridgeOperationRecord(Base): + """Durable identity and outcome for a continuity-bound response.create.""" + + __tablename__ = "http_bridge_operations" + + operation_id: Mapped[str] = mapped_column(String(80), primary_key=True) + session_id: Mapped[str] = mapped_column( + String(36), + ForeignKey("http_bridge_sessions.id", ondelete="CASCADE"), + nullable=False, + ) + request_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False) + account_id: Mapped[str | None] = mapped_column(String, nullable=True) + model: Mapped[str | None] = mapped_column(String, nullable=True) + parent_response_id: Mapped[str | None] = mapped_column(Text, nullable=True) + request_text: Mapped[str | None] = mapped_column(Text, nullable=True) + state: Mapped[str] = mapped_column(String(32), nullable=False, server_default=text("'submitted'")) + response_id: Mapped[str | None] = mapped_column(Text, nullable=True) + recovery_dispatch_count: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0")) + event_bytes: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0")) + event_spool_complete: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false")) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=func.now(), server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=func.now(), server_default=func.now(), onupdate=func.now() + ) + + __table_args__ = ( + UniqueConstraint( + "session_id", + "request_fingerprint", + name="uq_http_bridge_operations_session_fingerprint", + ), + Index( + "uq_http_bridge_operations_request_fingerprint", + "request_fingerprint", + unique=True, + ), + Index("idx_http_bridge_operations_session_parent_state", "session_id", "parent_response_id", "state"), + Index("idx_http_bridge_operations_parent_state", "parent_response_id", "state", "updated_at"), + Index("idx_http_bridge_operations_state_updated", "state", "updated_at"), + ) + + +class HttpBridgeOperationEvent(Base): + """Replayable upstream SSE blocks for a durable bridge operation.""" + + __tablename__ = "http_bridge_operation_events" + + event_id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + operation_id: Mapped[str] = mapped_column( + String(80), + ForeignKey("http_bridge_operations.operation_id", ondelete="CASCADE"), + nullable=False, + ) + sequence_number: Mapped[int] = mapped_column(Integer, nullable=False) + event_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False) + event_text: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=func.now(), server_default=func.now() + ) + + __table_args__ = ( + UniqueConstraint( + "operation_id", + "event_fingerprint", + name="uq_http_bridge_operation_events_operation_fingerprint", + ), + Index("idx_http_bridge_operation_events_operation_sequence", "operation_id", "sequence_number"), + ) + + class HttpBridgeSessionAlias(Base): __tablename__ = "http_bridge_session_aliases" @@ -2122,6 +2252,17 @@ class HttpBridgeRetryCircuit(Base): postgresql_include=["used_percent", "reset_at", "window_minutes", "id"], ) Index("idx_accounts_email", Account.email) +# Pending-deletion queue: every replica probes ``delete_requested_at IS NOT +# NULL LIMIT 1`` each worker interval and the leader orders the queue by +# (delete_requested_at, id); the partial index keeps both reads off the full +# accounts table and is empty in the steady state (no pending deletions). +Index( + "idx_accounts_delete_requested_at", + Account.delete_requested_at, + Account.id, + postgresql_where=text("delete_requested_at IS NOT NULL"), + sqlite_where=text("delete_requested_at IS NOT NULL"), +) Index("idx_api_keys_name", ApiKey.name) Index("idx_logs_account_time", RequestLog.account_id, RequestLog.requested_at) Index("idx_logs_model_source_time", RequestLog.model_source_id, RequestLog.requested_at) diff --git a/app/db/session.py b/app/db/session.py index fa8867b825..afdd32df8c 100644 --- a/app/db/session.py +++ b/app/db/session.py @@ -1,18 +1,21 @@ from __future__ import annotations import asyncio +import inspect import logging import os import sqlite3 +import time from contextlib import asynccontextmanager from enum import StrEnum from pathlib import Path -from typing import TYPE_CHECKING, AsyncIterator, Awaitable, Callable, Protocol, TypeVar +from typing import TYPE_CHECKING, Any, AsyncIterator, Awaitable, Callable, Protocol, TypeVar import anyio from anyio import to_thread from sqlalchemy import event, text -from sqlalchemy.engine import Engine +from sqlalchemy import util as sqlalchemy_util +from sqlalchemy.engine import Connection, Engine from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.pool import NullPool @@ -34,6 +37,47 @@ _SQLITE_BUSY_TIMEOUT_MS = 30_000 _SQLITE_BUSY_TIMEOUT_SECONDS = _SQLITE_BUSY_TIMEOUT_MS / 1000 +# A write transaction holding SQLite's single writer slot past the busy +# timeout is exactly the holder that makes every other writer surface +# "database is locked" (issue #1682); the watchdog below reports it with the +# statements it ran, since the stall is nondeterministic and self-recovers. +_SQLITE_LONG_WRITE_TRANSACTION_WARN_SECONDS = _SQLITE_BUSY_TIMEOUT_SECONDS +_SQLITE_WRITE_STATEMENT_PREFIXES = ( + "insert", + "update", + "delete", + "replace", + "create", + "drop", + "alter", + "vacuum", + # BEGIN IMMEDIATE/EXCLUSIVE acquire the writer slot with no DML at all + # (e.g. AccountsRepository._acquire_sqlite_merge_lock); a plain deferred + # BEGIN does not and stays untracked. + "begin immediate", + "begin exclusive", +) +_SQLITE_WATCHDOG_STATEMENT_PREVIEW_CHARS = 300 +# Hard deadline for the shielded rollback/close teardown on SQLite (part 2 of +# the issue #1682 plan). A teardown wedged behind a stuck aiosqlite worker +# keeps holding the single writer slot, so it must be reclaimed well before +# other writers exhaust their busy timeout and surface "database is locked"; +# one-sixth of the busy timeout (5s) matches the leader-gated shielded-drain +# grace. Abandoning the wedged await alone would NOT release the lock — the +# aiosqlite worker thread still holds it — so on timeout the reclaim below +# interrupts the driver and invalidates the connection, disposing the worker. +_SQLITE_TEARDOWN_TIMEOUT_SECONDS = _SQLITE_BUSY_TIMEOUT_SECONDS / 6 +# Session.info marker set once a teardown step was abandoned as wedged: the +# session must never be driven by another coroutine again (the abandoned +# greenlet may still resume), and the deferred cleanup takes over. +_SQLITE_TEARDOWN_WEDGED_INFO_KEY = "sqlite_teardown_wedged" +# Abandoned wedged teardown tasks and the deferred bookkeeping closes they +# schedule on late completion, owned until completion so shutdown (close_db) +# drains them instead of closing the event loop over pending tasks. The +# bookkeeping closes are bounded by _SQLITE_TEARDOWN_TIMEOUT_SECONDS; an +# abandoned teardown may outlive its reclaim (the interrupt is best-effort), +# so the close_db drain is explicitly bounded as a whole. +_wedged_teardown_cleanup_tasks: set[asyncio.Task[Any]] = set() # PostgreSQL pool checkout timeout and connection recycle window. Fixed # application constants (issue #1340): recycle keeps pooled connections @@ -42,9 +86,6 @@ # ``database_pool_size`` / ``database_max_overflow``. _POSTGRES_POOL_TIMEOUT_SECONDS = 30.0 _POSTGRES_POOL_RECYCLE_SECONDS = 1800 -DATABASE_POOL_UNAVAILABLE_CODE = "database_pool_unavailable" -DATABASE_POOL_UNAVAILABLE_MESSAGE = "Database capacity is temporarily unavailable; retry shortly." -DATABASE_POOL_RETRY_AFTER_SECONDS = 1 _database_url = normalize_sqlite_url(_settings.database_url) @@ -81,7 +122,7 @@ def _postgres_async_connect_args(url: str) -> dict[str, object] | None: # mis-evaluated. Forcing UTC keeps stored timestamps correct regardless of # the container time zone. server_settings: dict[str, str] = {"timezone": "UTC"} - search_path = postgres_search_path(_settings.database_postgres_schema) + search_path = postgres_search_path(getattr(_settings, "database_postgres_schema", None)) if search_path is not None: server_settings["search_path"] = search_path connect_args: dict[str, object] = {"server_settings": server_settings} @@ -144,6 +185,135 @@ def _set_sqlite_pragmas(dbapi_connection: sqlite3.Connection, _: object) -> None finally: cursor.close() + _install_sqlite_long_write_watchdog(engine) + + +def _current_task_name_best_effort() -> str: + # Sync engine events run inside the greenlet driving the async engine on + # the event loop thread, so the owning task is normally visible; never let + # diagnostics raise into the query path. + try: + task = asyncio.current_task() + except RuntimeError: + return "" + if task is None: + return "" + return task.get_name() + + +def _install_sqlite_long_write_watchdog(engine: Engine) -> None: + """Report write transactions that outlive the SQLite busy timeout. + + In WAL mode a transaction takes the single writer slot at its first write + statement, not at BEGIN, so the window is measured from the first write to + commit/rollback. The report fires when the holder finally ends — the stall + in issue #1682 self-recovers, so identifying the holder post-hoc is the + point; a live sampler is not needed to attribute it. + """ + + @event.listens_for(engine, "after_cursor_execute") + def _track_write_statements( + conn: object, + cursor: object, + statement: str, + parameters: object, + context: object, + executemany: bool, + ) -> None: + # after_cursor_execute, not before: the first write statement may wait + # up to busy_timeout for the writer slot before failing, and timing + # from before the execute would report that victim as the holder. The + # slot is only held once a write statement has SUCCEEDED, so the clock + # starts there. + stripped = statement.lstrip().lower() + if not stripped.startswith(_SQLITE_WRITE_STATEMENT_PREFIXES): + return + info = getattr(conn, "info", None) + if info is None: + return + preview = statement[:_SQLITE_WATCHDOG_STATEMENT_PREVIEW_CHARS] + if "sqlite_write_started_at" not in info: + info["sqlite_write_started_at"] = time.monotonic() + info["sqlite_first_write_statement"] = preview + info["sqlite_write_task"] = _current_task_name_best_effort() + info["sqlite_last_write_statement"] = preview + + def _finalize_pending_report(info: dict[str, object]) -> None: + pending = info.pop("sqlite_write_pending_report", None) + if not isinstance(pending, tuple): + return + started_at, outcome, first_statement, last_statement, task_name = pending + held_seconds = time.monotonic() - float(started_at) + if held_seconds < _SQLITE_LONG_WRITE_TRANSACTION_WARN_SECONDS: + return + logger.warning( + "sqlite_long_write_transaction held_seconds=%.1f outcome=%s task=%s first_statement=%r " + "last_statement=%r — this writer starved every other writer past busy_timeout (issue #1682)", + held_seconds, + outcome, + task_name, + first_statement, + last_statement, + ) + + def _mark_transaction_ending(conn: object, *, outcome: str) -> None: + # ConnectionEvents.commit/rollback fire BEFORE the DBAPI call, and a + # wedged rollback is exactly the holder this watchdog hunts — reporting + # here would exclude the wedge itself from the measured hold. Stash the + # report and finalize it at the first proof the DBAPI transaction ended: + # the connection's next transaction beginning, or the connection going + # back to the pool. + info = getattr(conn, "info", None) + if info is None: + return + started_at = info.pop("sqlite_write_started_at", None) + first_statement = info.pop("sqlite_first_write_statement", None) + last_statement = info.pop("sqlite_last_write_statement", None) + task_name = info.pop("sqlite_write_task", None) + if started_at is None: + # A rollback after a commit whose DBAPI call raised: the pending + # report already holds outcome=commit, but the transaction is in + # fact ending by rollback — rewrite the outcome so the report does + # not claim a durable commit that never happened. + pending = info.get("sqlite_write_pending_report") + if outcome == "rollback" and isinstance(pending, tuple) and pending[1] == "commit": + info["sqlite_write_pending_report"] = (pending[0], "commit_failed_rollback", *pending[2:]) + return + info["sqlite_write_pending_report"] = (started_at, outcome, first_statement, last_statement, task_name) + + @event.listens_for(engine, "commit") + def _mark_on_commit(conn: object) -> None: + _mark_transaction_ending(conn, outcome="commit") + + @event.listens_for(engine, "rollback") + def _mark_on_rollback(conn: object) -> None: + _mark_transaction_ending(conn, outcome="rollback") + + @event.listens_for(engine, "begin") + def _finalize_on_next_begin(conn: object) -> None: + info = getattr(conn, "info", None) + if info is not None: + _finalize_pending_report(info) + + @event.listens_for(engine, "checkin") + def _finalize_on_checkin(dbapi_connection: object, connection_record: object) -> None: + info = getattr(connection_record, "info", None) + if info is not None: + _finalize_pending_report(info) + + @event.listens_for(engine, "handle_error") + def _flip_outcome_on_failed_end(exception_context: object) -> None: + # A DBAPI commit that raises still ends the transaction, but by + # rollback; the pending report marked at the commit event must not + # claim a durable commit that never happened. + connection = getattr(exception_context, "connection", None) + info = getattr(connection, "info", None) if connection is not None else None + if info is None: + return + pending = info.get("sqlite_write_pending_report") + if isinstance(pending, tuple) and pending[1] == "commit": + info["sqlite_write_pending_report"] = (pending[0], "commit_failed_rollback", *pending[2:]) + def _create_main_engine(url: str) -> AsyncEngine: if not _is_sqlite_url(url): @@ -200,33 +370,343 @@ def _startup_sqlite_check_mode(raw_mode: str) -> SqliteIntegrityCheckMode | None async def _shielded(awaitable: Awaitable[object]) -> None: task = asyncio.ensure_future(awaitable) + cancellation: asyncio.CancelledError | None = None + with anyio.CancelScope(shield=True): + while True: + try: + await asyncio.shield(task) + break + except asyncio.CancelledError as exc: + if task.cancelled(): + raise + cancellation = cancellation or exc + if cancellation is not None: + raise cancellation + + +async def _shielded_bounded(awaitable: Awaitable[object], timeout: float) -> asyncio.Task[object] | None: + """Shield ``awaitable`` from the caller's cancellation, waiting at most ``timeout``. + + Returns ``None`` when the awaitable finished inside the bound (re-raising + its exception like ``_shielded``); returns the still-running task when the + deadline passed — the caller must treat the underlying connection as + wedged and reclaim it, because the abandoned await does not release + anything the aiosqlite worker thread holds (issue #1682). + """ + task = asyncio.ensure_future(awaitable) + waiter = asyncio.ensure_future(asyncio.wait({task}, timeout=timeout)) + while not waiter.done(): + try: + await asyncio.shield(waiter) + except asyncio.CancelledError: + # Teardown runs in ``finally`` blocks: the bound, not the caller's + # cancellation, decides abandonment. ``asyncio.wait`` cannot + # outlive its timeout, so this drain stays bounded. + continue + if task.done(): + task.result() + return None + return task + + +def _sqlite_uri_mode_active(url: Any) -> bool: + """Whether the pysqlite/aiosqlite dialect will connect in URI mode. + + Mirrors the dialect's ``create_connect_args``: URI mode is enabled only + when the URL query carries a ``uri`` value that coerces to true. + """ + query = getattr(url, "query", None) + if query is None: + return False try: - await asyncio.shield(task) - except asyncio.CancelledError: - await task - raise + value = query.get("uri") + except Exception: + return False + values = value if isinstance(value, (tuple, list)) else (value,) + for item in values: + if item is None: + continue + try: + if sqlalchemy_util.asbool(str(item)): + return True + except Exception: + # An unrecognized value would fail at connect time anyway; + # classify conservatively as file-backed (bounded). + continue + return False + + +def _session_teardown_bound_seconds(session: AsyncSession) -> float | None: + """Teardown deadline for this session, or None for the unbounded path. + + Only file-backed SQLite gets a bound: its single writer slot turns a + wedged teardown into a database-wide write stall (issue #1682). + PostgreSQL teardown semantics are deliberately untouched, and in-memory + SQLite shares one StaticPool connection with the whole process — the + reclaim's invalidation would destroy the entire database (the + database-backends spec requires preserving shared in-memory state), and + with a single shared connection there is no cross-connection writer + contention to starve in the first place. + """ + try: + bind = session.get_bind() + except Exception: + return None + if getattr(getattr(bind, "dialect", None), "name", None) != "sqlite": + return None + url = getattr(bind, "url", None) + if url is not None: + database = getattr(url, "database", None) + if not database: + return None + database_text = str(database) + if database_text == ":memory:": + return None + # SQLite URI forms (``sqlite:///file:name?mode=memory&cache=shared&uri=true``) + # are in-memory only when the pysqlite/aiosqlite dialect actually + # passes the database string to the driver as a URI, which it does + # only when the URL query carries a truthy ``uri`` — and SQLite itself + # parses a filename as a URI only when it starts with ``file:``. + # Without ``uri=true``, ``file:name?mode=memory`` is a *file-backed* + # database whose filename literally contains those characters, so it + # must keep the bounded teardown. + if _sqlite_uri_mode_active(url) and database_text.startswith("file:"): + # ``mode=memory`` normally rides the parsed URL's query; it only + # appears inside ``url.database`` when the URL escaped the query + # into the database portion. + if ":memory:" in database_text or "mode=memory" in database_text: + return None + query = getattr(url, "query", None) + if query is not None: + try: + mode = query.get("mode") + except Exception: + mode = None + modes = mode if isinstance(mode, (tuple, list)) else (mode,) + if any(str(value).lower() == "memory" for value in modes if value is not None): + return None + return _SQLITE_TEARDOWN_TIMEOUT_SECONDS + + +def _session_is_teardown_wedged(session: AsyncSession) -> bool: + try: + return bool(session.info.get(_SQLITE_TEARDOWN_WEDGED_INFO_KEY)) + except Exception: + return False + + +def _session_sync_connections(session: AsyncSession) -> tuple[Connection, ...]: + """Best-effort snapshot of the sync Connections held by the session's transaction. + + Captured before a teardown attempt so a wedged rollback can be attributed + and its connection reclaimed. Diagnostics only — never raises. + """ + try: + transaction = session.sync_session.get_transaction() + if transaction is None: + return () + connections = getattr(transaction, "_connections", None) + if not isinstance(connections, dict): + return () + # The transaction tracks each Connection under two keys (the + # Connection itself and its Engine); deduplicate by identity. + unique: dict[int, Connection] = {} + for value in connections.values(): + if isinstance(value, tuple) and value and isinstance(value[0], Connection): + unique[id(value[0])] = value[0] + return tuple(unique.values()) + except Exception: + return () + + +def _sqlite_watchdog_identifiers(connection: Connection) -> str: + """Render the long-write watchdog's identifiers for the wedged connection. + + Invalidation prevents the connection from ever reaching the watchdog's + deferred report (next begin / pool checkin), so the reclaim log carries + the same attribution instead. + """ + try: + info = connection.info + started_at = info.get("sqlite_write_started_at") + first_statement = info.get("sqlite_first_write_statement") + last_statement = info.get("sqlite_last_write_statement") + task_name = info.get("sqlite_write_task") + if started_at is None: + # The watchdog's commit/rollback listener already moved the + # identifiers into the deferred report — the wedge happened inside + # the transaction-ending call itself, exactly the issue #1682 + # shape. + pending = info.get("sqlite_write_pending_report") + if isinstance(pending, tuple) and len(pending) == 5: + started_at, _, first_statement, last_statement, task_name = pending + held = f"{time.monotonic() - started_at:.1f}" if isinstance(started_at, float) else "unknown" + return ( + f"write_held_seconds={held} write_task={task_name!r} " + f"first_statement={first_statement!r} last_statement={last_statement!r}" + ) + except Exception: + return "write_held_seconds=unknown" + + +async def _reclaim_wedged_sqlite_session( + session: AsyncSession, + abandoned: asyncio.Task[object], + connections: tuple[Connection, ...], + *, + phase: str, +) -> None: + """Release what a wedged SQLite teardown still holds and fence the session. + + Abandoning the wedged rollback/close is not enough: the aiosqlite worker + thread keeps holding the write lock (issue #1682). Interrupting the driver + aborts the C-level call the worker is stuck in, and invalidating the + connection terminates it at the pool — aiosqlite's ``stop()`` queues a + hard close of the underlying ``sqlite3`` connection, which releases the + writer slot and disposes the worker thread — so leader election and every + other writer recover instead of stalling behind the wedge. The invalidated + connection can never be handed out again. + """ + try: + session.info[_SQLITE_TEARDOWN_WEDGED_INFO_KEY] = True + except Exception: + logger.exception("Failed to fence a wedged SQLite session during teardown reclaim") + # Own the abandoned teardown before this coroutine's first await: if + # close_db runs concurrently with the reclaim, it must already see the + # pending task in the registry instead of returning while the rollback is + # still pending. The completion callbacks are attached only after the + # connection is invalidated below, so the deferred bookkeeping close can + # never touch a live connection. + _wedged_teardown_cleanup_tasks.add(abandoned) + for connection in connections: + logger.warning( + "sqlite_wedged_teardown phase=%s bound_seconds=%.1f %s — interrupting and invalidating the " + "connection so the writer slot is released instead of stalling every writer (issue #1682)", + phase, + _SQLITE_TEARDOWN_TIMEOUT_SECONDS, + _sqlite_watchdog_identifiers(connection), + ) + try: + driver = connection.connection.driver_connection + if driver is not None: + # aiosqlite's ``interrupt`` runs sqlite3_interrupt inline on + # this task — it never enters the (wedged) worker queue. In + # the pinned aiosqlite (0.22.x) it is a coroutine function; + # await the result only when it is awaitable so a driver that + # makes ``interrupt`` synchronous keeps working. + result = driver.interrupt() + if inspect.isawaitable(result): + await result + except Exception: + logger.warning( + "Interrupting a wedged SQLite connection failed — the invalidation below still " + "reclaims the writer slot, but the stuck statement may run to completion first", + exc_info=True, + ) + try: + connection.invalidate() + except Exception: + logger.debug("Invalidating a wedged SQLite connection failed", exc_info=True) + if not connections: + logger.warning( + "sqlite_wedged_teardown phase=%s bound_seconds=%.1f — no held connection to reclaim; " + "abandoning the wedged %s (issue #1682)", + phase, + _SQLITE_TEARDOWN_TIMEOUT_SECONDS, + phase, + ) + # The abandoned teardown is owned until completion (registered above, so + # close_db drains it and shutdown waits for — or boundedly abandons — the + # reclaimed rollback/close instead of returning while it is still + # pending). The discard callback is registered first so that when the task + # completes during the drain, deregistration happens before + # _finish_abandoned_teardown registers the follow-up bookkeeping close. + abandoned.add_done_callback(_wedged_teardown_cleanup_tasks.discard) + abandoned.add_done_callback(lambda task: _finish_abandoned_teardown(session, task, phase=phase)) + + +def _finish_abandoned_teardown(session: AsyncSession, task: asyncio.Task[object], *, phase: str) -> None: + if not task.cancelled(): + # The wedged teardown resuming into an interrupted/invalidated + # connection is expected to error; consume it so the abandoned task + # never logs "exception was never retrieved". + task.exception() + logger.info("Wedged SQLite teardown finished late phase=%s", phase) + if phase != "rollback": + return + + # The session was abandoned before ``close`` ran. Now that no other + # coroutine can be driving it, close it for bookkeeping — the connection + # is already invalidated, so this cannot touch the database. + async def _close_late() -> None: + try: + await asyncio.wait_for(session.close(), timeout=_SQLITE_TEARDOWN_TIMEOUT_SECONDS) + except BaseException: + logger.debug("Late close of a wedged SQLite session failed", exc_info=True) + + try: + cleanup_task = asyncio.get_running_loop().create_task(_close_late()) + except RuntimeError: + # Event loop already gone (shutdown); the invalidated connection was + # closed at the pool, nothing is leaked. + return + # Own the task until completion: close_db drains it so shutdown cannot + # skip the promised bookkeeping close or leave a pending-task warning. + _wedged_teardown_cleanup_tasks.add(cleanup_task) + cleanup_task.add_done_callback(_wedged_teardown_cleanup_tasks.discard) async def _safe_rollback(session: AsyncSession) -> None: if not session.in_transaction(): return + if _session_is_teardown_wedged(session): + # A previous bounded teardown abandoned a wedged rollback; the + # abandoned greenlet may still resume, so never drive this session + # concurrently. The reclaim already released the connection. + return + bound = _session_teardown_bound_seconds(session) + if bound is None: + try: + await _shielded(session.rollback()) + except BaseException: + return + return + held_connections = _session_sync_connections(session) try: - await _shielded(session.rollback()) + abandoned = await _shielded_bounded(session.rollback(), bound) except BaseException: return + if abandoned is not None: + await _reclaim_wedged_sqlite_session(session, abandoned, held_connections, phase="rollback") async def _safe_close(session: AsyncSession) -> None: + if _session_is_teardown_wedged(session): + # Deferred cleanup owns the session now; see _finish_abandoned_teardown. + return + bound = _session_teardown_bound_seconds(session) + if bound is None: + try: + await _shielded(session.close()) + except BaseException: + return + return + held_connections = _session_sync_connections(session) try: - await _shielded(session.close()) + abandoned = await _shielded_bounded(session.close(), bound) except BaseException: return + if abandoned is not None: + await _reclaim_wedged_sqlite_session(session, abandoned, held_connections, phase="close") async def close_session(session: AsyncSession) -> None: - if session.in_transaction(): - await _safe_rollback(session) - await _safe_close(session) + async def _close() -> None: + if session.in_transaction(): + await _safe_rollback(session) + await _safe_close(session) + + await _shielded(_close()) def detach_session_objects(session: AsyncSession) -> None: @@ -289,7 +769,7 @@ def init_background_db(url: str | None = None) -> None: @asynccontextmanager async def get_background_session() -> AsyncIterator[AsyncSession]: - """Session provider for detached background tasks and schedulers. + """Session provider for background tasks, schedulers, and auth dependencies. Uses the separate background pool if initialized, otherwise falls back to main pool. """ @@ -304,25 +784,6 @@ async def get_background_session() -> AsyncIterator[AsyncSession]: await close_session(session) -@asynccontextmanager -async def get_request_session() -> AsyncIterator[AsyncSession]: - """Explicit session boundary for foreground request work. - - Request middleware and authentication cache misses must use the main pool - so detached refresh and scheduler saturation cannot consume their checkout - budget. The session still owns rollback and close across cancellation and - timeout paths. - """ - session = SessionLocal() - try: - yield session - except BaseException: - await _safe_rollback(session) - raise - finally: - await close_session(session) - - async def relax_commit_durability(session: AsyncSession) -> None: """Relax commit durability for the current telemetry write transaction. @@ -371,8 +832,14 @@ async def sqlite_writer_section() -> AsyncIterator[None]: async def get_session() -> AsyncIterator[AsyncSession]: - async with get_request_session() as session: + session = SessionLocal() + try: yield session + except BaseException: + await _safe_rollback(session) + raise + finally: + await close_session(session) async def init_db() -> None: @@ -490,6 +957,30 @@ async def init_db() -> None: async def close_db() -> None: + if _wedged_teardown_cleanup_tasks: + # Abandoned wedged teardowns plus their deferred bookkeeping closes. + # Drain until the registry is stable — an abandoned teardown that + # completes during the drain schedules its bookkeeping close only + # after any one-time snapshot — and bound the whole drain so a + # teardown still wedged despite the reclaim (the interrupt is + # best-effort) cannot wedge shutdown too: one deadline covers the + # abandoned teardown and the bounded close it chains. + loop = asyncio.get_running_loop() + deadline = loop.time() + 2 * _SQLITE_TEARDOWN_TIMEOUT_SECONDS + while _wedged_teardown_cleanup_tasks: + remaining = deadline - loop.time() + if remaining <= 0: + logger.warning( + "close_db abandoned %d still-pending wedged-teardown task(s) after the bounded " + "drain; their connections were already reclaimed (issue #1682)", + len(_wedged_teardown_cleanup_tasks), + ) + break + await asyncio.wait(tuple(_wedged_teardown_cleanup_tasks), timeout=remaining) + # Completion callbacks (deregistration and scheduling of the + # deferred bookkeeping close) run via call_soon; yield once so + # the registry reflects them before the next stability check. + await asyncio.sleep(0) await engine.dispose() if _background_engine is not None: await _background_engine.dispose() diff --git a/app/dependencies.py b/app/dependencies.py index 09a709b2bb..8fd8131e07 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -8,9 +8,8 @@ from fastapi import Depends, FastAPI, Request, WebSocket from sqlalchemy.ext.asyncio import AsyncSession -from app.db.session import get_background_session, get_request_session, get_session +from app.db.session import get_background_session, get_session from app.modules.accounts.auth_manager import AuthManager -from app.modules.accounts.background_repository import BackgroundAccountsRepository from app.modules.accounts.repository import AccountsRepository from app.modules.accounts.service import AccountsService from app.modules.api_keys.repository import ApiKeysRepository @@ -170,7 +169,7 @@ def get_accounts_context( usage_repository, additional_usage_repository, limit_warmup_repository, - auth_manager=AuthManager(repository, refresh_repo_factory=_accounts_refresh_repo_context), + auth_manager=AuthManager(repository, refresh_repo_factory=_accounts_repo_context), ) return AccountsContext( session=session, @@ -205,26 +204,15 @@ def get_usage_context( ) -@asynccontextmanager -async def _accounts_refresh_repo_context() -> AsyncIterator[BackgroundAccountsRepository]: - # Shielded refresh work must own only short, per-operation sessions. A - # context-held AccountsRepository would pin a checkout across upstream I/O. - yield BackgroundAccountsRepository() - - @asynccontextmanager async def _accounts_repo_context() -> AsyncIterator[AccountsRepository]: - # OAuth token persistence runs after the external exchange has completed; - # keep its existing single-transaction repository contract. async with get_background_session() as session: yield AccountsRepository(session) @asynccontextmanager async def _proxy_repo_context() -> AsyncIterator[ProxyRepositories]: - # This factory serves foreground proxy request work. Detached schedulers - # and refresh persistence use their explicit background repositories. - async with get_request_session() as session: + async with get_background_session() as session: yield ProxyRepositories( accounts=AccountsRepository(session), usage=UsageRepository(session), @@ -261,10 +249,7 @@ def get_proxy_service_for_app(app: FastAPI) -> ProxyService: state = app.state service = getattr(state, "proxy_service", None) if not isinstance(service, ProxyService): - service = ProxyService( - repo_factory=_proxy_repo_context, - refresh_repo_factory=_accounts_refresh_repo_context, - ) + service = ProxyService(repo_factory=_proxy_repo_context) setattr(state, "proxy_service", service) return service diff --git a/app/main.py b/app/main.py index f76f9f1ed3..83300cc21e 100644 --- a/app/main.py +++ b/app/main.py @@ -2,6 +2,7 @@ import asyncio import logging +import mimetypes import os import stat import sys @@ -45,6 +46,7 @@ add_request_body_limit_middleware, add_request_decompression_middleware, add_request_id_middleware, + add_required_capability_http_middleware, add_trusted_proxy_headers_middleware, ) from app.core.middleware.dashboard_gzip import add_dashboard_gzip_middleware @@ -52,15 +54,18 @@ from app.core.openai.model_refresh_scheduler import build_model_refresh_scheduler from app.core.resilience.backpressure import BackpressureMiddleware from app.core.resilience.bulkhead import BulkheadMiddleware, get_bulkhead +from app.core.resilience.loop_lag_monitor import run_event_loop_lag_monitor from app.core.resilience.memory_monitor import configure as configure_memory_monitor from app.core.retention.scheduler import build_data_retention_scheduler from app.core.scheduling.leader_election import get_leader_election from app.core.shutdown import close_control_plane_task_admission +from app.core.timeout_invariants import validate_runtime_timeout_invariants from app.core.usage.refresh_scheduler import build_usage_refresh_scheduler from app.core.usage.reset_credits_refresh_scheduler import build_rate_limit_reset_credits_scheduler from app.core.utils.time import utcnow from app.db.session import SessionLocal, close_db, close_session, init_background_db, init_db from app.modules.accounts import api as accounts_api +from app.modules.accounts.deletion import build_account_deletion_scheduler from app.modules.accounts.repository import AccountsRepository from app.modules.accounts.usage_rollup_scheduler import build_account_usage_rollup_scheduler from app.modules.api_keys import api as api_keys_api @@ -78,7 +83,6 @@ from app.modules.model_sources import api as model_sources_api from app.modules.oauth import api as oauth_api from app.modules.proxy import api as proxy_api -from app.modules.proxy import rowless_recovery_api from app.modules.proxy.cap_partitioning import refresh_cap_partition from app.modules.proxy.durable_bridge_coordinator import DurableBridgeSessionCoordinator from app.modules.proxy.durable_bridge_repository import missing_durable_bridge_tables @@ -102,12 +106,65 @@ _abandoned_bridge_retention_seconds, build_sticky_session_cleanup_scheduler, ) +from app.modules.telemetry import api as telemetry_api +from app.modules.telemetry.scheduler import build_telemetry_scheduler from app.modules.usage import api as usage_api from app.modules.usage.additional_quota_keys import reload_additional_quota_registry from app.modules.usage.live_ingest import start_live_usage_ingestor, stop_live_usage_ingestor logger = logging.getLogger(__name__) +# On Windows, ``mimetypes`` merges HKCR registry mappings where third-party +# software commonly remaps web extensions (``.js`` -> ``text/plain``), and +# browsers enforce strict MIME checking for ES module scripts, so a poisoned +# mapping renders the dashboard as a blank page (issue #1698). ``FileResponse`` +# resolves ``media_type`` through ``mimetypes.guess_type``, so pin every +# extension the built dashboard serves; ``add_type`` wins over the merged +# registry table on all platforms. +_WEB_ASSET_MIME_TYPES: dict[str, str] = { + ".js": "text/javascript", + ".mjs": "text/javascript", + ".css": "text/css", + ".svg": "image/svg+xml", + ".json": "application/json", + ".woff2": "font/woff2", + ".woff": "font/woff", + ".html": "text/html", +} + + +def _ensure_web_asset_mime_types() -> None: + for extension, mime_type in _WEB_ASSET_MIME_TYPES.items(): + mimetypes.add_type(mime_type, extension) + + +_ensure_web_asset_mime_types() + + +async def run_http_bridge_heartbeat_maintenance(proxy_service: Any) -> None: + """Per-replica bridge upkeep driven by the ring heartbeat. + + Both passes are request-independent by design: durable ownership must be + reconciled even on a replica nothing is routing to, and the idle sweep is + otherwise only reached from ``_get_or_create_http_bridge_session``, so a + replica that stops taking bridge requests would keep its idle sessions' + upstream WebSockets open until restart (issue #1354). Each pass is isolated + so one failing cannot skip the other or stop the heartbeat. + """ + if proxy_service is None: + return + for attribute, failure_message in ( + ("reconcile_durable_http_bridge_ownership", "HTTP bridge durable ownership reconciliation failed"), + ("prune_idle_http_bridge_sessions", "HTTP bridge idle sweep failed"), + ): + pass_callable = getattr(proxy_service, attribute, None) + if pass_callable is None: + continue + try: + await pass_callable() + except Exception: + logger.warning(failure_message, exc_info=True) + def _log_abandoned_lease_release(task: asyncio.Task[None]) -> None: if task.cancelled(): @@ -143,6 +200,26 @@ async def _release_leader_lease_within(timeout: float) -> None: logger.warning("Failed to release scheduler leader lease during shutdown", exc_info=exc) +async def _drain_proxy_persistence_tasks( + proxy_service: Any, + timeout_seconds: float, + *, + task_name_prefixes: tuple[str, ...] | None = None, + failure_message: str, +) -> bool: + """Drain proxy persistence work within the caller's committed deadline.""" + if proxy_service is None or not hasattr(proxy_service, "drain_persistence_tasks"): + return True + try: + kwargs: dict[str, Any] = {"timeout_seconds": timeout_seconds} + if task_name_prefixes is not None: + kwargs["task_name_prefixes"] = task_name_prefixes + return bool(await proxy_service.drain_persistence_tasks(**kwargs)) + except Exception: + logger.warning(failure_message, exc_info=True) + return False + + async def _drain_detached_control_plane_tasks(timeout_seconds: float) -> None: # Closing admission is synchronous with producer checks on the event loop, # so no task can appear after the stable drain passes complete. @@ -257,6 +334,7 @@ async def lifespan(app: FastAPI): reload_additional_quota_registry() settings = get_settings() warn_removed_settings() + validate_runtime_timeout_invariants(settings) # Anchor round-robin tie-break decorrelation to this replica's stable bridge # instance identity so peer replicas spread exact ties across equally-good # accounts instead of all herding onto the lexicographically-first account. @@ -289,6 +367,15 @@ async def lifespan(app: FastAPI): "deleted": deleted_bridge_rows, }, ) + purged_operation_rows = await DurableBridgeSessionCoordinator(SessionLocal).purge_operation_spool( + cutoff=utcnow() + - timedelta(seconds=settings.http_responses_session_bridge_operation_spool_retention_seconds), + ) + if purged_operation_rows > 0: + logger.info( + "Purged expired durable HTTP bridge operation transcript rows", + extra={"deleted": purged_operation_rows}, + ) from app.core.auth.api_key_cache import get_api_key_cache from app.core.cache.invalidation import ( NAMESPACE_ACCOUNT_ROUTING, @@ -405,8 +492,13 @@ async def lifespan(app: FastAPI): automations_scheduler = build_automations_scheduler() rate_limit_reset_credits_scheduler = build_rate_limit_reset_credits_scheduler() account_usage_rollup_scheduler = build_account_usage_rollup_scheduler() + account_deletion_scheduler = build_account_deletion_scheduler() data_retention_scheduler = build_data_retention_scheduler() - start_live_usage_ingestor() + telemetry_scheduler = build_telemetry_scheduler() + # Hold the instance: this lifespan owns it (and keeps it strongly rooted) + # even if a nested lifespan on another loop replaces the module-global + # singleton in the meantime; shutdown below stops exactly this instance. + live_usage_ingestor = start_live_usage_ingestor() await usage_scheduler.start() await api_key_limit_reset_scheduler.start() await api_key_last_used_flush_scheduler.start() @@ -417,7 +509,9 @@ async def lifespan(app: FastAPI): await automations_scheduler.start() await rate_limit_reset_credits_scheduler.start() await account_usage_rollup_scheduler.start() + await account_deletion_scheduler.start() await data_retention_scheduler.start() + await telemetry_scheduler.start() if settings.metrics_enabled and PROMETHEUS_AVAILABLE: import uvicorn @@ -484,12 +578,7 @@ async def _heartbeat_only(svc: RingMembershipService, iid: str) -> None: await svc.heartbeat(iid, endpoint_base_url=bridge_endpoint_base_url) except Exception: logger.warning("Ring heartbeat failed", exc_info=True) - proxy_service = getattr(app.state, "proxy_service", None) - if proxy_service is not None and hasattr(proxy_service, "reconcile_durable_http_bridge_ownership"): - try: - await proxy_service.reconcile_durable_http_bridge_ownership() - except Exception: - logger.warning("HTTP bridge durable ownership reconciliation failed", exc_info=True) + await run_http_bridge_heartbeat_maintenance(getattr(app.state, "proxy_service", None)) await refresh_cap_partition(svc.list_active, iid) async def _register_and_heartbeat(svc: RingMembershipService, iid: str) -> None: @@ -519,6 +608,13 @@ async def _activate_bridge_membership(svc: RingMembershipService, iid: str) -> N ring_service = RingMembershipService(SessionLocal) instance_id = settings.http_responses_session_bridge_instance_id heartbeat_task = asyncio.create_task(_register_and_heartbeat(ring_service, instance_id)) + loop_lag_task: asyncio.Task[None] | None = None + if settings.event_loop_lag_warn_threshold_seconds > 0: + loop_lag_task = asyncio.create_task( + run_event_loop_lag_monitor( + warn_threshold_seconds=settings.event_loop_lag_warn_threshold_seconds, + ) + ) startup_module._startup_complete = True try: @@ -538,14 +634,13 @@ async def _activate_bridge_membership(svc: RingMembershipService, iid: str) -> N recovery_settlements_drained = True # Settle detached recovery journals while their origin leases are # still held; bridge teardown below may release those owner fences. - if proxy_service is not None and hasattr(proxy_service, "drain_persistence_tasks"): - try: - recovery_settlements_drained = await proxy_service.drain_persistence_tasks( - timeout_seconds=settings.shutdown_drain_timeout_seconds, - task_name_prefixes=("http-bridge-recovery-settlement-",), - ) - except Exception: - logger.warning("Failed to pre-drain proxy settlement tasks during shutdown", exc_info=True) + remaining_drain_seconds = shutdown_state.remaining_drain_timeout_seconds() or 0.0 + recovery_settlements_drained = await _drain_proxy_persistence_tasks( + proxy_service, + remaining_drain_seconds, + task_name_prefixes=("http-bridge-recovery-settlement-",), + failure_message="Failed to pre-drain proxy settlement tasks during shutdown", + ) if ( recovery_settlements_drained and proxy_service is not None @@ -563,12 +658,12 @@ async def _activate_bridge_membership(svc: RingMembershipService, iid: str) -> N # Drain AFTER the bridge teardown: failing a bridge's pending # requests writes their request logs, which enqueues more # persistence tasks that this drain must cover. - if proxy_service is not None and hasattr(proxy_service, "drain_persistence_tasks"): - try: - remaining_drain_seconds = shutdown_state.remaining_drain_timeout_seconds() or 0.0 - await proxy_service.drain_persistence_tasks(timeout_seconds=remaining_drain_seconds) - except Exception: - logger.warning("Failed to drain proxy persistence tasks during shutdown", exc_info=True) + remaining_drain_seconds = shutdown_state.remaining_drain_timeout_seconds() or 0.0 + await _drain_proxy_persistence_tasks( + proxy_service, + remaining_drain_seconds, + failure_message="Failed to drain proxy persistence tasks during shutdown", + ) # Cancel heartbeat and age the shared ring row near expiry. if heartbeat_task is not None: @@ -578,6 +673,13 @@ async def _activate_bridge_membership(svc: RingMembershipService, iid: str) -> N except (asyncio.CancelledError, TimeoutError): pass + if loop_lag_task is not None: + loop_lag_task.cancel() + try: + await asyncio.wait_for(loop_lag_task, timeout=2) + except (asyncio.CancelledError, TimeoutError): + pass + if ring_service is not None and instance_id is not None: try: await asyncio.wait_for( @@ -636,10 +738,12 @@ async def _activate_bridge_membership(svc: RingMembershipService, iid: str) -> N # touch is never parked in a pending map with no remaining flusher. await api_key_last_used_flush_scheduler.stop() await usage_scheduler.stop() - await stop_live_usage_ingestor() + await stop_live_usage_ingestor(live_usage_ingestor) await rate_limit_reset_credits_scheduler.stop() await account_usage_rollup_scheduler.stop() + await account_deletion_scheduler.stop() await data_retention_scheduler.stop() + await telemetry_scheduler.stop() # Release the scheduler leader lease only after every leader-gated # scheduler has stopped so no local tick re-acquires it; followers can # then take over immediately instead of waiting out the lease TTL. @@ -690,6 +794,7 @@ def create_app() -> FastAPI: add_request_decompression_middleware(app) add_request_body_limit_middleware(app) add_multipart_content_encoding_middleware(app) + add_required_capability_http_middleware(app) add_request_id_middleware(app) add_api_firewall_middleware(app) app.add_middleware(cast(Any, MetricsMiddleware), enabled=settings.metrics_enabled) @@ -734,11 +839,11 @@ def create_app() -> FastAPI: app.include_router(quota_planner_api.router) app.include_router(reports_api.router) app.include_router(conversation_archive_api.router) - app.include_router(rowless_recovery_api.router) app.include_router(runtime_api.router) app.include_router(oauth_api.router) app.include_router(dashboard_auth_api.router) app.include_router(settings_api.router) + app.include_router(telemetry_api.router) app.include_router(firewall_api.router) app.include_router(fleet_api.router) app.include_router(sticky_sessions_api.router) @@ -802,9 +907,9 @@ async def _ensure_bridge_durable_schema_ready(settings) -> bool: return True missing = ", ".join(missing_tables) if settings.database_migrations_fail_fast: - raise RuntimeError(f"HTTP bridge durable schema is missing required items: {missing}") + raise RuntimeError(f"HTTP bridge durable schema is missing required tables: {missing}") logger.warning( - "HTTP bridge durable schema is missing required items but startup fail-fast is disabled", + "HTTP bridge durable schema is missing required tables but startup fail-fast is disabled", extra={"missing_tables": missing_tables}, ) return False diff --git a/app/modules/accounts/auth_manager.py b/app/modules/accounts/auth_manager.py index 483ce72fc2..206c66e4af 100644 --- a/app/modules/accounts/auth_manager.py +++ b/app/modules/accounts/auth_manager.py @@ -30,6 +30,7 @@ from app.core.crypto import TokenEncryptor from app.core.plan_types import coerce_account_plan_type from app.core.upstream_proxy import UpstreamProxyRouteError, resolve_upstream_route +from app.core.utils.shared_future import wait_on_shared_future from app.core.utils.time import utcnow from app.db.models import Account, AccountProxyBinding, AccountStatus from app.db.session import get_background_session @@ -196,7 +197,12 @@ async def run( self._inflight[key] = task task.add_done_callback(lambda done, *, cache_key=key: self._schedule_complete(cache_key, done)) assert task is not None - return await asyncio.shield(task) + # Not asyncio.shield: shield attaches per-waiter callbacks to the + # shared singleflight task, which degrades to O(N^2) removal scans + # when piled-up waiters are cancelled (see shared_future.py). The + # helper preserves shield semantics: a cancelled waiter detaches + # without aborting the refresh. + return await wait_on_shared_future(task) def _schedule_complete(self, key: _RefreshSingleflightKey, task: asyncio.Task[Account]) -> None: asyncio.create_task(self._complete(key, task)) @@ -205,8 +211,13 @@ async def _complete(self, key: _RefreshSingleflightKey, task: asyncio.Task[Accou try: async with self._lock: current = self._inflight.get(key) - if current is task: - self._inflight.pop(key, None) + if current is not task: + # A successor owns settlement for this key; consume the + # stale task's result without touching its cache state. + if not task.cancelled(): + task.exception() + return + self._inflight.pop(key, None) if task.cancelled(): self._recent_failures.pop(key, None) return @@ -286,9 +297,9 @@ async def ensure_fresh(self, account: Account, *, force: bool = False) -> Accoun async def _run_refresh(self, account: Account) -> Account: """Singleflight body for token refresh. - Runs inside a detached task that the singleflight keeps alive with - ``asyncio.shield`` (so concurrent waiters share one refresh and a - cancelled waiter does not abort it). Because the task outlives the + Runs inside a detached task that the singleflight keeps alive via + ``wait_on_shared_future`` (so concurrent waiters share one refresh and + a cancelled waiter does not abort it). Because the task outlives the caller, it MUST NOT use the caller's request-scoped session: when a client disconnects, the caller is cancelled and its ``async with get_background_session()`` closes that session, while this diff --git a/app/modules/accounts/deletion.py b/app/modules/accounts/deletion.py new file mode 100644 index 0000000000..70bd766b46 --- /dev/null +++ b/app/modules/accounts/deletion.py @@ -0,0 +1,539 @@ +"""Background account deletion: fast-marked accounts drained in bounded chunks. + +``DELETE /api/accounts/{id}`` used to detach (or delete) the account's entire +raw history in ONE transaction while holding the fold-state lock: for a +long-lived account that is hundreds of thousands of rows across ``request_logs`` +(18 indexes) and ``usage_history``, minutes of fold blockage, one pinned pool +connection, and an HTTP client timeout. The API now only stamps the +pending-deletion marker (``AccountsRepository.begin_delete``); this module +drains the bulk rows afterwards, ``DELETE_BATCH_SIZE`` rows per transaction, +and finalizes with the exact transaction shape the synchronous path used. + +Fold-safety argument — why the chunk transactions do NOT take the fold-state +lock, yet a deleted account's folded rows can never resurrect: + +1. Chunk transactions touch only raw rows (``usage_history``, + ``additional_usage_history``, ``request_logs``); they never write a rollup + table and never move a watermark. ``usage_history`` tables are not + fold-governed at all. +2. A fold slice that interleaves between chunks may aggregate rows still + attributed to the account (adding folded rows under the account dimension) + or rows a chunk already detached (adding them under the orphaned-deleted + dimension — exactly the soft-path end state). Both are converged by the + finalization transaction: it takes ``lock_fold_state()`` and only then + detaches/deletes the residual raw rows and runs the lifecycle mirrors, + which move or remove EVERY folded row carrying the account dimension, + including rows folded mid-drain. +3. Every fold slice holds the fold-state row lock from before it reads raw + rows until its commit. A slice therefore commits either before + finalization (its account-attributed output exists when the mirrors run + and is moved/removed by them) or after (it observes the post-finalization + raw state, which carries no attribution to the account). No slice can + commit pre-deletion attribution after the mirrors ran — the exact + resurrection the single-transaction path guarded against. + +Restart safety and idempotency: all progress lives in the database (the +marker columns plus the shrinking predicate ``WHERE account_id = :id``), so a +leader restart resumes mid-drain, and re-running any chunk is a no-op. +A credential replacement (re-import/reauth) clears the marker and supersedes +the deletion: every chunk transaction re-reads the marker under the account +row lock (PostgreSQL ``FOR NO KEY UPDATE``; the SQLite writer section +serializes writers) before touching rows, and finalization re-checks it the +same way, so no chunk can commit row work after a replacement committed and +a superseded account is never finalized. + +Fairness: a deletion pass round-robins one chunk per pending account and +re-scans for newly marked accounts between rounds, so one account's +multi-minute drain can neither starve another marked account nor delay a +delete request that arrives mid-pass. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import importlib +import logging +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from typing import Protocol, TypeVar, cast + +from sqlalchemy import Select, delete, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.auth.api_key_cache import get_api_key_cache +from app.core.cache.invalidation import NAMESPACE_API_KEY, get_cache_invalidation_poller +from app.core.upstream_proxy.cache import get_upstream_route_cache +from app.core.utils.time import utcnow +from app.db.models import ( + Account, + AccountStatus, + AdditionalUsageHistory, + ApiKeyAccountAssignment, + RequestLog, + UsageHistory, +) +from app.db.session import get_background_session, sqlite_writer_section +from app.modules.accounts.repository import ( + ACCOUNT_PENDING_DELETION_REASON, + AccountsRepository, + credentials_replaced_since_wipe, +) +from app.modules.proxy.account_cache import ( + get_account_selection_cache, + mark_account_routing_unavailable, + propagate_account_routing_change, +) +from app.modules.usage.repository import _clear_bulk_history_since_sqlite_cache + +logger = logging.getLogger(__name__) + +# Worker tick; the fast delete path additionally wakes the local worker, so a +# single-replica deployment (or a delete that lands on the leader) starts +# draining immediately and the tick only covers follower-received requests +# and restart resume. +DELETION_INTERVAL_SECONDS = 30 +# Rows per chunk transaction. Every chunk holds the account row lock +# (``FOR NO KEY UPDATE``) for its full duration, so a supersede +# (``replace_reauthorized``) or a fenced settlement write can wait for at most +# one chunk. Measured production rates: ~1.2s/10k usage_history deletes and +# ~23s/10k request_logs detaches (18 indexes, non-HOT updates) — 1k bounds the +# worst table at ~2.3s per transaction (5k would have been ~11.5s there). +DELETE_BATCH_SIZE = 1_000 +# Between consecutive row-touching rounds the pass sleeps a fraction of the +# round's own duration (capped), so a multi-hundred-chunk drain leaves the +# 2-vCPU database headroom for foreground traffic instead of running chunk +# transactions back-to-back. +INTER_ROUND_PAUSE_RATIO = 0.25 +INTER_ROUND_PAUSE_CAP_SECONDS = 2.0 + +_T = TypeVar("_T") + + +class _LeaderElectionLike(Protocol): + async def run_if_leader(self, fn: Callable[[], Awaitable[_T]]) -> _T | None: ... + + +def _get_leader_election() -> _LeaderElectionLike: + module = importlib.import_module("app.core.scheduling.leader_election") + return cast(_LeaderElectionLike, module.get_leader_election()) + + +async def run_account_deletion_pass(*, batch_size: int = DELETE_BATCH_SIZE) -> dict[str, str]: + """Drain and finalize every account marked for deletion. + + Round-robin fairness: each round advances every pending account by at + most one nonempty chunk, and the pending set is re-scanned between + rounds, so a newly marked account starts draining within one chunk + transaction of its request even while another account's long drain is in + progress. + + Returns an outcome per account id: ``finalized`` (rows drained, account + row removed), ``superseded`` (marker cleared mid-drain by a credential + replacement — deletion abandoned), or ``error`` (logged; retried on the + next tick). + """ + outcomes: dict[str, str] = {} + # Tables observed empty for an account earlier in THIS pass are not + # re-probed on later rounds: a drained table stays drained for the rest of + # the drain (rows that land afterwards — e.g. a stream settling a log row + # mid-drain — are swept by finalization's residual pass), and re-probing + # would cost one account-row-locking transaction per table per round. + drained: dict[str, set[str]] = {} + loop = asyncio.get_running_loop() + while True: + runnable = [ + account_id for account_id in await _pending_deletion_ids() if outcomes.get(account_id) in (None, "draining") + ] + if not runnable: + break + round_started = loop.time() + for account_id in runnable: + try: + outcomes[account_id] = await _advance_account( + account_id, batch_size=batch_size, drained=drained.setdefault(account_id, set()) + ) + except Exception: + logger.exception("Background account deletion failed account_id=%s", account_id) + outcomes[account_id] = "error" + if any(outcomes.get(account_id) == "draining" for account_id in runnable): + elapsed = loop.time() - round_started + await asyncio.sleep(min(INTER_ROUND_PAUSE_CAP_SECONDS, elapsed * INTER_ROUND_PAUSE_RATIO)) + # ``draining`` cannot survive the loop: an account leaves the runnable + # set only through a terminal outcome or by vanishing from the pending + # scan (its marker was cleared — a supersede that raced the scan). + for account_id, outcome in outcomes.items(): + if outcome == "draining": + outcomes[account_id] = "superseded" + if outcomes: + logger.info("Account deletion pass outcomes=%s", outcomes) + return outcomes + + +async def _pending_deletion_ids() -> list[str]: + async with get_background_session() as session: + rows = await session.execute( + select(Account.id) + .where(Account.delete_requested_at.is_not(None)) + .order_by(Account.delete_requested_at.asc(), Account.id.asc()) + ) + return list(rows.scalars().all()) + + +async def _advance_account(account_id: str, *, batch_size: int, drained: set[str] | None = None) -> str: + """One bounded round of work for one account. + + Runs the drain tables in order (usage snapshots first — not + fold-governed — then the raw request logs) but stops after the first + NONEMPTY chunk so the caller can round-robin other pending accounts — + each round commits at most one row-touching transaction per account: + ``draining`` means more work may remain. Only tables whose chunk came up + empty are known drained; when every table is, finalize. ``drained`` + (caller-owned, per pass) records those tables so later rounds skip their + probes instead of re-running one locking transaction per table per round. + """ + if drained is None: + drained = set() + for label, chunk_fn in ( + ("usage_history", _usage_history_chunk), + ("additional_usage_history", _additional_usage_history_chunk), + ("request_logs", _request_logs_chunk), + ): + if label in drained: + continue + affected = await _run_chunk(chunk_fn, account_id, batch_size=batch_size) + if affected is None: + return "superseded" + if not affected: + drained.add(label) + continue + if label == "usage_history": + # Same hygiene as retention pruning: bulk usage-history reads are + # cached on SQLite and must not serve the drained account. + _clear_bulk_history_since_sqlite_cache() + return "draining" + # Finalization: residual rows (streams that settled a log row mid-drain), + # folded-bucket mirrors, sticky/rollup rows, and the account row itself — + # one fold-state-locked transaction, identical in shape to the historical + # synchronous delete but over a residual row set instead of full history. + async with get_background_session() as session: + finalized = await AccountsRepository(session).delete(account_id, only_pending=True) + if not finalized: + return "superseded" + # Invalidate immediately (not at end of pass): the account row is gone + # and ids are deterministic, so cached routing/API-key snapshots must not + # outlive it while the pass keeps draining other accounts. + await _invalidate_account_caches() + return "finalized" + + +_ChunkFn = Callable[..., Awaitable[int]] + + +async def _run_chunk(chunk_fn: _ChunkFn, account_id: str, *, batch_size: int) -> int | None: + """Run one chunk transaction; None when the pending marker disappeared + (deletion superseded).""" + drift_repaired = False + async with get_background_session() as session: + async with sqlite_writer_section(): + state = await _pending_state(session, account_id) + if state is None: + await session.rollback() + return None + delete_history, drift_repaired = state + affected = await chunk_fn(session, account_id, delete_history=delete_history, batch_size=batch_size) + await session.commit() + if drift_repaired: + # The drift a pre-upgrade replica wrote may already be cached in + # selection/API-key snapshots on this or peer replicas; repairing the + # database alone would leave those caches selecting the wiped account + # (or honoring a stale assignment) until expiry. Same invalidation + # fan-out as the delete request itself. + mark_account_routing_unavailable(account_id) + await _invalidate_account_caches() + return affected + + +async def _pending_state(session: AsyncSession, account_id: str) -> tuple[bool, bool] | None: + """``(delete_history, drift_repaired)``, or None when no longer pending. + + On PostgreSQL the read locks the account row (``FOR NO KEY UPDATE``) for + the rest of the chunk transaction, so a credential replacement cannot + clear the marker between this read and the chunk's row mutations — the + replacement blocks until the chunk commits, then the next chunk observes + the cleared marker and stops. On SQLite the writer section already + serializes this transaction against every other writer. + + A replacement handled by a pre-upgrade replica (rolling deploy) writes + fresh credentials but cannot clear marker columns its ORM does not know; + fresh non-wiped ciphertext on a marked row is therefore itself the + supersede signal — the marker is cleared here, under the same lock. + """ + stmt = select( + Account.delete_requested_at, + Account.delete_history_requested, + Account.access_token_encrypted, + Account.refresh_token_encrypted, + Account.id_token_encrypted, + Account.status, + Account.deactivation_reason, + ).where(Account.id == account_id) + if session.get_bind().dialect.name == "postgresql": + stmt = stmt.with_for_update(key_share=True) + row = (await session.execute(stmt)).first() + if row is None or row[0] is None: + return None + if credentials_replaced_since_wipe(row[2], row[3], row[4]): + await session.execute( + update(Account) + .where(Account.id == account_id) + .values(delete_requested_at=None, delete_history_requested=False) + ) + await session.commit() + return None + # Self-heal drift written by pre-upgrade replicas during a rolling + # deploy (their writers are unfenced): a late settlement may have + # replaced the terminal status — making the wiped account selectable + # again — and an unconditional assignment insert may have recreated an + # API-key assignment begin_delete removed. Re-fence both under the row + # lock held above; any drift is bounded by one chunk transaction. The + # credentials check above already excluded genuine replacements, so a + # non-DEACTIVATED status here can only be such drift. The caller + # propagates cache invalidation after commit when drift was repaired. + drift_repaired = False + if row[5] is not AccountStatus.DEACTIVATED or row[6] != ACCOUNT_PENDING_DELETION_REASON: + drift_repaired = True + await session.execute( + update(Account) + .where(Account.id == account_id) + .values( + status=AccountStatus.DEACTIVATED, + deactivation_reason=ACCOUNT_PENDING_DELETION_REASON, + reset_at=None, + blocked_at=None, + ) + ) + assignment_rows = await session.execute( + delete(ApiKeyAccountAssignment) + .where(ApiKeyAccountAssignment.account_id == account_id) + .returning(ApiKeyAccountAssignment.account_id) + ) + if assignment_rows.scalars().first() is not None: + drift_repaired = True + return bool(row[1]), drift_repaired + + +# Chunk batch shape — why a RANGE predicate plus an index-matching ORDER BY +# instead of plain ``account_id = :id LIMIT n``: +# +# With an equality predicate the PostgreSQL planner folds ``account_id`` into +# a constant, drops it from the sort pathkeys, and — whenever the per-account +# row estimate is large (exactly the accounts this drain exists for) — plans +# the LIMIT subquery as an early-terminating Seq Scan (or a scan of an +# unrelated time index with a filter), betting on uniformly interleaved +# matches. That bet loses precisely mid-drain: detached/deleted rows no +# longer match, so each chunk re-scans a growing dead prefix, and once the +# table is drained but statistics are stale, every empty probe is a FULL heap +# scan (0 matches → no early termination). Verified against the production +# planner (606,970-row account): Seq Scans on all three tables. +# +# ``account_id >= :id AND account_id <= :id`` selects the same rows but keeps +# ``account_id`` out of the constant-equivalence class, so it survives as the +# leading ORDER BY pathkey; the ORDER BY then lists the target index's exact +# column order, making that account-leading index the only sort-free plan: +# +# usage_history → idx_usage_account_time (account_id, recorded_at) +# additional_usage_history → ix_additional_usage_distinct_labels +# (account_id, quota_key, limit_name, metered_feature) +# request_logs → idx_logs_account_kind_deleted_latest +# (account_id, request_kind, deleted_at, +# requested_at, id) — covering: Index Only Scan +# +# Verified on the production planner: all three plan as index (or index-only) +# scans with the account range as the Index Cond, and a drained-table probe +# costs one index descent (~8 cost units) instead of a heap scan. Chunk scan +# work is therefore bounded by the account's own remaining rows regardless of +# statistics staleness, and detached rows leave the scanned key range +# immediately. Row order is irrelevant to correctness (every row is drained); +# the ORDER BY exists purely to pin the plan. + + +def _usage_history_batch(account_id: str, batch_size: int) -> Select[tuple[int]]: + return ( + select(UsageHistory.id) + .where(UsageHistory.account_id >= account_id, UsageHistory.account_id <= account_id) + .order_by(UsageHistory.account_id, UsageHistory.recorded_at) + .limit(batch_size) + ) + + +def _additional_usage_history_batch(account_id: str, batch_size: int) -> Select[tuple[int]]: + return ( + select(AdditionalUsageHistory.id) + .where( + AdditionalUsageHistory.account_id >= account_id, + AdditionalUsageHistory.account_id <= account_id, + ) + .order_by( + AdditionalUsageHistory.account_id, + AdditionalUsageHistory.quota_key, + AdditionalUsageHistory.limit_name, + AdditionalUsageHistory.metered_feature, + ) + .limit(batch_size) + ) + + +def _request_logs_batch(account_id: str, batch_size: int) -> Select[tuple[int]]: + return ( + select(RequestLog.id) + .where(RequestLog.account_id >= account_id, RequestLog.account_id <= account_id) + .order_by( + RequestLog.account_id, + RequestLog.request_kind, + RequestLog.deleted_at, + RequestLog.requested_at, + RequestLog.id, + ) + .limit(batch_size) + ) + + +async def _usage_history_chunk(session: AsyncSession, account_id: str, *, delete_history: bool, batch_size: int) -> int: + batch = _usage_history_batch(account_id, batch_size).scalar_subquery() + result = await session.execute(delete(UsageHistory).where(UsageHistory.id.in_(batch)).returning(UsageHistory.id)) + return len(result.scalars().all()) + + +async def _additional_usage_history_chunk( + session: AsyncSession, account_id: str, *, delete_history: bool, batch_size: int +) -> int: + batch = _additional_usage_history_batch(account_id, batch_size).scalar_subquery() + result = await session.execute( + delete(AdditionalUsageHistory).where(AdditionalUsageHistory.id.in_(batch)).returning(AdditionalUsageHistory.id) + ) + return len(result.scalars().all()) + + +async def _request_logs_chunk(session: AsyncSession, account_id: str, *, delete_history: bool, batch_size: int) -> int: + """Detach (soft) or delete (hard) one chunk of the account's raw logs. + + Deliberately NOT fold-state-locked and NOT mirrored: see the module + docstring for why interleaved fold slices converge at finalization. + """ + batch = _request_logs_batch(account_id, batch_size).scalar_subquery() + if delete_history: + result = await session.execute(delete(RequestLog).where(RequestLog.id.in_(batch)).returning(RequestLog.id)) + else: + result = await session.execute( + update(RequestLog) + .where(RequestLog.id.in_(batch)) + .values(account_id=None, deleted_at=utcnow()) + .returning(RequestLog.id) + ) + return len(result.scalars().all()) + + +async def _invalidate_account_caches() -> None: + """Post-finalization invalidation, mirroring the synchronous delete path. + + Account ids are deterministic (delete-then-re-import regenerates the same + id), so cached route outcomes and API-key assignment snapshots must not + survive the account row's removal. + """ + get_account_selection_cache().invalidate() + get_api_key_cache().clear() + await get_upstream_route_cache().invalidate() + await propagate_account_routing_change() + poller = get_cache_invalidation_poller() + if poller is not None: + await poller.bump(NAMESPACE_API_KEY) + + +@dataclass(slots=True) +class AccountDeletionScheduler: + """Leader-gated worker tick with a local wake signal. + + Each tick first checks — with one cheap indexed-table read, before any + leader-election work — whether any account is pending deletion, so the + steady state (no pending deletions) costs one SELECT per interval. + """ + + interval_seconds: int + _task: asyncio.Task[None] | None = None + _stop: asyncio.Event = field(default_factory=asyncio.Event) + _wake: asyncio.Event = field(default_factory=asyncio.Event) + _lock: asyncio.Lock = field(default_factory=asyncio.Lock) + + async def start(self) -> None: + if self._task and not self._task.done(): + return + self._stop.clear() + self._task = asyncio.create_task(self._run_loop()) + + async def stop(self) -> None: + if not self._task: + return + self._stop.set() + self._task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._task + self._task = None + + def wake(self) -> None: + """Start the next pass immediately (fast delete path just committed).""" + self._wake.set() + + async def _run_loop(self) -> None: + while not self._stop.is_set(): + # Clear BEFORE running so a wake that lands mid-pass (a second + # delete request) schedules another pass instead of being lost. + self._wake.clear() + await self._run_once() + try: + await asyncio.wait_for(self._wake.wait(), timeout=self.interval_seconds) + except asyncio.TimeoutError: + continue + + async def _run_once(self) -> None: + try: + if not await _any_pending_deletion(): + return + except Exception: + logger.exception("Failed to check for pending account deletions") + return + await _get_leader_election().run_if_leader(self._run_as_leader) + + async def _run_as_leader(self) -> None: + async with self._lock: + try: + await run_account_deletion_pass() + except Exception: + logger.exception("Account deletion pass failed") + + +async def _any_pending_deletion() -> bool: + async with get_background_session() as session: + row = await session.execute(select(Account.id).where(Account.delete_requested_at.is_not(None)).limit(1)) + return row.scalar_one_or_none() is not None + + +_scheduler: AccountDeletionScheduler | None = None + + +def build_account_deletion_scheduler() -> AccountDeletionScheduler: + global _scheduler + _scheduler = AccountDeletionScheduler(interval_seconds=DELETION_INTERVAL_SECONDS) + return _scheduler + + +def request_account_deletion_run() -> None: + """Nudge the local worker after a delete request commits. + + A follower's nudge is a no-op (``run_if_leader`` declines) and the + leader's periodic tick picks the request up within the interval; when the + receiving replica IS the leader — the common single-replica case — the + drain starts immediately. + """ + if _scheduler is not None: + _scheduler.wake() diff --git a/app/modules/accounts/repository.py b/app/modules/accounts/repository.py index c56039e80a..58d1094461 100644 --- a/app/modules/accounts/repository.py +++ b/app/modules/accounts/repository.py @@ -1,20 +1,23 @@ from __future__ import annotations -import hashlib import json +import time import uuid from dataclasses import dataclass from datetime import datetime from typing import Any -from sqlalchemy import delete, or_, select, text, update +from sqlalchemy import case, delete, func, or_, select, text, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import AsyncSession +from app.core.auth import extract_id_token_claims, resolve_seat_identity +from app.core.crypto import TokenEncryptor from app.core.upstream_proxy.cache import get_upstream_route_cache from app.core.utils.time import utcnow +from app.db.account_identity_lock import advisory_lock_key, lock_postgresql_account_identities from app.db.models import ( Account, AccountLimitWarmup, @@ -50,6 +53,42 @@ _SETTINGS_ROW_ID = 1 _DUPLICATE_ACCOUNT_SUFFIX = "__copy" +# deactivation_reason stamped by the fast DELETE path while the background +# worker drains the account's rows. The authoritative pending marker is +# accounts.delete_requested_at; the reason string is operator-facing only. +ACCOUNT_PENDING_DELETION_REASON = "pending_deletion" + + +def credentials_replaced_since_wipe( + access_token_encrypted: bytes, + refresh_token_encrypted: bytes, + id_token_encrypted: bytes, +) -> bool: + """True when a marked account's token ciphertext is no longer the + empty-credential wipe stamped by :meth:`AccountsRepository.begin_delete`. + + New-code credential replacements clear the pending-deletion marker in the + same transaction, but a replacement handled by a PRE-UPGRADE replica + during a rolling deploy writes fresh ciphertext without knowing the + marker columns. Fresh (non-wiped) credentials on a still-marked row are + therefore themselves the supersede signal; the caller must clear the + marker and abandon the deletion. ALL THREE token fields are inspected: a + legal replacement may carry an empty refresh token while providing fresh + access/id material, and mistaking it for the wipe would finalize a + freshly replaced account. Undecryptable material also counts as + replaced — never finalize a row whose credentials we cannot attribute to + our own wipe. + """ + encryptor = TokenEncryptor() + for ciphertext in (access_token_encrypted, refresh_token_encrypted, id_token_encrypted): + try: + if encryptor.decrypt(ciphertext) != "": + return True + except Exception: + return True + return False + + _UNSET = object() _HARD_STICKY_UNAVAILABLE_STATUSES = frozenset( (AccountStatus.PAUSED, AccountStatus.RATE_LIMITED, AccountStatus.QUOTA_EXCEEDED) @@ -65,6 +104,62 @@ class AccountRequestUsageSummary: total_cost_usd: float +# The account-listing request-usage summary dedupes and re-aggregates the +# un-folded raw tail on every dashboard accounts load, and the displayed +# lifetime totals tolerate short staleness. Cache the merged summaries per +# account-id signature for a small fixed TTL, mirroring the request-log +# COUNT cache (issue #1340 / PRINCIPLES.md P2); the test suite patches the +# TTL to 0 so summaries stay exact within a test. Account deletion and +# duplicate-identity consolidation clear the cache because they re-attribute +# usage rather than merely append to it. +_SUMMARY_CACHE_TTL_SECONDS = 30.0 +_SUMMARY_CACHE_MAX_ENTRIES = 64 +_request_usage_summary_cache: dict[tuple[str, ...] | None, tuple[dict[str, AccountRequestUsageSummary], float]] = {} +# Invalidation generation: a fill that was already computing when a clear +# happened must not re-populate the cache with its pre-clear result. Fills +# capture the generation before their first await and stores are discarded +# on mismatch. Deletion/consolidation clear synchronously right after their +# commit (no await in between), so every store either precedes the commit +# (its stale data is wiped by the clear) or observes the bumped generation. +_summary_cache_generation = 0 + + +def _clear_request_usage_summary_cache() -> None: + global _summary_cache_generation + _summary_cache_generation += 1 + _request_usage_summary_cache.clear() + + +def _cached_request_usage_summaries( + key: tuple[str, ...] | None, +) -> dict[str, AccountRequestUsageSummary] | None: + entry = _request_usage_summary_cache.get(key) + if entry is None: + return None + summaries, expires_at = entry + if time.monotonic() >= expires_at: + _request_usage_summary_cache.pop(key, None) + return None + return summaries + + +def _store_request_usage_summaries( + key: tuple[str, ...] | None, + summaries: dict[str, AccountRequestUsageSummary], + ttl_seconds: float, + generation: int, +) -> None: + if generation != _summary_cache_generation: + return + if len(_request_usage_summary_cache) >= _SUMMARY_CACHE_MAX_ENTRIES: + oldest = min( + _request_usage_summary_cache, + key=lambda existing: _request_usage_summary_cache[existing][1], + ) + _request_usage_summary_cache.pop(oldest, None) + _request_usage_summary_cache[key] = (summaries, time.monotonic() + ttl_seconds) + + class AccountIdentityConflictError(Exception): def __init__(self, email: str) -> None: self.email = email @@ -74,6 +169,10 @@ def __init__(self, email: str) -> None: ) +class AccountIdentityRelockError(RuntimeError): + """Raised after identity membership changes across both bounded lock attempts.""" + + class AccountsRepository: def __init__(self, session: AsyncSession) -> None: self._session = session @@ -99,7 +198,11 @@ async def get_by_id_fresh(self, account_id: str) -> Account | None: return result.scalar_one_or_none() async def list_accounts(self, *, refresh_existing: bool = False) -> list[Account]: - stmt = select(Account).order_by(Account.email) + # Accounts marked for background deletion are already deleted from the + # operator's point of view: they never appear in listings (dashboard, + # usage refresh, automations) even though their rows survive until the + # deletion worker finishes draining them. + stmt = select(Account).where(Account.delete_requested_at.is_(None)).order_by(Account.email) if refresh_existing: stmt = stmt.execution_options(populate_existing=True) result = await self._session.execute(stmt) @@ -108,7 +211,12 @@ async def list_accounts(self, *, refresh_existing: bool = False) -> list[Account async def list_accounts_by_ids(self, account_ids: list[str], *, refresh_existing: bool = False) -> list[Account]: if not account_ids: return [] - stmt = select(Account).where(Account.id.in_(account_ids)).order_by(Account.email) + stmt = ( + select(Account) + .where(Account.id.in_(account_ids)) + .where(Account.delete_requested_at.is_(None)) + .order_by(Account.email) + ) if refresh_existing: stmt = stmt.execution_options(populate_existing=True) result = await self._session.execute(stmt) @@ -118,6 +226,13 @@ async def list_request_usage_summary_by_account( self, account_ids: list[str] | None = None, ) -> dict[str, AccountRequestUsageSummary]: + ttl_seconds = _SUMMARY_CACHE_TTL_SECONDS + cache_key = tuple(sorted(account_ids)) if account_ids is not None else None + generation = _summary_cache_generation + if ttl_seconds > 0: + cached = _cached_request_usage_summaries(cache_key) + if cached is not None: + return dict(cached) rollup_repo = AccountUsageRollupRepository(self._session) folded, watermark = await rollup_repo.read_state(account_ids) @@ -161,6 +276,9 @@ async def list_request_usage_summary_by_account( cached_input_tokens=cached_total, total_cost_usd=round(float(total_cost_usd), 6), ) + if ttl_seconds > 0: + _store_request_usage_summaries(cache_key, summaries, ttl_seconds, generation) + return dict(summaries) return summaries async def exists_active_chatgpt_account_id(self, chatgpt_account_id: str) -> bool: @@ -197,6 +315,7 @@ async def _upsert_unlocked( *, merge_by_email: bool | None = None, merge_by_chatgpt_identity: bool = False, + _identity_lock_attempt: int = 0, ) -> Account: dialect_name = self._dialect_name() sqlite_lock_acquired = False @@ -212,27 +331,34 @@ async def _upsert_unlocked( # exclusive on this dialect. await self._acquire_sqlite_merge_lock() elif dialect_name == "postgresql": - # Identity-keyed advisory lock must always be acquired when - # identity reconciliation is in play, regardless of - # merge_by_email. Two concurrent reauths for the same - # upstream chatgpt_account_id but different email claims - # (e.g. user changed email upstream) would otherwise take - # different email-scoped locks, both miss the canonical-row - # lookup below, and both INSERT a duplicate row for the - # same identity. - # - # Ordering identity-first, then email, gives a stable - # acquisition order across all callers so two concurrent - # reauths that overlap on either key serialize without - # deadlock. - identity_locked = False - if merge_by_chatgpt_identity and account.chatgpt_account_id: - await self._acquire_postgresql_identity_lock(f"chatgpt:{account.chatgpt_account_id}") - identity_locked = True + # Upstream identity membership always serializes before email + # locks and account row locks. This applies to ordinary imports as + # well as explicit identity reconciliation because either can add, + # replace, or remove a membership used by live-usage fallback. + locked_identities = await self._lock_postgresql_upsert_identity_candidates( + account, + include_email=bool(merge_by_email), + ) if merge_by_email: await self._acquire_postgresql_merge_lock(account.email) - elif not identity_locked: + elif not locked_identities: await self._acquire_postgresql_identity_lock(account.id) + if not await self._postgresql_upsert_identity_candidates_are_locked( + account, + include_email=bool(merge_by_email), + locked_identities=locked_identities, + ): + await self._session.rollback() + if _identity_lock_attempt >= 1: + raise AccountIdentityRelockError( + "Account identity candidates changed during PostgreSQL upsert locking" + ) + return await self._upsert_unlocked( + account, + merge_by_email=merge_by_email, + merge_by_chatgpt_identity=merge_by_chatgpt_identity, + _identity_lock_attempt=_identity_lock_attempt + 1, + ) # Identity-aware reconciliation runs before the deterministic-id # check so that a deactivated row whose refresh token was revoked @@ -262,6 +388,10 @@ async def _upsert_unlocked( await self._session.commit() if usage_cache_dirty: _clear_bulk_history_since_sqlite_cache() + # Consolidation re-attributes request logs and rollup sums + # to the canonical account; cached listing summaries would + # keep reporting the deleted duplicates until TTL expiry. + _clear_request_usage_summary_cache() # Duplicate reconciliation deletes Account rows, cascading # any account_proxy_bindings they owned. The route cache is # keyed by deterministic account id, so stale duplicate-id @@ -302,7 +432,13 @@ async def upsert_reauthorized(self, account: Account) -> Account: async def replace_reauthorized(self, account_id: str, account: Account) -> Account | None: """Replace credentials on the exact local row selected for reauthentication.""" async with sqlite_writer_section(): - existing = await self._session.get(Account, account_id) + if self._dialect_name() == "postgresql": + existing = await self._lock_postgresql_account_identity_membership( + account_id, + account.chatgpt_account_id, + ) + else: + existing = await self._session.get(Account, account_id) if existing is None: return None await self._apply_account_replacement(existing, account) @@ -344,6 +480,7 @@ async def _upsert_account_slot_unlocked( *, preserve_unknown_workspace_duplicates: bool | None = None, preserve_identity_slots: bool = False, + _identity_lock_attempt: int = 0, ) -> Account: if preserve_unknown_workspace_duplicates is None: preserve_unknown_workspace_duplicates = not await self._merge_by_email_enabled() @@ -351,6 +488,10 @@ async def _upsert_account_slot_unlocked( if dialect_name == "sqlite": await self._acquire_sqlite_merge_lock() elif dialect_name == "postgresql": + locked_identities = await self._lock_postgresql_upsert_identity_candidates( + account, + include_email=True, + ) for lock_key in sorted( _slot_lock_keys( account, @@ -358,6 +499,22 @@ async def _upsert_account_slot_unlocked( ) ): await self._acquire_postgresql_identity_lock(lock_key) + if not await self._postgresql_upsert_identity_candidates_are_locked( + account, + include_email=True, + locked_identities=locked_identities, + ): + await self._session.rollback() + if _identity_lock_attempt >= 1: + raise AccountIdentityRelockError( + "Account identity candidates changed during PostgreSQL slot locking" + ) + return await self._upsert_account_slot_unlocked( + account, + preserve_unknown_workspace_duplicates=preserve_unknown_workspace_duplicates, + preserve_identity_slots=preserve_identity_slots, + _identity_lock_attempt=_identity_lock_attempt + 1, + ) existing = await self._account_by_slot_identity(account) if existing: @@ -568,7 +725,17 @@ async def update_status( if blocked_at is not _UNSET: values["blocked_at"] = blocked_at result = await self._session.execute( - update(Account).where(Account.id == account_id).values(**values).returning(Account.id) + update(Account) + .where(Account.id == account_id) + # An account marked for background deletion is terminal: a + # stale in-flight settlement (e.g. a 429 from a request that + # was selected before the DELETE) must not replace the + # DEACTIVATED/pending_deletion state and make the account + # selectable again mid-drain. Only a credential replacement + # (which clears the marker) may resurrect the row. + .where(Account.delete_requested_at.is_(None)) + .values(**values) + .returning(Account.id) ) updated_id = result.scalar_one_or_none() if updated_id is not None and self._hard_sticky_outage_started(previous_status, status): @@ -584,6 +751,10 @@ async def update_security_work_authorized(self, account_id: str, enabled: bool) result = await self._session.execute( update(Account) .where(Account.id == account_id) + # Marked-for-deletion rows are gone from the operator's + # perspective: ID-based mutations must report not-found, as the + # synchronous delete did once the row was removed. + .where(Account.delete_requested_at.is_(None)) .values(security_work_authorized=enabled) .returning(Account.id) ) @@ -616,6 +787,9 @@ async def update_status_if_current( update(Account) .where(Account.id == account_id) .where(Account.status == expected_status) + # Same pending-deletion fence as ``update_status``: marked + # rows are terminal for ordinary status writers. + .where(Account.delete_requested_at.is_(None)) .values(**values) .returning(Account.id) ) @@ -750,7 +924,14 @@ async def _close_http_bridge_sessions_for_account(self, account_id: str) -> None async def update_alias(self, account_id: str, alias: str | None) -> bool: async with sqlite_writer_section(): result = await self._session.execute( - update(Account).where(Account.id == account_id).values(alias=alias).returning(Account.id) + update(Account) + .where(Account.id == account_id) + # Marked-for-deletion rows are gone from the operator's + # perspective: ID-based mutations must report not-found, as the + # synchronous delete did once the row was removed. + .where(Account.delete_requested_at.is_(None)) + .values(alias=alias) + .returning(Account.id) ) await self._session.commit() return result.scalar_one_or_none() is not None @@ -760,6 +941,10 @@ async def update_limit_warmup_enabled(self, account_id: str, enabled: bool) -> b result = await self._session.execute( update(Account) .where(Account.id == account_id) + # Marked-for-deletion rows are gone from the operator's + # perspective: ID-based mutations must report not-found, as the + # synchronous delete did once the row was removed. + .where(Account.delete_requested_at.is_(None)) .values(limit_warmup_enabled=enabled) .returning(Account.id) ) @@ -771,20 +956,211 @@ async def update_routing_policy(self, account_id: str, routing_policy: str) -> b result = await self._session.execute( update(Account) .where(Account.id == account_id) + # Marked-for-deletion rows are gone from the operator's + # perspective: ID-based mutations must report not-found, as the + # synchronous delete did once the row was removed. + .where(Account.delete_requested_at.is_(None)) .values(routing_policy=routing_policy) .returning(Account.id) ) await self._session.commit() return result.scalar_one_or_none() is not None - async def delete(self, account_id: str, *, delete_history: bool = False) -> bool: + async def begin_delete(self, account_id: str, *, delete_history: bool = False) -> bool: + """Mark an account for background deletion; commits in milliseconds. + + Fast path of ``DELETE /api/accounts/{id}``: the account becomes + terminal (``DEACTIVATED`` — every serving path already excludes it) + and carries the pending-deletion marker that hides it from listings + and enqueues it for the deletion worker, which drains its bulk rows + in chunks and finalizes via :meth:`delete` with ``only_pending=True``. + + The stored token ciphertext is overwritten with empty-credential + ciphertext in the same transaction: the row outlives the DELETE + response by the drain duration, and no reader — including a + pre-upgrade replica during a rolling deploy, whose export endpoints + do not know the marker — may still be able to produce usable + credentials from it. A credential replacement (the only supersede + path) writes fresh ciphertext, and token rotation is CAS-guarded on + the pre-wipe refresh ciphertext, so a stale in-flight rotation + misses rather than resurrecting the old material. Before the wipe, + the non-secret seat identity is preserved: legacy rows whose + ``chatgpt_user_id`` was never backfilled carry it only inside the + id-token claims, and targeted reauthentication — the promised + supersede path — verifies the seat against exactly those two + sources, so ``chatgpt_user_id`` is backfilled from the claims when + absent. + + API-key account assignments are removed here as well (the FK cascade + used to do this when the synchronous delete removed the row), so key + listings and pooled-usage projections exclude the account + immediately; the key's ``account_assignment_scope_enabled`` flag is + persisted separately and keeps the key scoped. + + Idempotent: a repeat request on an already-marked account succeeds + without changing the frozen ``delete_history`` choice (first request + wins — matching the synchronous behavior, where a second DELETE after + the first completed found nothing left to escalate). + """ + # Repeat requests short-circuit BEFORE the writer section / row lock: + # a drain chunk holds the account row (and, on SQLite, the writer + # section) for up to a few seconds, and the fast-path contract is a + # millisecond-scale response. The unlocked read is safe because the + # repeat changes nothing — the first request froze the variant and + # the wipe/cleanup already ran — and a replacement racing this read + # supersedes the deletion exactly as if it landed after this + # response. When credentials were replaced WITHOUT clearing the + # marker (a pre-upgrade replica's replacement), fall through to the + # full path so an explicit re-delete re-wipes and re-arms. + marked_row = ( + await self._session.execute( + select( + Account.delete_requested_at, + Account.access_token_encrypted, + Account.refresh_token_encrypted, + Account.id_token_encrypted, + ).where(Account.id == account_id) + ) + ).first() + if ( + marked_row is not None + and marked_row[0] is not None + and not credentials_replaced_since_wipe(marked_row[1], marked_row[2], marked_row[3]) + ): + return True + encryptor = TokenEncryptor() + wiped_token = encryptor.encrypt("") async with sqlite_writer_section(): + seat_stmt = select(Account.chatgpt_user_id, Account.id_token_encrypted).where(Account.id == account_id) + if self._dialect_name() == "postgresql": + # Hold the row through the mark so the derived seat identity + # cannot go stale between this read and the update below. + seat_stmt = seat_stmt.with_for_update(key_share=True) + seat_row = (await self._session.execute(seat_stmt)).first() + if seat_row is None: + await self._session.rollback() + return False + seat_user_id: str | None = seat_row[0] + if seat_user_id is None: + try: + claims = extract_id_token_claims(encryptor.decrypt(seat_row[1])) + seat_user_id = resolve_seat_identity(claims, claims.auth) + except Exception: + seat_user_id = None + values: dict[str, Any] = { + "status": AccountStatus.DEACTIVATED, + "deactivation_reason": ACCOUNT_PENDING_DELETION_REASON, + "reset_at": None, + "blocked_at": None, + "access_token_encrypted": wiped_token, + "refresh_token_encrypted": wiped_token, + "id_token_encrypted": wiped_token, + "delete_requested_at": func.coalesce(Account.delete_requested_at, utcnow()), + "delete_history_requested": case( + (Account.delete_requested_at.is_(None), delete_history), + else_=Account.delete_history_requested, + ), + } + if seat_user_id is not None: + values["chatgpt_user_id"] = seat_user_id + result = await self._session.execute( + update(Account).where(Account.id == account_id).values(**values).returning(Account.id) + ) + updated_id = result.scalar_one_or_none() + if updated_id is not None: + # Same immediate cleanup the DEACTIVATED transition performs: + # sticky mappings and bridge sessions must not outlive the + # account's routability. + await self._session.execute(delete(StickySession).where(StickySession.account_id == account_id)) + await self._close_http_bridge_sessions_for_account(account_id) + await self._session.execute( + delete(ApiKeyAccountAssignment).where(ApiKeyAccountAssignment.account_id == account_id) + ) + await self._session.commit() + return updated_id is not None + + async def delete( + self, + account_id: str, + *, + delete_history: bool = False, + only_pending: bool = False, + ) -> bool: + async with sqlite_writer_section(): + if self._dialect_name() == "postgresql": + # Identity membership precedes the fold-state lock so live + # settlement and deletion cannot form an identity/fold cycle. + locked_account = await self._lock_postgresql_account_identity_membership(account_id, None) + pending_state = ( + None + if locked_account is None + else ( + locked_account.delete_requested_at, + locked_account.delete_history_requested, + locked_account.access_token_encrypted, + locked_account.refresh_token_encrypted, + locked_account.id_token_encrypted, + ) + ) + else: + pending_state = ( + await self._session.execute( + select( + Account.delete_requested_at, + Account.delete_history_requested, + Account.access_token_encrypted, + Account.refresh_token_encrypted, + Account.id_token_encrypted, + ).where(Account.id == account_id) + ) + ).first() + if only_pending: + # Background finalization: a credential replacement + # (re-import/reauth) that cleared the marker supersedes the + # deletion, so touch nothing. The variant comes from the + # persisted flag frozen at request time, never the caller. + # On PostgreSQL the identity-membership row lock held above + # keeps the marker stable through this transaction; on SQLite + # the writer section serializes all writers. + if pending_state is None or pending_state[0] is None: + await self._session.rollback() + return False + if credentials_replaced_since_wipe(pending_state[2], pending_state[3], pending_state[4]): + # A pre-upgrade replica replaced the credentials without + # being able to clear marker columns its ORM does not + # know. That replacement supersedes the deletion: clear + # the marker (we hold the row lock) and abandon. + await self._session.execute( + update(Account) + .where(Account.id == account_id) + .values(delete_requested_at=None, delete_history_requested=False) + ) + await self._session.commit() + return False + delete_history = bool(pending_state[1]) # Serialize against fold passes before touching the account's # request logs: without the fold-state lock an in-flight hourly # slice could aggregate the pre-delete attribution but commit # after this transaction, resurrecting the account's folded rows # the mirrors below just moved or removed. await lock_fold_state(self._session) + if self._dialect_name() == "postgresql": + # Upgrade the account row to a full FOR UPDATE lock BEFORE the + # raw sweeps. FOR UPDATE conflicts with the KEY SHARE taken by + # concurrent request-log FK inserts, so every in-flight + # stream's log row either commits before this point (and the + # sweeps below see it) or its insert blocks until this + # transaction commits and then fails its FK against the + # deleted row — the same outcome a post-delete insert always + # had. Without the upgrade, an insert could commit between + # the sweep and the account-row delete: the FK's ON DELETE + # SET NULL would leave a live (deleted_at IS NULL) orphan on + # the soft path, or surviving raw history under + # delete_history. Lock order (identity -> fold -> row + # exclusive) matches the historical transaction, where the + # final DELETE acquired this same exclusive lock after the + # fold lock. + await self._session.execute(select(Account.id).where(Account.id == account_id).with_for_update()) await self._session.execute(delete(UsageHistory).where(UsageHistory.account_id == account_id)) if delete_history: await self._session.execute(delete(RequestLog).where(RequestLog.account_id == account_id)) @@ -809,6 +1185,10 @@ async def delete(self, account_id: str, *, delete_history: bool = False) -> bool await self._session.commit() if deleted_id is not None: _clear_bulk_history_since_sqlite_cache() + # Deletion drops the account's rollup row and detaches or + # deletes its request logs; cached listing summaries would + # keep reporting the account until TTL expiry. + _clear_request_usage_summary_cache() return deleted_id is not None async def rotate_tokens( @@ -843,6 +1223,8 @@ async def rotate_tokens( material at all). """ async with sqlite_writer_section(): + if self._dialect_name() == "postgresql": + await self._lock_postgresql_account_identity_membership(account_id, chatgpt_account_id) values: dict[str, bytes | datetime | str] = { "access_token_encrypted": access_token_encrypted, "refresh_token_encrypted": refresh_token_encrypted, @@ -901,6 +1283,8 @@ async def update_account_metadata( no-op existence check. """ async with sqlite_writer_section(): + if self._dialect_name() == "postgresql": + await self._lock_postgresql_account_identity_membership(account_id, chatgpt_account_id) values: dict[str, str | datetime] = {} if plan_type is not None: values["plan_type"] = plan_type @@ -1047,6 +1431,75 @@ async def _account_by_slot_identity(self, account: Account) -> Account | None: return matched return None + async def _lock_postgresql_account_identity_membership( + self, + account_id: str, + incoming_chatgpt_account_id: str | None, + *, + second_attempt: bool = False, + ) -> Account | None: + """Lock one row's old/new upstream memberships before mutating it.""" + observed_identity = await self._session.scalar( + select(Account.chatgpt_account_id).where(Account.id == account_id) + ) + await lock_postgresql_account_identities( + self._session, + (observed_identity, incoming_chatgpt_account_id), + ) + locked_account = await self._session.scalar( + select(Account) + .where(Account.id == account_id) + # PostgreSQL FOR NO KEY UPDATE stabilizes identity membership but + # remains compatible with the KEY SHARE lock taken by concurrent + # rollup FK inserts. Deletion upgrades only after the fold lock. + .with_for_update(key_share=True) + .execution_options(populate_existing=True) + ) + locked_identity = locked_account.chatgpt_account_id if locked_account is not None else None + if locked_identity == observed_identity: + return locked_account + await self._session.rollback() + if second_attempt: + raise AccountIdentityRelockError("Account identity changed during PostgreSQL membership lock acquisition") + return await self._lock_postgresql_account_identity_membership( + account_id, + incoming_chatgpt_account_id, + second_attempt=True, + ) + + async def _lock_postgresql_upsert_identity_candidates( + self, + account: Account, + *, + include_email: bool, + ) -> frozenset[str]: + predicates = _upsert_identity_candidate_predicates(account, include_email=include_email) + observed = ( + (await self._session.execute(select(Account.chatgpt_account_id).where(or_(*predicates)))).scalars().all() + ) + identities = frozenset(identity for identity in (*observed, account.chatgpt_account_id) if identity) + await lock_postgresql_account_identities(self._session, identities) + return identities + + async def _postgresql_upsert_identity_candidates_are_locked( + self, + account: Account, + *, + include_email: bool, + locked_identities: frozenset[str], + ) -> bool: + predicates = _upsert_identity_candidate_predicates(account, include_email=include_email) + current = ( + ( + await self._session.execute( + select(Account.chatgpt_account_id).where(or_(*predicates)).with_for_update(key_share=True) + ) + ) + .scalars() + .all() + ) + return all(identity is None or identity in locked_identities for identity in current) + def _dialect_name(self) -> str: return self._session.get_bind().dialect.name @@ -1062,14 +1515,14 @@ async def _acquire_sqlite_merge_lock(self) -> None: await self._session.execute(text("UPDATE accounts SET id = id WHERE 1 = 0")) async def _acquire_postgresql_merge_lock(self, email: str) -> None: - lock_key = _advisory_lock_key("merge-email", email) + lock_key = advisory_lock_key("merge-email", email) await self._session.execute( text("SELECT pg_advisory_xact_lock(:lock_key)"), {"lock_key": lock_key}, ) async def _acquire_postgresql_identity_lock(self, account_id: str) -> None: - lock_key = _advisory_lock_key("account-id", account_id) + lock_key = advisory_lock_key("account-id", account_id) await self._session.execute( text("SELECT pg_advisory_xact_lock(:lock_key)"), {"lock_key": lock_key}, @@ -1097,6 +1550,12 @@ def _apply_account_updates(target: Account, source: Account) -> None: target.deactivation_reason = source.deactivation_reason target.reset_at = source.reset_at target.blocked_at = source.blocked_at + # A credential replacement (re-import/reauth) supersedes a pending + # background deletion: clearing the marker makes the deletion worker + # abandon the account before finalizing (rows already drained stay + # detached — history loss was requested by the earlier delete). + target.delete_requested_at = None + target.delete_history_requested = False def _slot_lock_key(account: Account, *, preserve_unknown_workspace_duplicates: bool = True) -> str: @@ -1125,6 +1584,15 @@ def _slot_lock_keys(account: Account, *, preserve_unknown_workspace_duplicates: return (f"slot-local:{account.id}",) +def _upsert_identity_candidate_predicates(account: Account, *, include_email: bool) -> list[Any]: + predicates = [Account.id == account.id] + if account.chatgpt_account_id: + predicates.append(Account.chatgpt_account_id == account.chatgpt_account_id) + if include_email and account.email: + predicates.append(Account.email == account.email) + return predicates + + def _same_unknown_workspace_identity(existing: Account, incoming: Account) -> bool: return ( _workspace_slot_key(existing) is None @@ -1176,8 +1644,3 @@ def _can_reuse_email_fallback(existing: Account, incoming: Account) -> bool: or not existing.chatgpt_account_id or existing.chatgpt_account_id == incoming.chatgpt_account_id ) - - -def _advisory_lock_key(scope: str, value: str) -> int: - digest = hashlib.sha256(f"{scope}:{value}".encode("utf-8")).digest() - return int.from_bytes(digest[:8], byteorder="big", signed=True) diff --git a/app/modules/accounts/service.py b/app/modules/accounts/service.py index f6c53d7309..71f0b19de7 100644 --- a/app/modules/accounts/service.py +++ b/app/modules/accounts/service.py @@ -40,6 +40,7 @@ from app.db.models import Account, AccountStatus, DashboardSettings from app.db.session import get_background_session from app.modules.accounts.auth_manager import AuthManager +from app.modules.accounts.deletion import request_account_deletion_run from app.modules.accounts.mappers import build_account_summaries, build_account_usage_trends from app.modules.accounts.repository import AccountsRepository from app.modules.accounts.schemas import ( @@ -233,7 +234,7 @@ async def list_accounts(self, *, account_ids: list[str] | None = None) -> list[A ) async def get_account_trends(self, account_id: str) -> AccountTrendsResponse | None: - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if not account or not self._usage_repo: return None now = utcnow() @@ -255,7 +256,7 @@ async def get_account_trends(self, account_id: str) -> AccountTrendsResponse | N ) async def get_usage_reset_credits(self, account_id: str) -> AccountUsageResetCreditsResponse | None: - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if account is None: return None if account.status in (AccountStatus.PAUSED, AccountStatus.REAUTH_REQUIRED, AccountStatus.DEACTIVATED): @@ -318,7 +319,7 @@ async def consume_usage_reset_credit( *, redeem_request_id: str | None = None, ) -> AccountUsageResetConsumeResponse | None: - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if account is None: return None if account.status in (AccountStatus.PAUSED, AccountStatus.REAUTH_REQUIRED, AccountStatus.DEACTIVATED): @@ -423,8 +424,19 @@ async def _resolve_usage_reset_credit_route( encryptor=self._encryptor, ) - async def export_opencode_auth(self, account_id: str) -> AccountOpenCodeAuthExportResponse | None: + async def _get_visible_account(self, account_id: str) -> Account | None: + """Account fetch for ID-based operator routes; marked-for-deletion + rows are gone from the operator's perspective (the synchronous delete + returned 404 on every one of these routes once the row was removed) + and MUST NOT keep serving reads, mutations, or decrypted tokens + during the background drain window.""" account = await self._repo.get_by_id(account_id) + if account is None or account.delete_requested_at is not None: + return None + return account + + async def export_opencode_auth(self, account_id: str) -> AccountOpenCodeAuthExportResponse | None: + account = await self._get_visible_account(account_id) if account is None: return None @@ -449,7 +461,7 @@ async def export_opencode_auth(self, account_id: str) -> AccountOpenCodeAuthExpo ) async def export_auth(self, account_id: str) -> AccountAuthExportResponse | None: - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if account is None: return None @@ -592,6 +604,11 @@ async def reactivate_account(self, account_id: str) -> bool: account = await self._repo.get_by_id(account_id) if account is None: return False + if account.delete_requested_at is not None: + # Marked for background deletion: already invisible in listings + # and about to be removed — report it as gone rather than racing + # the deletion worker back to ACTIVE. + return False if account.status == AccountStatus.REAUTH_REQUIRED: raise AccountStateTransitionError("Account requires re-authentication and cannot be reactivated directly") result = await self._repo.update_status_if_current( @@ -614,7 +631,7 @@ async def reactivate_account(self, account_id: str) -> bool: return result async def pause_account(self, account_id: str) -> bool: - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if account is None: return False if account.status in (AccountStatus.REAUTH_REQUIRED, AccountStatus.DEACTIVATED): @@ -659,19 +676,25 @@ async def set_routing_policy(self, account_id: str, routing_policy: str) -> bool return result async def delete_account(self, account_id: str, *, delete_history: bool = False) -> bool: - result = await self._repo.delete(account_id, delete_history=delete_history) + # Fast path: stamp the pending-deletion marker (terminal status, hidden + # from listings, sticky/bridge cleanup) and return in milliseconds; the + # background deletion worker drains the bulk rows and removes the + # account row afterwards (see app.modules.accounts.deletion). + result = await self._repo.begin_delete(account_id, delete_history=delete_history) if result: mark_account_routing_unavailable(account_id) get_account_selection_cache().invalidate() get_api_key_cache().clear() - # Deletion cascades the account_proxy_bindings row away, and account - # ids are deterministic (delete-then-re-import regenerates the same - # id), so the cached route outcome must not survive the deletion. + # Finalization cascades the account_proxy_bindings row away, and + # account ids are deterministic (delete-then-re-import regenerates + # the same id), so the cached route outcome must not survive the + # delete request; the worker invalidates again after finalizing. await get_upstream_route_cache().invalidate() await propagate_account_routing_change() poller = get_cache_invalidation_poller() if poller is not None: await poller.bump(NAMESPACE_API_KEY) + request_account_deletion_run() return result async def set_account_alias(self, account_id: str, alias: str | None) -> bool: @@ -681,7 +704,7 @@ async def set_account_alias(self, account_id: str, alias: str | None) -> bool: return await self._repo.update_alias(account_id, normalized) async def export_account(self, account_id: str) -> AccountExportResponse | None: - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if not account: return None access_token = self._encryptor.decrypt(account.access_token_encrypted) @@ -722,7 +745,7 @@ async def probe_account( before/after snapshot so the operator can see whether the upstream state changed. """ - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if account is None: return None if account.status in (AccountStatus.PAUSED, AccountStatus.REAUTH_REQUIRED, AccountStatus.DEACTIVATED): diff --git a/app/modules/accounts/usage_rollup.py b/app/modules/accounts/usage_rollup.py index 9a4cd95912..e8c7da70c5 100644 --- a/app/modules/accounts/usage_rollup.py +++ b/app/modules/accounts/usage_rollup.py @@ -18,13 +18,34 @@ # Rows younger than the lag stay on the live side of the fold boundary. # The lag MUST exceed the maximum possible distance between a log row's -# requested_at and its actual insertion time: requested_at is the request -# START, but the row is written at stream END, so a long-running stream -# inserts a row dated its full duration in the past — if that lands below an -# already-advanced watermark it is neither folded nor in the live tail and -# vanishes from totals. 24h dwarfs any survivable stream duration and the -# post-stream duplicate/model/cost rewrite paths (which settle in seconds). -FOLD_LAG = timedelta(hours=24) +# requested_at and the moment its insert becomes visible: a row landing below +# an already-advanced watermark is neither folded nor in the live tail and +# vanishes from totals. Every insert path stamps requested_at inside +# ``RequestLogsRepository.add_log`` at write time (``requested_at or +# utcnow()``; no live caller passes an explicit value — the parameter exists +# for tests), so the distance is only clock skew between replicas plus the +# single-row insert transaction's commit latency, NOT the request duration: +# a stream-end log write is dated at the write, not at the request start. +# Measured over one full production history (6.0M rows) the worst insert +# landed 7.9s below the requested_at frontier (p99.9 = 30ms). Post-insert +# mutators need no allowance here either: ``update_model_for_request`` +# skips rows at/below the watermarks, and consolidation/deletion reassign +# logs under the fold-state lock while mirroring the folded sums. +# +# 2h keeps a ~900x margin over the observed worst case (covering scheduler +# stalls, NTP drift, paused VMs) while bounding the raw tail every account +# and API-key summary read must dedupe and re-aggregate; the previous 24h +# lag — sized for a stream-start dating scheme this codebase never actually +# had — left an always-rescanned tail of ~660k rows (11% of the table) on +# the reference deployment and degraded the listing aggregate to a full +# seq-scan hash join (measured 60s cold / 2.1s warm vs 63ms at 2h). +# +# This lag is a shared visibility contract: the hourly/conversation fold +# targets and the retention min-gate (``now - 2 * FOLD_LAG`` freshness, +# ``watermark - FOLD_LAG`` prune floor) derive from the same constant. +# A fold pass after an upgrade from the 24h lag absorbs the watermark jump +# as one ordinary backfill slice (FOLD_SLICE bounds it). +FOLD_LAG = timedelta(hours=2) # Historical backfill folds at most this much history per transaction. FOLD_SLICE = timedelta(days=7) diff --git a/app/modules/api_keys/api.py b/app/modules/api_keys/api.py index 9f8fc122c6..09323c2874 100644 --- a/app/modules/api_keys/api.py +++ b/app/modules/api_keys/api.py @@ -46,6 +46,7 @@ def _to_response(row: ApiKeyData) -> ApiKeyResponse: apply_to_codex_model=row.apply_to_codex_model, enforced_model=row.enforced_model, enforced_reasoning_effort=row.enforced_reasoning_effort, + allowed_reasoning_efforts=row.allowed_reasoning_efforts, enforced_service_tier=row.enforced_service_tier, traffic_class=row.traffic_class, transport_policy_override=row.transport_policy_override, @@ -135,6 +136,7 @@ async def create_api_key( apply_to_codex_model=payload.apply_to_codex_model, enforced_model=payload.enforced_model, enforced_reasoning_effort=payload.enforced_reasoning_effort, + allowed_reasoning_efforts=payload.allowed_reasoning_efforts, enforced_service_tier=payload.enforced_service_tier, traffic_class=payload.traffic_class or "foreground", transport_policy_override=payload.transport_policy_override, @@ -195,6 +197,8 @@ async def update_api_key( enforced_model_set="enforced_model" in fields, enforced_reasoning_effort=payload.enforced_reasoning_effort, enforced_reasoning_effort_set="enforced_reasoning_effort" in fields, + allowed_reasoning_efforts=payload.allowed_reasoning_efforts, + allowed_reasoning_efforts_set="allowed_reasoning_efforts" in fields, enforced_service_tier=payload.enforced_service_tier, enforced_service_tier_set="enforced_service_tier" in fields, traffic_class=payload.traffic_class, diff --git a/app/modules/api_keys/repository.py b/app/modules/api_keys/repository.py index 63bbdec207..02cdea8de2 100644 --- a/app/modules/api_keys/repository.py +++ b/app/modules/api_keys/repository.py @@ -6,9 +6,9 @@ from enum import Enum from typing import Any -from sqlalchemy import BigInteger, Integer, cast, delete, func, or_, select, true, update +from sqlalchemy import BigInteger, Integer, cast, delete, func, insert, literal, or_, select, true, update from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import load_only, selectinload +from sqlalchemy.orm import load_only, raiseload, selectinload from app.core.utils.time import utcnow from app.db.models import ( @@ -168,6 +168,46 @@ async def get_by_id(self, key_id: str) -> ApiKey | None: result = await self._session.execute(self._select_api_key().where(ApiKey.id == key_id)) return result.scalar_one_or_none() + async def get_for_limit_enforcement(self, key_id: str) -> ApiKey | None: + """Admission-path load for ``enforce_limits_for_request``. + + The enforcement transaction reads only ``is_active``/``expires_at`` + plus the ``limits`` collection, so this skips the + ``account_assignments``/``source_assignments`` selectin round trips + that ``get_by_id`` pays on every proxied request. ``raiseload`` keeps + the narrowing fail-loud: any future enforcement code that touches an + unlisted column or relationship raises instead of silently lazy + loading. ``populate_existing`` stays required because the lazy limit + reset commits mid-enforcement and the refetch must re-hydrate rows + already in the identity map (sessions use ``expire_on_commit=False``). + + Session-isolation invariant: ``populate_existing`` + ``raiseload`` + would poison a *fully loaded* ``ApiKey`` already in this session's + identity map — re-populating it flips its unlisted columns and + relationships into raise-on-access state for every other holder of + that instance. That is unreachable today because every caller runs + this query in a dedicated short-lived session that never full-loads + an ``ApiKey`` first (``_enforce_request_limits`` and the websocket + reservation path open fresh background sessions/repo bundles; the + quota-planner warmup session never loads ``ApiKey`` rows), and the + only prior instance this query can re-populate is the one it loaded + itself with these same options. Do not call this on a session that + may already hold a fully loaded ``ApiKey`` (e.g. via ``get_by_id`` / + ``get_by_hash``) without dropping the narrowing first. + """ + result = await self._session.execute( + select(ApiKey) + .execution_options(populate_existing=True) + .options( + load_only(ApiKey.is_active, ApiKey.expires_at, raiseload=True), + selectinload(ApiKey.limits), + raiseload(ApiKey.account_assignments), + raiseload(ApiKey.source_assignments), + ) + .where(ApiKey.id == key_id) + ) + return result.scalar_one_or_none() + async def get_by_hash(self, key_hash: str) -> ApiKey | None: result = await self._session.execute(self._select_api_key().where(ApiKey.key_hash == key_hash)) return result.scalar_one_or_none() @@ -183,6 +223,11 @@ async def list_accounts_by_ids(self, account_ids: list[str]) -> list[Account]: select(Account) .options(load_only(Account.id, Account.plan_type, Account.status)) .where(Account.id.in_(account_ids)) + # An account marked for background deletion is already deleted + # from the operator's point of view: assignment validation must + # reject it (the synchronous delete removed the row outright) and + # pooled-usage projections must not count it while its rows drain. + .where(Account.delete_requested_at.is_(None)) ) return list(result.scalars().all()) @@ -199,6 +244,11 @@ async def list_all_accounts(self) -> list[Account]: .where( ~Account.status.in_((AccountStatus.REAUTH_REQUIRED, AccountStatus.DEACTIVATED, AccountStatus.PAUSED)) ) + # Status alone is not enough: an unfenced pre-upgrade replica can + # briefly replace a marked account's terminal status during a + # rolling deploy, and a deleted account must never re-enter the + # unscoped pooled-usage projections. + .where(Account.delete_requested_at.is_(None)) ) return list(result.scalars().all()) @@ -304,6 +354,7 @@ async def update( apply_to_codex_model: bool | _Unset = _UNSET, enforced_model: str | None | _Unset = _UNSET, enforced_reasoning_effort: str | None | _Unset = _UNSET, + allowed_reasoning_efforts: str | None | _Unset = _UNSET, enforced_service_tier: str | None | _Unset = _UNSET, traffic_class: str | _Unset = _UNSET, transport_policy_override: str | None | _Unset = _UNSET, @@ -334,6 +385,9 @@ async def update( if enforced_reasoning_effort is not _UNSET: assert enforced_reasoning_effort is None or isinstance(enforced_reasoning_effort, str) row.enforced_reasoning_effort = enforced_reasoning_effort + if allowed_reasoning_efforts is not _UNSET: + assert allowed_reasoning_efforts is None or isinstance(allowed_reasoning_efforts, str) + row.allowed_reasoning_efforts = allowed_reasoning_efforts if enforced_service_tier is not _UNSET: assert enforced_service_tier is None or isinstance(enforced_service_tier, str) row.enforced_service_tier = enforced_service_tier @@ -436,9 +490,33 @@ async def upsert_limits(self, key_id: str, limits: list[ApiKeyLimit], *, commit: return await self.get_limits_by_key(key_id) async def replace_account_assignments(self, key_id: str, account_ids: list[str], *, commit: bool = True) -> None: + # Re-check the pending-deletion marker atomically with the write: + # validation ran in an earlier transaction, and an account DELETE can + # commit in between — the marked row still exists (background drain), + # so a plain FK insert would succeed and resurrect an assignment + # begin_delete just removed. The FOR SHARE lock (PostgreSQL) + # conflicts with begin_delete's row update, so either this + # transaction commits first (and begin_delete's assignment cleanup + # removes its rows) or the marker is visible below and the account is + # skipped. The account locks are taken BEFORE the assignment-row + # delete to match begin_delete's order (account row, then assignment + # rows) — taking them after would form a lock cycle with a + # concurrent begin_delete and deadlock. SQLite serializes writers, + # so the marker predicate alone is race-free there. + if account_ids and self._session.get_bind().dialect.name == "postgresql": + await self._session.execute( + select(Account.id).where(Account.id.in_(account_ids)).with_for_update(read=True) + ) await self._session.execute(delete(ApiKeyAccountAssignment).where(ApiKeyAccountAssignment.api_key_id == key_id)) - for account_id in account_ids: - self._session.add(ApiKeyAccountAssignment(api_key_id=key_id, account_id=account_id)) + if account_ids: + assignment_source = ( + select(literal(key_id), Account.id) + .where(Account.id.in_(account_ids)) + .where(Account.delete_requested_at.is_(None)) + ) + await self._session.execute( + insert(ApiKeyAccountAssignment).from_select(["api_key_id", "account_id"], assignment_source) + ) if commit: await self._session.commit() parent = await self._session.get(ApiKey, key_id) diff --git a/app/modules/api_keys/schemas.py b/app/modules/api_keys/schemas.py index a8616aa1bc..10aa0dfe10 100644 --- a/app/modules/api_keys/schemas.py +++ b/app/modules/api_keys/schemas.py @@ -32,7 +32,8 @@ class ApiKeyCreateRequest(DashboardModel): enforced_reasoning_effort: str | None = Field( default=None, pattern=r"(?i)^(none|minimal|low|medium|high|xhigh|max|ultra)$" ) - enforced_service_tier: str | None = Field(default=None, pattern=r"(?i)^(auto|default|priority|flex|fast)$") + allowed_reasoning_efforts: list[str] | None = None + enforced_service_tier: str | None = Field(default=None, pattern=r"(?i)^(auto|default|priority|flex|(ultra)?fast)$") traffic_class: str | None = Field(default=None, pattern=r"(?i)^(foreground|opportunistic)$") transport_policy_override: str | None = None usage_sections: str | None = None @@ -51,7 +52,8 @@ class ApiKeyUpdateRequest(DashboardModel): enforced_reasoning_effort: str | None = Field( default=None, pattern=r"(?i)^(none|minimal|low|medium|high|xhigh|max|ultra)$" ) - enforced_service_tier: str | None = Field(default=None, pattern=r"(?i)^(auto|default|priority|flex|fast)$") + allowed_reasoning_efforts: list[str] | None = None + enforced_service_tier: str | None = Field(default=None, pattern=r"(?i)^(auto|default|priority|flex|(ultra)?fast)$") traffic_class: str | None = Field(default=None, pattern=r"(?i)^(foreground|opportunistic)$") transport_policy_override: str | None = None usage_sections: str | None = None @@ -79,6 +81,7 @@ class ApiKeyResponse(DashboardModel): apply_to_codex_model: bool = False enforced_model: str | None enforced_reasoning_effort: str | None + allowed_reasoning_efforts: list[str] | None enforced_service_tier: str | None traffic_class: str transport_policy_override: str | None = None diff --git a/app/modules/api_keys/service.py b/app/modules/api_keys/service.py index 096b274841..a68bce8db0 100644 --- a/app/modules/api_keys/service.py +++ b/app/modules/api_keys/service.py @@ -11,7 +11,7 @@ from math import ceil from typing import Protocol -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import IntegrityError, OperationalError from app.core.auth.api_key_cache import get_api_key_cache from app.core.cache.invalidation import NAMESPACE_API_KEY, get_cache_invalidation_poller @@ -51,6 +51,7 @@ TRAFFIC_CLASS_OPPORTUNISTIC = "opportunistic" _SUPPORTED_TRAFFIC_CLASSES = frozenset({TRAFFIC_CLASS_FOREGROUND, TRAFFIC_CLASS_OPPORTUNISTIC}) _SUPPORTED_TRANSPORT_POLICY_OVERRIDES = frozenset({"smart", "always_http", "always_websocket"}) +_REASONING_POLICY_EXCLUSIVE_CONSTRAINT = "ck_api_keys_reasoning_policy_exclusive" class ApiKeysRepositoryProtocol(Protocol): @@ -58,6 +59,8 @@ async def create(self, row: ApiKey, *, commit: bool = True) -> ApiKey: ... async def get_by_id(self, key_id: str) -> ApiKey | None: ... + async def get_for_limit_enforcement(self, key_id: str) -> ApiKey | None: ... + async def get_by_hash(self, key_hash: str) -> ApiKey | None: ... async def list_all(self) -> list[ApiKey]: ... @@ -85,6 +88,7 @@ async def update( apply_to_codex_model: bool | _Unset = ..., enforced_model: str | None | _Unset = ..., enforced_reasoning_effort: str | None | _Unset = ..., + allowed_reasoning_efforts: str | None | _Unset = ..., enforced_service_tier: str | None | _Unset = ..., traffic_class: str | _Unset = ..., transport_policy_override: str | None | _Unset = ..., @@ -270,6 +274,7 @@ class ApiKeyCreateData: apply_to_codex_model: bool = False enforced_model: str | None = None enforced_reasoning_effort: str | None = None + allowed_reasoning_efforts: list[str] | None = None enforced_service_tier: str | None = None traffic_class: str = TRAFFIC_CLASS_FOREGROUND transport_policy_override: str | None = None @@ -292,6 +297,8 @@ class ApiKeyUpdateData: enforced_model_set: bool = False enforced_reasoning_effort: str | None = None enforced_reasoning_effort_set: bool = False + allowed_reasoning_efforts: list[str] | None = None + allowed_reasoning_efforts_set: bool = False enforced_service_tier: str | None = None enforced_service_tier_set: bool = False traffic_class: str | None = None @@ -326,6 +333,7 @@ class ApiKeyData: is_active: bool created_at: datetime last_used_at: datetime | None + allowed_reasoning_efforts: list[str] | None = None apply_to_codex_model: bool = False traffic_class: str = TRAFFIC_CLASS_FOREGROUND transport_policy_override: str | None = None @@ -468,11 +476,16 @@ async def create_key(self, payload: ApiKeyCreateData) -> ApiKeyCreatedData: assigned_source_ids = await self._resolve_assigned_source_ids(payload.assigned_source_ids) enforced_model = _normalize_model_slug(payload.enforced_model) enforced_reasoning_effort = _normalize_reasoning_effort(payload.enforced_reasoning_effort) + allowed_reasoning_efforts = _normalize_allowed_reasoning_efforts(payload.allowed_reasoning_efforts) enforced_service_tier = _normalize_service_tier(payload.enforced_service_tier) traffic_class = _normalize_traffic_class(payload.traffic_class) transport_policy_override = _normalize_transport_policy_override(payload.transport_policy_override) usage_sections = _normalize_usage_sections(payload.usage_sections) _validate_model_enforcement(enforced_model=enforced_model, allowed_models=normalized_allowed_models) + _validate_reasoning_effort_policy( + enforced_reasoning_effort=enforced_reasoning_effort, + allowed_reasoning_efforts=allowed_reasoning_efforts, + ) row = ApiKey( id=str(__import__("uuid").uuid4()), name=_normalize_name(payload.name), @@ -482,6 +495,7 @@ async def create_key(self, payload: ApiKeyCreateData) -> ApiKeyCreatedData: apply_to_codex_model=bool(payload.apply_to_codex_model), enforced_model=enforced_model, enforced_reasoning_effort=enforced_reasoning_effort, + allowed_reasoning_efforts=_serialize_allowed_reasoning_efforts(allowed_reasoning_efforts), enforced_service_tier=enforced_service_tier, account_assignment_scope_enabled=bool(assigned_account_ids), source_assignment_scope_enabled=bool(assigned_source_ids), @@ -506,8 +520,12 @@ async def create_key(self, payload: ApiKeyCreateData) -> ApiKeyCreatedData: await self._repository.upsert_limits(created.id, limit_rows, commit=False) await self._repository.commit() - except Exception: + except Exception as exc: await self._repository.rollback() + if isinstance(exc, IntegrityError) and _is_reasoning_policy_constraint_error(exc): + raise ApiKeyValidationError( + "enforced_reasoning_effort and allowed_reasoning_efforts cannot be configured together" + ) from exc raise created = await self._repository.get_by_id(created.id) @@ -607,6 +625,11 @@ async def update_key(self, key_id: str, payload: ApiKeyUpdateData) -> ApiKeyData else: enforced_reasoning_effort = None + if payload.allowed_reasoning_efforts_set: + allowed_reasoning_efforts = _normalize_allowed_reasoning_efforts(payload.allowed_reasoning_efforts) + else: + allowed_reasoning_efforts = None + if payload.enforced_service_tier_set: enforced_service_tier = _normalize_service_tier(payload.enforced_service_tier) else: @@ -634,6 +657,22 @@ async def update_key(self, key_id: str, payload: ApiKeyUpdateData) -> ApiKeyData allowed_models=effective_allowed_models, ) + if payload.enforced_reasoning_effort_set or payload.allowed_reasoning_efforts_set: + effective_enforced_reasoning_effort = ( + enforced_reasoning_effort + if payload.enforced_reasoning_effort_set + else _normalize_reasoning_effort_lenient(existing.enforced_reasoning_effort) + ) + effective_allowed_reasoning_efforts = ( + allowed_reasoning_efforts + if payload.allowed_reasoning_efforts_set + else _deserialize_allowed_reasoning_efforts(existing.allowed_reasoning_efforts) + ) + _validate_reasoning_effort_policy( + enforced_reasoning_effort=effective_enforced_reasoning_effort, + allowed_reasoning_efforts=effective_allowed_reasoning_efforts, + ) + limit_rows: list[ApiKeyLimit] | None = None if payload.limits_set: now = utcnow() @@ -662,6 +701,11 @@ async def update_key(self, key_id: str, payload: ApiKeyUpdateData) -> ApiKeyData enforced_reasoning_effort=( enforced_reasoning_effort if payload.enforced_reasoning_effort_set else _UNSET ), + allowed_reasoning_efforts=( + _serialize_allowed_reasoning_efforts(allowed_reasoning_efforts) + if payload.allowed_reasoning_efforts_set + else _UNSET + ), enforced_service_tier=(enforced_service_tier if payload.enforced_service_tier_set else _UNSET), traffic_class=traffic_class_update, transport_policy_override=transport_policy_override_update, @@ -686,8 +730,12 @@ async def update_key(self, key_id: str, payload: ApiKeyUpdateData) -> ApiKeyData await self._repository.upsert_limits(key_id, limit_rows, commit=False) await self._repository.commit() - except Exception: + except Exception as exc: await self._repository.rollback() + if isinstance(exc, IntegrityError) and _is_reasoning_policy_constraint_error(exc): + raise ApiKeyValidationError( + "enforced_reasoning_effort and allowed_reasoning_efforts cannot be configured together" + ) from exc raise if ( @@ -699,6 +747,7 @@ async def update_key(self, key_id: str, payload: ApiKeyUpdateData) -> ApiKeyData or payload.apply_to_codex_model_set or payload.enforced_model_set or payload.enforced_reasoning_effort_set + or payload.allowed_reasoning_efforts_set or payload.enforced_service_tier_set or payload.traffic_class_set or payload.transport_policy_override_set @@ -800,7 +849,7 @@ async def enforce_limits_for_request( request_model: str | None, request_service_tier: str | None = None, request_usage_budget: ApiKeyRequestUsageBudget | None = None, - ) -> ApiKeyUsageReservationData: + ) -> ApiKeyUsageReservationData | None: for attempt in range(_SQLITE_BUSY_RETRY_ATTEMPTS): try: return await self._enforce_limits_for_request_once( @@ -824,18 +873,23 @@ async def _enforce_limits_for_request_once( request_model: str | None, request_service_tier: str | None, request_usage_budget: ApiKeyRequestUsageBudget | None, - ) -> ApiKeyUsageReservationData: + ) -> ApiKeyUsageReservationData | None: now = utcnow() async with sqlite_writer_section(): - row = _ensure_valid_api_key_row(await self._repository.get_by_id(key_id)) + row = _ensure_valid_api_key_row(await self._repository.get_for_limit_enforcement(key_id)) if row.expires_at is not None and row.expires_at < now: raise ApiKeyInvalidError("API key has expired") limits_reset = await _lazy_reset_expired_limits(self._repository, row.limits, now=now) - refreshed = _ensure_valid_api_key_row(await self._repository.get_by_id(key_id)) if limits_reset else row + refreshed = ( + _ensure_valid_api_key_row(await self._repository.get_for_limit_enforcement(key_id)) + if limits_reset + else row + ) if refreshed.expires_at is not None and refreshed.expires_at < now: raise ApiKeyInvalidError("API key has expired") reservation_items: list[UsageReservationItemData] = [] + reservation_id: str | None = None normalized_usage_budget = _normalize_request_usage_budget(request_usage_budget) try: for limit in refreshed.limits: @@ -866,18 +920,55 @@ async def _enforce_limits_for_request_once( ) ) - reservation_id = _next_usage_reservation_id() - await self._repository.create_usage_reservation( - reservation_id, - key_id=key_id, - model=request_model or "", - items=reservation_items, - ) - await self._repository.commit() + if not reservation_items: + # No configured limit applies to this request, so there is + # nothing to reserve and nothing to settle. Skip the empty + # reservation INSERT and its full-durability commit (the + # lazy expired-limit reset above commits inside + # ``reset_limit`` itself, so no write is pending here). + # Every downstream consumer treats a missing reservation + # as "nothing to settle". Commit to close the implicit + # transaction opened by the admission SELECTs: on sessions + # that outlive this call (e.g. the quota-planner warmup + # service) an open transaction would otherwise idle across + # the upstream round-trip until the next commit. This must + # be ``commit()`` rather than ``rollback()``: + # ``AsyncSession.rollback()`` expires every tracked ORM + # object regardless of ``expire_on_commit``, and the warmup + # service shares this session with already-loaded + # ``account``/``decision`` rows whose attribute access + # after expiry raises ``MissingGreenlet``. ``commit()`` + # with ``expire_on_commit=False`` (app/db/session.py) + # leaves tracked state loaded, and is semantically + # equivalent here because the transaction holds only the + # admission SELECTs — no reservation write ran, and every + # production caller either dedicates a session to + # admission (proxy paths) or commits each prior write + # inside its repository methods (quota-planner), so no + # unrelated dirty state can be flushed by this commit. + await self._repository.commit() + else: + reservation_id = _next_usage_reservation_id() + await self._repository.create_usage_reservation( + reservation_id, + key_id=key_id, + model=request_model or "", + items=reservation_items, + ) + await self._repository.commit() except Exception: await self._repository.rollback() raise + if reservation_id is None: + # Settlement is the only other production writer of + # ``last_used_at``; without a reservation it never runs, so record + # the last-used touch at admission instead. Recorded outside + # sqlite_writer_section() for the same reason as settlement: the + # shutdown write-through flush takes the writer section itself. + await self._last_used_coalescer.record(key_id, utcnow()) + return None + return ApiKeyUsageReservationData( reservation_id=reservation_id, key_id=key_id, @@ -1292,6 +1383,12 @@ def _serialize_allowed_models(allowed_models: list[str] | None) -> str | None: return json.dumps(allowed_models) +def _serialize_allowed_reasoning_efforts(allowed_reasoning_efforts: list[str] | None) -> str | None: + if allowed_reasoning_efforts is None: + return None + return json.dumps(allowed_reasoning_efforts) + + def _deserialize_allowed_models(payload: str | None) -> list[str] | None: if payload is None: return None @@ -1302,6 +1399,22 @@ def _deserialize_allowed_models(payload: str | None) -> list[str] | None: return models +def _deserialize_allowed_reasoning_efforts(payload: str | None) -> list[str] | None: + if payload is None: + return None + try: + parsed = json.loads(payload) + if not isinstance(parsed, list): + return [] + return _normalize_allowed_reasoning_efforts(parsed) + except (ApiKeyValidationError, TypeError, json.JSONDecodeError): + return [] + + +def _is_reasoning_policy_constraint_error(exc: IntegrityError) -> bool: + return _REASONING_POLICY_EXCLUSIVE_CONSTRAINT in str(exc).lower() + + def _normalize_allowed_models(allowed_models: list[str] | None) -> list[str] | None: if allowed_models is None: return None @@ -1345,8 +1458,10 @@ def _normalize_model_slug(value: str | None) -> str | None: return normalized -_SUPPORTED_REASONING_EFFORTS = frozenset({"none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"}) -_SUPPORTED_SERVICE_TIERS = frozenset({"auto", "default", "priority", "flex"}) +_REASONING_EFFORT_ORDER = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra") +_SUPPORTED_REASONING_EFFORTS = frozenset({"none", *_REASONING_EFFORT_ORDER}) +_SUPPORTED_SELECTABLE_REASONING_EFFORTS = frozenset(_REASONING_EFFORT_ORDER) +_SUPPORTED_SERVICE_TIERS = frozenset({"auto", "default", "priority", "flex", "ultrafast"}) def _normalize_expires_at(value: datetime | None) -> datetime | None: @@ -1378,6 +1493,25 @@ def _normalize_reasoning_effort_lenient(value: str | None) -> str | None: return None +def _normalize_allowed_reasoning_efforts(values: list[str] | None) -> list[str] | None: + if values is None: + return None + + normalized: set[str] = set() + for value in values: + if not isinstance(value, str): + raise ApiKeyValidationError("Allowed reasoning efforts must be strings") + effort = value.strip().lower() + if effort not in _SUPPORTED_SELECTABLE_REASONING_EFFORTS: + options = ", ".join(_REASONING_EFFORT_ORDER) + raise ApiKeyValidationError(f"Unsupported allowed reasoning effort '{effort}'. Expected one of: {options}") + normalized.add(effort) + + if not normalized: + raise ApiKeyValidationError("Allowed reasoning efforts must not be empty") + return [effort for effort in _REASONING_EFFORT_ORDER if effort in normalized] + + def _normalize_service_tier(value: str | None) -> str | None: if value is None: return None @@ -1448,6 +1582,17 @@ def _validate_model_enforcement(*, enforced_model: str | None, allowed_models: l ) +def _validate_reasoning_effort_policy( + *, + enforced_reasoning_effort: str | None, + allowed_reasoning_efforts: list[str] | None, +) -> None: + if enforced_reasoning_effort is not None and allowed_reasoning_efforts is not None: + raise ApiKeyValidationError( + "enforced_reasoning_effort and allowed_reasoning_efforts cannot be configured together" + ) + + def _to_limit_rule_data(limit: ApiKeyLimit) -> LimitRuleData: return LimitRuleData( id=limit.id, @@ -1639,6 +1784,7 @@ def _to_created_data(data: ApiKeyData, key: str) -> ApiKeyCreatedData: apply_to_codex_model=data.apply_to_codex_model, enforced_model=data.enforced_model, enforced_reasoning_effort=data.enforced_reasoning_effort, + allowed_reasoning_efforts=data.allowed_reasoning_efforts, enforced_service_tier=data.enforced_service_tier, traffic_class=data.traffic_class, transport_policy_override=data.transport_policy_override, @@ -1674,6 +1820,9 @@ def _to_api_key_data( apply_to_codex_model=getattr(row, "apply_to_codex_model", False), enforced_model=_normalize_model_slug(row.enforced_model), enforced_reasoning_effort=_normalize_reasoning_effort_lenient(row.enforced_reasoning_effort), + allowed_reasoning_efforts=_deserialize_allowed_reasoning_efforts( + getattr(row, "allowed_reasoning_efforts", None) + ), enforced_service_tier=_normalize_service_tier_lenient(row.enforced_service_tier), traffic_class=_normalize_traffic_class_lenient(getattr(row, "traffic_class", TRAFFIC_CLASS_FOREGROUND)), transport_policy_override=_normalize_transport_policy_override_lenient( diff --git a/app/modules/dashboard/repository.py b/app/modules/dashboard/repository.py index 3fe42f6172..9d4ca0d7e4 100644 --- a/app/modules/dashboard/repository.py +++ b/app/modules/dashboard/repository.py @@ -51,8 +51,17 @@ async def bulk_usage_history_since( since: datetime, *, cutoffs: dict[str, datetime] | None = None, + per_account_row_cap: int | None = None, + uncapped_recent_floor: datetime | None = None, ) -> dict[str, list[UsageHistorySnapshot]]: - return await self._usage_repo.bulk_history_since(account_ids, window, since, cutoffs=cutoffs) + return await self._usage_repo.bulk_history_since( + account_ids, + window, + since, + cutoffs=cutoffs, + per_account_row_cap=per_account_row_cap, + uncapped_recent_floor=uncapped_recent_floor, + ) async def latest_window_minutes(self, window: str) -> int | None: return await self._usage_repo.latest_window_minutes(window) diff --git a/app/modules/dashboard/service.py b/app/modules/dashboard/service.py index eb6026769e..606aebb7ca 100644 --- a/app/modules/dashboard/service.py +++ b/app/modules/dashboard/service.py @@ -39,6 +39,22 @@ ) from app.modules.usage.mappers import usage_history_to_window_row +# Newest-first per-account row bound for the projections history fetch +# (PostgreSQL; the SQLite snapshot cache keeps the shared floor). Live +# snapshot ingestion appends usage rows per proxied request, so one busy +# account's 7-day secondary window can hold tens of thousands of rows while +# the consumers only read the recent tail. The cap alone covers the +# tail-weighted consumers: the EWMA depletion/burn rates (alpha 0.4 — a +# sample's contribution decays by 0.6^n within a few dozen newer samples) +# are insensitive to samples this deep regardless of write cadence. The one +# equal-weight consumer — the weekly-pace smoothing mean over the configured +# window (<= 240 minutes) — is protected by ``uncapped_recent_floor`` +# instead, because ingestion writes on every fingerprint change and a burst +# could out-write any fixed cap inside the smoothing window. 4320 rows cover +# the 6-hour recent-burn window at the ingestor's 5-second per-account write +# throttle floor; sparse accounts stay under the cap entirely. +_PROJECTION_HISTORY_PER_ACCOUNT_ROW_CAP = 4320 + def _parse_weekly_pace_working_days(value: str) -> set[int]: try: @@ -181,15 +197,16 @@ async def get_projections(self) -> DashboardProjectionsResponse: encryptor=self._encryptor, include_auth=False, ) + dashboard_settings = await self._repo.get_settings() primary_history, secondary_history = await _load_projection_histories( self._repo, primary_usage, secondary_usage, now, + smoothing_window_minutes=dashboard_settings.weekly_pace_smoothing_minutes, ) pri_depletion, sec_depletion = _build_depletion_by_window(primary_history, secondary_history, now) settings = get_settings() - dashboard_settings = await self._repo.get_settings() weekly_credit_pace = build_weekly_credit_pace( accounts=accounts, account_summaries=account_summaries, @@ -211,6 +228,8 @@ async def _load_projection_histories( primary_usage: dict[str, UsageHistory], secondary_usage: dict[str, UsageHistory], now: datetime, + *, + smoothing_window_minutes: int, ) -> tuple[dict[str, list[UsageHistory]], dict[str, list[UsageHistory]]]: # Compute depletion separately for primary-window and secondary-window # accounts so the aggregate is not skewed by mixing different window durations. @@ -279,13 +298,32 @@ async def _load_projection_histories( if acct_since < sec_since: sec_since = acct_since + # The weekly-pace smoothing mean weighs every sample in its window + # equally, so rows inside the configured smoothing window are exempt from + # the row cap (ingestion writes per fingerprint change; a burst could + # otherwise out-write the cap and silently shift the smoothed values). + smoothing_floor = now - timedelta(minutes=smoothing_window_minutes) all_pri_rows = ( - await repo.bulk_usage_history_since(pri_fetch_ids, "primary", pri_since, cutoffs=pri_cutoffs) + await repo.bulk_usage_history_since( + pri_fetch_ids, + "primary", + pri_since, + cutoffs=pri_cutoffs, + per_account_row_cap=_PROJECTION_HISTORY_PER_ACCOUNT_ROW_CAP, + uncapped_recent_floor=smoothing_floor, + ) if pri_fetch_ids else {} ) all_sec_rows = ( - await repo.bulk_usage_history_since(sec_fetch_ids, "secondary", sec_since, cutoffs=sec_cutoffs) + await repo.bulk_usage_history_since( + sec_fetch_ids, + "secondary", + sec_since, + cutoffs=sec_cutoffs, + per_account_row_cap=_PROJECTION_HISTORY_PER_ACCOUNT_ROW_CAP, + uncapped_recent_floor=smoothing_floor, + ) if sec_fetch_ids else {} ) diff --git a/app/modules/limit_warmup/service.py b/app/modules/limit_warmup/service.py index af966281d4..f490f96b3b 100644 --- a/app/modules/limit_warmup/service.py +++ b/app/modules/limit_warmup/service.py @@ -16,7 +16,7 @@ from app.core.openai.models import OpenAIError, ResponseUsage from app.core.openai.parsing import parse_sse_event from app.core.openai.requests import ResponsesRequest -from app.core.plan_types import account_plan_matches_allowed +from app.core.plan_types import account_plan_matches_allowed, normalize_account_plan_type from app.core.upstream_proxy import ResolvedUpstreamRoute, UpstreamProxyRouteError, resolve_upstream_route from app.core.usage.pricing import get_pricing_for_model from app.core.utils.time import naive_utc_to_epoch, utcnow @@ -41,9 +41,9 @@ # Minimum reset_at forward jump (in seconds) to confirm a real quota window reset. # Upstream timestamp jitter of ~1 second must not trigger a warm-up. _RESET_CONFIRMED_MIN_JUMP_SECONDS = 60 -# Persist the upstream value, but treat nearby values as the same staggered-idle -# cycle. This avoids every boundary inherent in stateless timestamp bucketing. -_IDLE_RESET_AT_JITTER_TOLERANCE_SECONDS = 5 +# Persist the upstream value, but treat nearby values as the same reset. This +# avoids duplicate attempts when reset_at jitters between refresh cycles. +_RESET_AT_JITTER_TOLERANCE_SECONDS = 5 @dataclass(frozen=True, slots=True) @@ -163,6 +163,24 @@ async def send(self, account: Account, *, model: str, prompt: str) -> LimitWarmu try: async with self._auth_lock: fresh_account = await self._ensure_fresh(account) + if ( + fresh_account is None + or not _account_is_safe_candidate(fresh_account) + or not fresh_account.limit_warmup_enabled + ): + if fresh_account is None: + error_message = "Account no longer exists" + elif not fresh_account.limit_warmup_enabled: + error_message = "Limit warm-up is disabled for this account" + else: + error_message = f"Account status is {fresh_account.status.value}" + return LimitWarmupSendResult( + request_id=request_id, + success=False, + latency_ms=_elapsed_ms(started), + error_code="account_not_active", + error_message=error_message, + ) access_token = self._encryptor.decrypt(fresh_account.access_token_encrypted) chatgpt_account_id = fresh_account.chatgpt_account_id except RefreshError as exc: @@ -174,14 +192,6 @@ async def send(self, account: Account, *, model: str, prompt: str) -> LimitWarmu error_message=exc.message, ) - if fresh_account.status != AccountStatus.ACTIVE: - return LimitWarmupSendResult( - request_id=request_id, - success=False, - latency_ms=_elapsed_ms(started), - error_code="account_not_active", - error_message=f"Account status is {fresh_account.status.value}", - ) try: route = await self._resolve_upstream_route(fresh_account) except UpstreamProxyRouteError as exc: @@ -273,14 +283,23 @@ async def send(self, account: Account, *, model: str, prompt: str) -> LimitWarmu upstream_proxy_fallback_used=route_trace.fallback_used, ) - async def _ensure_fresh(self, account: Account) -> Account: + async def _ensure_fresh(self, account: Account) -> Account | None: if self._accounts_repo_factory is None: - return await self._auth_manager.ensure_fresh(account) + current = await self._accounts_repo.get_by_id_fresh(account.id) + if current is None or not _account_is_safe_candidate(current) or not current.limit_warmup_enabled: + return current + await self._auth_manager.ensure_fresh(current) + return await self._accounts_repo.get_by_id_fresh(account.id) async with self._accounts_repo_factory() as accounts_repo: - return await AuthManager( + current = await accounts_repo.get_by_id_fresh(account.id) + if current is None or not _account_is_safe_candidate(current) or not current.limit_warmup_enabled: + return current + await AuthManager( accounts_repo, refresh_repo_factory=self._accounts_repo_factory, - ).ensure_fresh(account) + ).ensure_fresh(current) + async with self._accounts_repo_factory() as accounts_repo: + return await accounts_repo.get_by_id_fresh(account.id) async def _resolve_upstream_route(self, account: Account) -> ResolvedUpstreamRoute | None: if self._accounts_repo_factory is not None: @@ -323,6 +342,7 @@ async def run_after_usage_refresh( before_secondary: dict[str, UsageHistory], after_primary: dict[str, UsageHistory], after_secondary: dict[str, UsageHistory], + previous_plan_types: dict[str, str | None] | None = None, refresh_started_at: datetime | None = None, usage_refresh_interval_seconds: int = _STAGGER_SLOT_GRACE_SECONDS, ) -> None: @@ -353,11 +373,6 @@ async def run_after_usage_refresh( if not account.limit_warmup_enabled: continue latest_attempt = latest_attempts.get(account.id) - if _in_cooldown( - latest_attempt, - cooldown_seconds=settings.limit_warmup_cooldown_seconds, - ): - continue windows_to_evaluate = list(selected_windows) if settings.limit_warmup_staggered_idle_enabled and "primary" not in windows_to_evaluate: @@ -372,10 +387,22 @@ async def run_after_usage_refresh( before_secondary=before_secondary, after_primary=after_primary, after_secondary=after_secondary, - exhausted_threshold_percent=settings.limit_warmup_exhausted_threshold_percent, min_available_percent=settings.limit_warmup_min_available_percent, ) - if candidate is None and settings.limit_warmup_staggered_idle_enabled and window == "primary": + if candidate is None and window == "secondary": + candidate = _build_paid_to_free_transition_candidate( + account=account, + previous_plan_type=(previous_plan_types or {}).get(account.id), + after_secondary=after_secondary, + refresh_started_at=refresh_started_at, + min_available_percent=settings.limit_warmup_min_available_percent, + ) + if ( + candidate is None + and _account_is_safe_candidate(account) + and settings.limit_warmup_staggered_idle_enabled + and window == "primary" + ): candidate = _build_staggered_idle_candidate( account=account, accounts=staggered_accounts, @@ -387,6 +414,11 @@ async def run_after_usage_refresh( ) if candidate is None: continue + if candidate.window == _IDLE_PRIMARY_WINDOW and _in_cooldown( + latest_attempt, + cooldown_seconds=settings.limit_warmup_cooldown_seconds, + ): + continue model = self._resolve_model(settings.limit_warmup_model, account) if model is None: @@ -667,7 +699,6 @@ def _build_candidate( before_secondary: dict[str, UsageHistory], after_primary: dict[str, UsageHistory], after_secondary: dict[str, UsageHistory], - exhausted_threshold_percent: float, min_available_percent: float, ) -> _WarmupCandidate | None: before = _effective_usage_entry( @@ -684,24 +715,71 @@ def _build_candidate( ) if before is None or after is None: return None - if before.reset_at is None or after.reset_at is None: + available_percent = 100.0 - after.used_percent + if min_available_percent < 100.0 and available_percent < min_available_percent: + return None + if not usage_reset_confirmed(before=before, after=after): return None + assert after.reset_at is not None + candidate_window = "monthly" if after.window == "monthly" else window + return _WarmupCandidate(reset_at=after.reset_at, window=candidate_window) + + +def usage_reset_confirmed(*, before: UsageHistory | None, after: UsageHistory | None) -> bool: + """Return whether consecutive samples prove a real, newly available quota window.""" + if before is None or after is None: + return False + if before.reset_at is None or after.reset_at is None: + return False if (before.window or "primary") != (after.window or "primary"): + return False + if after.used_percent >= 100.0: + return False + reset_at_jump = after.reset_at - before.reset_at + if reset_at_jump < _RESET_CONFIRMED_MIN_JUMP_SECONDS: + return False + before_observed_at = naive_utc_to_epoch(before.recorded_at) + observed_at = naive_utc_to_epoch(after.recorded_at) + window_started_at = after.reset_at - _rolling_window_seconds(after) + quota_recovered = after.used_percent < before.used_percent + crossed_previous_reset = before_observed_at <= before.reset_at <= observed_at < after.reset_at + reanchored_between_samples = ( + quota_recovered and before_observed_at <= window_started_at <= observed_at < after.reset_at + ) + # Scheduled resets cross the previous boundary. Early resets can happen + # when upstream restores quota and reanchors a complete window from the + # sampling interval. A reset_at update outside that interval is not a reset. + if not crossed_previous_reset and not reanchored_between_samples: + return False + return True + + +def _build_paid_to_free_transition_candidate( + *, + account: Account, + previous_plan_type: str | None, + after_secondary: dict[str, UsageHistory], + refresh_started_at: datetime | None, + min_available_percent: float, +) -> _WarmupCandidate | None: + normalized_previous_plan = normalize_account_plan_type(previous_plan_type) + if normalized_previous_plan is None or normalized_previous_plan == "free": return None - if before.used_percent < exhausted_threshold_percent: + if normalize_account_plan_type(account.plan_type) != "free": + return None + if refresh_started_at is None: + return None + after = after_secondary.get(account.id) + if after is None or after.window != "monthly" or after.reset_at is None: + return None + if after.recorded_at < refresh_started_at: return None if after.used_percent >= 100.0: return None available_percent = 100.0 - after.used_percent if min_available_percent < 100.0 and available_percent < min_available_percent: return None - # Require a meaningful reset_at forward jump (not just upstream timestamp jitter). - # Upstream can report reset_at values that fluctuate by ~1 second between - # refresh cycles; only treat a jump of at least 60 seconds as a real reset. - if after.reset_at - before.reset_at < _RESET_CONFIRMED_MIN_JUMP_SECONDS: - return None - candidate_window = "monthly" if after.window == "monthly" else window - return _WarmupCandidate(reset_at=after.reset_at, window=candidate_window) + return _WarmupCandidate(reset_at=after.reset_at, window="monthly") def _build_staggered_idle_candidate( @@ -867,6 +945,4 @@ def _truncate(value: str | None, limit: int = 1000) -> str | None: def _attempt_reset_at_tolerance(candidate: _WarmupCandidate) -> int: - if candidate.window == _IDLE_PRIMARY_WINDOW: - return _IDLE_RESET_AT_JITTER_TOLERANCE_SECONDS - return 0 + return _RESET_AT_JITTER_TOLERANCE_SECONDS diff --git a/app/modules/model_sources/catalog.py b/app/modules/model_sources/catalog.py index da7b25bbaa..39f846e25a 100644 --- a/app/modules/model_sources/catalog.py +++ b/app/modules/model_sources/catalog.py @@ -4,6 +4,7 @@ from app.core.openai.model_registry import ( MODEL_SOURCE_KIND_OPENAI_COMPATIBLE, + ReasoningLevel, UpstreamModel, ) from app.core.types import JsonValue @@ -58,15 +59,26 @@ def _to_upstream_model(source: ModelSource, source_model: ModelSourceModel) -> U input_modalities = ("text", "image") if source_model.supports_vision else ("text",) display_name = source_model.display_name or source_model.model + # The dashboard's single Reasoning switch is the master gate: it is the + # only reasoning control an operator has in the UI, so a model with it off + # must not advertise efforts it will never be allowed to use. Keeping the + # switch authoritative is what lets the Codex catalog, /v1/models and the + # dashboard checkbox agree; deriving levels regardless would advertise a + # capability the chat sanitizer then strips. + reasoning_opted_in = raw.get("supports_reasoning") is True + reasoning_levels = _reasoning_levels_from_metadata(raw) if reasoning_opted_in else () + default_reasoning_level = ( + _default_reasoning_level_from_metadata(raw, reasoning_levels) if reasoning_opted_in else None + ) return UpstreamModel( slug=source_model.model, display_name=display_name, description=display_name, context_window=context_window, input_modalities=input_modalities, - supported_reasoning_levels=(), - default_reasoning_level=None, - supports_reasoning_summaries=False, + supported_reasoning_levels=reasoning_levels, + default_reasoning_level=default_reasoning_level, + supports_reasoning_summaries=reasoning_opted_in and raw.get("supports_reasoning_summaries") is True, support_verbosity=False, default_verbosity=None, prefer_websockets=False, @@ -81,18 +93,104 @@ def _to_upstream_model(source: ModelSource, source_model: ModelSourceModel) -> U ) -def source_model_supports_reasoning(source: ModelSource, model: str) -> bool: - """Whether the source model opted into reasoning via raw catalog metadata. +def _reasoning_levels_from_metadata(raw: dict[str, JsonValue]) -> tuple[ReasoningLevel, ...]: + """Reasoning efforts advertised for a source model. + + Source catalogs have no first-class reasoning schema, so operators declare + the efforts their backend accepts under ``supported_reasoning_levels`` in + ``raw_metadata_json``. Both shapes are accepted:: + + ["low", "high", "max"] + [{"effort": "low", "description": "Low reasoning effort"}] - Source catalog entries have no first-class reasoning flag; a model that - genuinely supports reasoning can opt in with ``"supports_reasoning": true`` - in ``raw_metadata_json``. Everything else is treated as non-reasoning so - client-sent reasoning toggles are stripped before forwarding. + Efforts are normalized (trimmed and lowercased) and deduplicated. + Validation is on shape, not on membership of a fixed vocabulary: backends + disagree on which efforts exist (GLM exposes ``none``, Model Studio + includes it, others stop at ``low``/``high``/``max``), so an enum here + would silently drop efforts a provider really accepts. Malformed entries -- + a non-string, a mapping without a string ``effort``, an empty slug -- are + ignored, keeping the previous no-reasoning default for models that never + opted in. """ - entry = next( + declared = raw.get("supported_reasoning_levels") + if not is_json_list(declared): + return () + levels: list[ReasoningLevel] = [] + seen: set[str] = set() + for item in declared: + if isinstance(item, str): + effort = item + description = f"{item.strip().lower()} reasoning effort" + elif is_json_mapping(item): + effort_value = item.get("effort") + if not isinstance(effort_value, str): + continue + effort = effort_value + description_value = item.get("description") + description = ( + description_value + if isinstance(description_value, str) + else f"{effort.strip().lower()} reasoning effort" + ) + else: + continue + effort = effort.strip().lower() + if not effort or effort in seen: + continue + seen.add(effort) + levels.append(ReasoningLevel(effort=effort, description=description)) + return tuple(levels) + + +def _default_reasoning_level_from_metadata( + raw: dict[str, JsonValue], + levels: tuple[ReasoningLevel, ...], +) -> str | None: + """Operator-declared default effort, restricted to the advertised levels.""" + declared = raw.get("default_reasoning_level") + if not isinstance(declared, str): + return None + normalized = declared.strip().lower() + if not any(level.effort == normalized for level in levels): + return None + return normalized + + +def _enabled_source_model(source: ModelSource, model: str) -> ModelSourceModel | None: + return next( (candidate for candidate in source.models if candidate.model == model and candidate.is_enabled), None, ) + + +def source_model_reasoning_levels(source: ModelSource, model: str) -> tuple[ReasoningLevel, ...]: + """Reasoning efforts an opted-in source model declared. + + Gated on ``supports_reasoning`` like the catalog derivation, so the + unsupported-effort restore cannot hand a declared effort to a model whose + operator left the Reasoning switch off. + """ + entry = _enabled_source_model(source, model) + if entry is None: + return () + raw = _raw_metadata(entry) + if raw.get("supports_reasoning") is not True: + return () + return _reasoning_levels_from_metadata(raw) + + +def source_model_supports_reasoning(source: ModelSource, model: str) -> bool: + """Whether the operator turned the model's Reasoning switch on. + + ``"supports_reasoning": true`` in ``raw_metadata_json`` is the single + opt-in, written by the dashboard's Reasoning checkbox. Declared levels do + not imply it: they describe *which* efforts an opted-in backend accepts, + not *whether* reasoning is allowed at all, and the catalog derivation is + gated on this same flag so the two can never disagree. Everything else is + treated as non-reasoning, so client-sent reasoning toggles are stripped + before forwarding on the chat path. + """ + entry = _enabled_source_model(source, model) if entry is None: return False return _raw_metadata(entry).get("supports_reasoning") is True diff --git a/app/modules/model_sources/forwarding.py b/app/modules/model_sources/forwarding.py index cd30a7bbc0..19dd4895f9 100644 --- a/app/modules/model_sources/forwarding.py +++ b/app/modules/model_sources/forwarding.py @@ -1,7 +1,8 @@ from __future__ import annotations +import asyncio import json -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Awaitable, Mapping from contextlib import AsyncExitStack from dataclasses import dataclass from json import JSONDecodeError @@ -9,6 +10,7 @@ from typing import cast import aiohttp +import anyio from app.core.clients.http import lease_http_session from app.core.crypto import TokenEncryptor @@ -81,6 +83,13 @@ class SourceAudioTranscription: upstream_status_code: int +@dataclass(frozen=True, slots=True) +class SourceEmbeddings: + payload: dict[str, JsonValue] + usage: SourceUsage | None + upstream_status_code: int + + @dataclass(frozen=True, slots=True) class SourceChatStream: body: AsyncIterator[bytes] @@ -101,42 +110,85 @@ class SourceUsageHolder: timings: SourceTimings | None = None +async def _await_cleanup_deferring_cancellation(awaitable: Awaitable[object]) -> None: + """Finish owned upstream cleanup even if the caller is cancelled again.""" + + task = asyncio.ensure_future(awaitable) + with anyio.CancelScope(shield=True): + while True: + try: + await asyncio.shield(task) + return + except asyncio.CancelledError: + if task.cancelled(): + raise + + +async def _await_result_deferring_cancellation(awaitable: Awaitable[object]) -> bool: + """Finish owned cleanup and report whether cancellation arrived mid-flight.""" + + task = asyncio.ensure_future(awaitable) + cancellation_deferred = False + with anyio.CancelScope(shield=True): + while True: + try: + await asyncio.shield(task) + return cancellation_deferred + except asyncio.CancelledError: + if task.cancelled(): + raise + cancellation_deferred = True + raise RuntimeError("unreachable shielded cancellation-deferral state") + + async def forward_chat_completion( source: ModelSource, payload: dict[str, JsonValue], *, encryptor: TokenEncryptor | None = None, ) -> SourceChatCompletion: + stack = AsyncExitStack() try: - async with lease_http_session() as session: - timeout = aiohttp.ClientTimeout(total=_source_timeout_seconds(source)) - async with session.post( + session = await stack.enter_async_context(lease_http_session()) + timeout = aiohttp.ClientTimeout(total=_source_timeout_seconds(source)) + response = await stack.enter_async_context( + session.post( _source_url(source, "/chat/completions"), headers=_source_headers(source, encryptor=encryptor), json=payload, timeout=timeout, - ) as response: - data = await _response_json(response) - if response.status >= 400: - raise ModelSourceForwardingError( - status_code=response.status, - payload=_redact_source_error_payload( - _error_payload(data), - source, - encryptor=encryptor, - ), - upstream_status_code=response.status, - ) - if data is None: - raise _invalid_upstream_response_error(response.status) - return SourceChatCompletion( - payload=data, - usage=_usage_from_chat_payload(data), - timings=_timings_from_payload(data), - upstream_status_code=response.status, - ) + ) + ) + data = await _response_json(response) + if response.status >= 400: + raise ModelSourceForwardingError( + status_code=response.status, + payload=_redact_source_error_payload( + _error_payload(data), + source, + encryptor=encryptor, + ), + upstream_status_code=response.status, + ) + if data is None: + raise _invalid_upstream_response_error(response.status) + result = SourceChatCompletion( + payload=data, + usage=_usage_from_chat_payload(data), + timings=_timings_from_payload(data), + upstream_status_code=response.status, + ) except (aiohttp.ClientError, TimeoutError) as exc: + await _await_cleanup_deferring_cancellation(stack.aclose()) raise _unreachable_error(exc) from exc + except BaseException: + await _await_cleanup_deferring_cancellation(stack.aclose()) + raise + + cleanup_cancelled = await _await_result_deferring_cancellation(stack.aclose()) + if cleanup_cancelled: + raise asyncio.CancelledError + return result async def stream_chat_completion( @@ -150,10 +202,15 @@ async def stream_chat_completion( stack, response = await _open_source_stream(source, "/chat/completions", payload, encryptor=encryptor) async def body() -> AsyncIterator[bytes]: - async with stack: + try: async for chunk in response.content.iter_chunked(4096): usage_parser.feed(chunk) yield chunk + finally: + # A plain ``async with stack`` unwinds unshielded: repeated + # cancellation delivery can interrupt ``__aexit__`` mid-unwind and + # leak the pooled HTTP session lease. + await _await_cleanup_deferring_cancellation(stack.aclose()) return SourceChatStream(body=body(), usage_holder=usage_holder, upstream_status_code=response.status) @@ -249,6 +306,43 @@ async def forward_audio_transcription( raise _unreachable_error(exc) from exc +async def forward_embeddings( + source: ModelSource, + payload: dict[str, JsonValue], + *, + encryptor: TokenEncryptor | None = None, +) -> SourceEmbeddings: + try: + async with lease_http_session() as session: + timeout = aiohttp.ClientTimeout(total=_source_timeout_seconds(source)) + async with session.post( + _source_url(source, "/embeddings"), + headers=_source_headers(source, encryptor=encryptor), + json=payload, + timeout=timeout, + ) as response: + data = await _response_json(response) + if response.status >= 400: + raise ModelSourceForwardingError( + status_code=response.status, + payload=_redact_source_error_payload( + _error_payload(data), + source, + encryptor=encryptor, + ), + upstream_status_code=response.status, + ) + if data is None: + raise _invalid_upstream_response_error(response.status) + return SourceEmbeddings( + payload=data, + usage=_usage_from_embeddings_payload(data), + upstream_status_code=response.status, + ) + except (aiohttp.ClientError, TimeoutError) as exc: + raise _unreachable_error(exc) from exc + + async def stream_responses( source: ModelSource, payload: dict[str, JsonValue], @@ -260,10 +354,15 @@ async def stream_responses( stack, response = await _open_source_stream(source, "/responses", payload, encryptor=encryptor) async def body() -> AsyncIterator[bytes]: - async with stack: + try: async for chunk in response.content.iter_chunked(4096): usage_parser.feed(chunk) yield chunk + finally: + # A plain ``async with stack`` unwinds unshielded: repeated + # cancellation delivery can interrupt ``__aexit__`` mid-unwind and + # leak the pooled HTTP session lease. + await _await_cleanup_deferring_cancellation(stack.aclose()) return SourceResponsesStream(body=body(), usage_holder=usage_holder, upstream_status_code=response.status) @@ -309,10 +408,10 @@ async def _open_source_stream( ) return stack, response except (aiohttp.ClientError, TimeoutError) as exc: - await stack.aclose() + await _await_cleanup_deferring_cancellation(stack.aclose()) raise _unreachable_error(exc) from exc except BaseException: - await stack.aclose() + await _await_cleanup_deferring_cancellation(stack.aclose()) raise @@ -483,6 +582,14 @@ def _usage_from_responses_payload(payload: Mapping[str, JsonValue]) -> SourceUsa return _usage_from_responses_mapping(usage) +def _usage_from_embeddings_payload(payload: Mapping[str, JsonValue]) -> SourceUsage | None: + """Embeddings responses report prompt/total tokens and no completion tokens.""" + usage = payload.get("usage") + if not is_json_mapping(usage): + return None + return _usage_from_mapping(usage) or _usage_from_total_tokens_mapping(usage) + + def _usage_from_audio_body(body: bytes, content_type: str | None) -> SourceUsage | None: if not _is_json_content_type(content_type): return None diff --git a/app/modules/model_sources/repository.py b/app/modules/model_sources/repository.py index 38fb42238b..19d39febcd 100644 --- a/app/modules/model_sources/repository.py +++ b/app/modules/model_sources/repository.py @@ -113,6 +113,31 @@ async def find_audio_transcriptions_source_for_model( result = await self._session.execute(stmt) return result.scalar_one_or_none() + async def find_embeddings_source_for_model( + self, + model: str, + *, + allowed_source_ids: set[str] | None = None, + ) -> ModelSource | None: + stmt = ( + select(ModelSource) + .options(selectinload(ModelSource.models)) + .join(ModelSourceModel, ModelSourceModel.source_id == ModelSource.id) + .where(ModelSource.kind == "openai_compatible") + .where(ModelSource.is_enabled.is_(True)) + .where(ModelSource.supports_embeddings.is_(True)) + .where(ModelSourceModel.model == model) + .where(ModelSourceModel.is_enabled.is_(True)) + .order_by(ModelSource.name, ModelSource.id) + .limit(1) + ) + if allowed_source_ids is not None: + if not allowed_source_ids: + return None + stmt = stmt.where(ModelSource.id.in_(allowed_source_ids)) + result = await self._session.execute(stmt) + return result.scalar_one_or_none() + async def create(self, row: ModelSource, *, commit: bool = True) -> ModelSource: self._session.add(row) if commit: diff --git a/app/modules/model_sources/schemas.py b/app/modules/model_sources/schemas.py index b556c4d91e..1643f40191 100644 --- a/app/modules/model_sources/schemas.py +++ b/app/modules/model_sources/schemas.py @@ -37,6 +37,7 @@ class ModelSourceCreateRequest(DashboardModel): supports_chat_completions: bool = True supports_responses: bool = False supports_audio_transcriptions: bool = False + supports_embeddings: bool = False timeout_seconds: int | None = Field(default=None, ge=1) max_concurrency: int | None = Field(default=None, ge=1) models: list[ModelSourceModelInput] = Field(default_factory=list) @@ -50,6 +51,7 @@ class ModelSourceUpdateRequest(DashboardModel): supports_chat_completions: bool | None = None supports_responses: bool | None = None supports_audio_transcriptions: bool | None = None + supports_embeddings: bool | None = None timeout_seconds: int | None = Field(default=None, ge=1) max_concurrency: int | None = Field(default=None, ge=1) models: list[ModelSourceModelInput] | None = None @@ -65,6 +67,7 @@ class ModelSourceResponse(DashboardModel): supports_chat_completions: bool supports_responses: bool supports_audio_transcriptions: bool + supports_embeddings: bool timeout_seconds: int | None max_concurrency: int | None created_at: datetime diff --git a/app/modules/model_sources/selection.py b/app/modules/model_sources/selection.py new file mode 100644 index 0000000000..b81837d83e --- /dev/null +++ b/app/modules/model_sources/selection.py @@ -0,0 +1,128 @@ +"""Shared model-source selection helpers. + +Both the HTTP request handlers and the WebSocket session path need to decide +whether a requested model is served by an OpenAI-compatible model source. +Keeping that decision in one module stops the two transports from drifting +apart: previously only the HTTP handlers consulted model sources, so a +source-owned model requested over WebSocket fell through to subscription +account selection and was rejected upstream. +""" + +from __future__ import annotations + +import logging + +from app.core.openai.model_registry import get_model_registry +from app.db.models import ModelSource +from app.db.session import detach_session_objects, get_background_session +from app.modules.api_keys.service import ApiKeyData +from app.modules.model_sources.repository import ModelSourcesRepository + +logger = logging.getLogger(__name__) + + +def allowed_source_ids_for_api_key(api_key: ApiKeyData | None) -> set[str] | None: + """Source ids an API key may use, or ``None`` when scoping is disabled.""" + if api_key is None or not api_key.source_assignment_scope_enabled: + return None + return set(api_key.assigned_source_ids) + + +async def select_responses_model_source( + model: str, + api_key: ApiKeyData | None, + *, + raw_model: str | None = None, + require_streaming: bool = False, +) -> tuple[ModelSource, str] | None: + """Resolve ``model`` to a Responses-capable model source, if any.""" + assigned_source_ids = allowed_source_ids_for_api_key(api_key) + exact_allowed_models = set(api_key.allowed_models) if api_key and api_key.allowed_models else None + candidates = [candidate for candidate in (raw_model, model) if candidate] + if not candidates: + return None + deduped_candidates = list(dict.fromkeys(candidates)) + registry_models = get_model_registry().get_models_with_fallback() + async with get_background_session() as session: + repository = ModelSourcesRepository(session) + for candidate in deduped_candidates: + if exact_allowed_models is not None and candidate not in exact_allowed_models: + continue + subscription_model = registry_models.get(candidate) + if assigned_source_ids is None and subscription_model is not None: + continue + source = await repository.find_responses_source_for_model( + candidate, + allowed_source_ids=assigned_source_ids, + require_streaming=require_streaming, + ) + if source is not None: + break + else: + source = None + # ``close_session`` rolls back the read transaction, which would + # expire the loaded row; detach it so the forwarding path can read + # its attributes after this session boundary. + detach_session_objects(session) + return (source, candidate) if source is not None else None + + +def effective_model_for_api_key(api_key: ApiKeyData | None, requested_model: str | None) -> str | None: + """The model an API key forces, falling back to the requested one.""" + if api_key is None or api_key.enforced_model is None: + return requested_model + return api_key.enforced_model + + +async def responses_model_is_source_owned( + model: str | None, + api_key: ApiKeyData | None, + *, + raw_model: str | None = None, +) -> bool: + """True when ``model`` is served by an enabled Responses-capable source. + + Used by the WebSocket path, which cannot forward to a model source and must + fail the session so the client falls back to the HTTP transport. + + The API key's ``enforced_model`` is considered alongside the requested + model, matching how the HTTP handlers build their candidate list: an + enforced model that resolves to a source must not slip through to + subscription-account selection. + + ``raw_model`` is the client's requested model captured before request + preparation normalized aliases (``gpt-5-high`` -> ``gpt-5``), mirroring the + HTTP path's ``raw_source_model``: the caller has already substituted the + API key's ``enforced_model`` and applied the fast-mode correction, so it is + used verbatim as the leading source candidate. When omitted (request states + that predate preparation, e.g. replayed turns), the raw candidate is + derived from ``enforced_model``/``model`` as before. + + Resolution failures fail open to ``False``. This helper only gates the + WebSocket transport, where the alternative is worse: the lookup runs after + the turn's usage reservation is acquired but before it is registered for + cleanup, so a propagating database error tears the whole session down and + strands the reservation until the stale reaper runs. Failing open degrades + to the pre-guard behaviour (the subscription upstream rejects the model), + and source forwarding could not have worked anyway — it needs the same + database for the source's credentials. The HTTP handlers deliberately do + not use this helper: they call ``select_responses_model_source`` directly + and must keep surfacing resolution errors rather than silently routing + source traffic to a subscription account. + """ + raw = raw_model if raw_model is not None else effective_model_for_api_key(api_key, model) + if not model and not raw: + return False + try: + return ( + await select_responses_model_source( + model or raw or "", + api_key, + raw_model=raw, + require_streaming=True, + ) + is not None + ) + except Exception: + logger.warning("model_source_resolution_failed_open", exc_info=True) + return False diff --git a/app/modules/model_sources/service.py b/app/modules/model_sources/service.py index a0420d0d29..b9fd89e858 100644 --- a/app/modules/model_sources/service.py +++ b/app/modules/model_sources/service.py @@ -57,6 +57,7 @@ async def create_source(self, payload: ModelSourceCreateRequest) -> ModelSourceR supports_chat_completions=payload.supports_chat_completions, supports_responses=payload.supports_responses, supports_audio_transcriptions=payload.supports_audio_transcriptions, + supports_embeddings=payload.supports_embeddings, timeout_seconds=payload.timeout_seconds, max_concurrency=payload.max_concurrency, models=model_rows, @@ -88,6 +89,8 @@ async def update_source(self, source_id: str, payload: ModelSourceUpdateRequest) row.supports_responses = payload.supports_responses if "supports_audio_transcriptions" in fields and payload.supports_audio_transcriptions is not None: row.supports_audio_transcriptions = payload.supports_audio_transcriptions + if "supports_embeddings" in fields and payload.supports_embeddings is not None: + row.supports_embeddings = payload.supports_embeddings if "timeout_seconds" in fields: row.timeout_seconds = payload.timeout_seconds if "max_concurrency" in fields: @@ -224,6 +227,7 @@ def _to_response(row: ModelSource) -> ModelSourceResponse: supports_chat_completions=row.supports_chat_completions, supports_responses=row.supports_responses, supports_audio_transcriptions=row.supports_audio_transcriptions, + supports_embeddings=row.supports_embeddings, timeout_seconds=row.timeout_seconds, max_concurrency=row.max_concurrency, created_at=row.created_at, diff --git a/app/modules/proxy/_load_balancer/sticky_selection.py b/app/modules/proxy/_load_balancer/sticky_selection.py index a1b61c2639..a4403f88f2 100644 --- a/app/modules/proxy/_load_balancer/sticky_selection.py +++ b/app/modules/proxy/_load_balancer/sticky_selection.py @@ -23,6 +23,7 @@ TrafficClass, select_account, ) +from app.core.utils.time import utcnow from app.db.models import Account, AccountStatus, AdditionalUsageHistory, StickySessionKind, UsageHistory from app.modules.accounts.repository import AccountsRepository from app.modules.proxy._load_balancer.types import ( @@ -39,7 +40,7 @@ fair_share_denial_message, ) from app.modules.proxy.repo_bundle import ProxyRepoFactory -from app.modules.proxy.sticky_repository import StickySessionsRepository +from app.modules.proxy.sticky_repository import StickyOwnerLookup, StickySessionsRepository from app.modules.quota_planner.logic import PlannerSettings, build_routing_costs # Preserve the established observability surface while implementation moves to @@ -77,6 +78,9 @@ class SelectionInputsProtocol(Protocol): @property def effective_continuity_owner_candidates(self) -> list[Account]: ... + @property + def effective_sticky_mutation_authority_account_ids(self) -> frozenset[str]: ... + SelectionInputsT = TypeVar("SelectionInputsT", bound=SelectionInputsProtocol) @@ -185,11 +189,13 @@ async def _select_with_stickiness( sticky_repo: StickySessionsRepository | None, routing_costs_by_account_id: RoutingCostsByAccount | None, sticky_existing_account_id: str | None | object, + initial_preferred_account_id: str | None, preserve_existing_mapping_on_fallback: bool, traffic_class: TrafficClass, ignore_standard_quota: bool, allow_usage_exhaustion_error: bool = True, usage_exhaustion_states: Iterable[AccountState] | None = None, + sticky_refresh_skip_deadline: datetime | None = None, ) -> _StickySelectionOutcome: ... async def release_account_lease(self, lease: AccountLease | None) -> None: ... @@ -203,7 +209,12 @@ class StickySelectionRequest(Generic[SelectionInputsT]): sticky_source: _CodexSessionSource | None legacy_sticky_key: str | None legacy_existing_account_id: str | None + legacy_abandoned_account_id: str | None + sticky_seed_key: str | None + sticky_seed_kind: StickySessionKind | None + sticky_seed_account_id: str | None spill_bare_session_on_account_cap: bool + abandon_unavailable_legacy_owner: bool require_unambiguous_account: bool sticky_max_age_seconds: int | None prefer_earlier_reset_accounts: bool @@ -227,6 +238,10 @@ class StickySelectionRequest(Generic[SelectionInputsT]): allow_usage_exhaustion_error: bool = True api_key_id: str | None = None api_key_stream_fair_share_threshold_pct: int = 0 + # First-iteration owner read performed by the caller inside its shared + # owner-lookup session (see load_balancer.select_account). Consumed exactly + # once; retries re-read fresh ownership evidence through a repo bundle. + initial_sticky_owner_lookup: StickyOwnerLookup | None = None @dataclass(frozen=True, slots=True) @@ -234,6 +249,12 @@ class _StickyMutation: # ``None`` is an intentional delete; absence of a mutation means preserve # the current mapping until final admission succeeds. account_id: str | None + # Set only when this mutation is a pure same-owner freshness rewrite of a + # row this request's lookup observed inside the repository's refresh-skip + # window. The persist site revalidates the deadline against the clock at + # write time and may then omit the statement entirely; a mutation that + # rebinds, deletes, or must initialize a seed mapping never carries it. + refresh_skip_deadline: datetime | None = None @dataclass(frozen=True, slots=True) @@ -265,7 +286,12 @@ async def run_sticky_selection_path( sticky_source = request.sticky_source legacy_sticky_key = request.legacy_sticky_key legacy_existing_account_id = request.legacy_existing_account_id + legacy_abandoned_account_id = request.legacy_abandoned_account_id + sticky_seed_key = request.sticky_seed_key + sticky_seed_kind = request.sticky_seed_kind + sticky_seed_account_id = request.sticky_seed_account_id spill_bare_session_on_account_cap = request.spill_bare_session_on_account_cap + abandon_unavailable_legacy_owner = request.abandon_unavailable_legacy_owner require_unambiguous_account = request.require_unambiguous_account sticky_max_age_seconds = request.sticky_max_age_seconds prefer_earlier_reset_accounts = request.prefer_earlier_reset_accounts @@ -313,41 +339,98 @@ def _direct_error( sticky_existing_account_id: str | None | object = _STICKY_EXISTING_UNSET sticky_continuity_abandoned = False + sticky_refresh_skip_deadline: datetime | None = None + # A thread row whose process seed exists but is still unowned must keep + # its retention write: that write doubles as the seed-initialization + # carrier (see the ``initialize_seed_key`` argument at the persist site + # below), and suppressing it would let sibling threads select divergent + # owners until the skip window closes. Thread-only affinity without a + # seed key has nothing to initialize and stays skippable. + seed_initialization_pending = ( + sticky_source == "thread_header" and sticky_seed_key is not None and sticky_seed_account_id is None + ) + # A source-qualified marker can be observed before this call or after a + # retirement CAS miss. In both cases its retained owner is authoritative + # exclusion evidence even though it is no longer affinity ownership for + # the matching session-header source. + retired_legacy_owner_account_ids = ( + {legacy_abandoned_account_id} if legacy_abandoned_account_id is not None else set() + ) attempt = 0 suppress_recovery_probe_candidates = False + pending_initial_owner_lookup = request.initial_sticky_owner_lookup while True: attempt += 1 sticky_existing_is_legacy = isinstance(legacy_existing_account_id, str) if sticky_kind is not None: async with owner._runtime_lock: pass - async with owner._repo_factory() as repos: - sticky_owner_lookup = await repos.sticky_sessions.get_account_id_and_abandonment( - sticky_key, - kind=sticky_kind, - max_age_seconds=sticky_max_age_seconds, - ) - sticky_existing_account_id = sticky_owner_lookup.account_id - # `is True` (not a truthy check): an un-configured test double - # for sticky_sessions may return an object whose attribute - # access auto-vivifies to a mock rather than a real bool, and - # that must fail safe as "not abandoned", the same as it - # always has, rather than silently bypassing the ambiguous - # owner check below. - sticky_continuity_abandoned = sticky_owner_lookup.continuity_abandoned is True - if sticky_kind == StickySessionKind.CODEX_SESSION and sticky_existing_is_legacy: + if pending_initial_owner_lookup is not None: + # The caller already read this iteration's owner inside its + # shared owner-lookup session. Consume it exactly once so + # every retry (including post-reset attempts that wrap + # ``attempt`` back to 1) still re-reads fresh evidence. + sticky_owner_lookup = pending_initial_owner_lookup + pending_initial_owner_lookup = None + else: + async with owner._repo_factory() as repos: + sticky_owner_lookup = await repos.sticky_sessions.get_account_id_and_abandonment( + sticky_key, + kind=sticky_kind, + max_age_seconds=sticky_max_age_seconds, + continuity_source=sticky_source, + ) + sticky_existing_account_id = sticky_owner_lookup.account_id + # `is True` (not a truthy check): an un-configured test double + # for sticky_sessions may return an object whose attribute + # access auto-vivifies to a mock rather than a real bool, and + # that must fail safe as "not abandoned", the same as it + # always has, rather than silently bypassing the ambiguous + # owner check below. + sticky_continuity_abandoned = sticky_owner_lookup.continuity_abandoned is True + # ``isinstance`` for the same test-double reason as above. The + # deadline is only ever an optimization hint: None always + # falls back to today's write-on-every-request refresh + # behavior, and seed-needing requests never skip. + observed_refresh_skip_deadline = sticky_owner_lookup.refresh_skip_deadline + sticky_refresh_skip_deadline = ( + observed_refresh_skip_deadline + if isinstance(observed_refresh_skip_deadline, datetime) and not seed_initialization_pending + else None + ) + sticky_abandoned_account_id = sticky_owner_lookup.abandoned_account_id + if sticky_owner_lookup.continuity_abandoned is True and isinstance( + sticky_abandoned_account_id, + str, + ): + retired_legacy_owner_account_ids.add(sticky_abandoned_account_id) + if sticky_existing_is_legacy: # Mixed-version replicas can create both rows on # different accounts. The raw row was loaded before # branch selection and always wins as possible hard # turn-state ownership. sticky_existing_account_id = legacy_existing_account_id sticky_continuity_abandoned = False + # The freshness observation belongs to the namespaced row, + # not the raw legacy owner that now shadows it. + sticky_refresh_skip_deadline = None async with owner._runtime_lock: states, account_map = owner._prepare_sticky_selection_states( selection_inputs, required_account_id=required_account_id, redact_sensitive_details=redact_sensitive_details, ) + if retired_legacy_owner_account_ids: + # Retirement is authoritative even when this selector loaded a + # pre-retirement account snapshot (or another replica still has + # one cached). Never let that stale snapshot immediately repin + # the account this request just proved durably unavailable. + states = [state for state in states if state.account_id not in retired_legacy_owner_account_ids] + account_map = { + account_id: account + for account_id, account in account_map.items() + if account_id not in retired_legacy_owner_account_ids + } effective_routing_costs = ( routing_costs_by_account_id if routing_costs_by_account_id is not None @@ -368,10 +451,8 @@ def _direct_error( and not sticky_existing_is_legacy ) cap_spillover_allowed = spill_bare_session_on_account_cap and lease_kind is not None and bare_session_key - hard_sticky = ( - sticky_kind == StickySessionKind.CODEX_SESSION - and isinstance(sticky_existing_account_id, str) - and not bare_session_key + hard_sticky = isinstance(sticky_existing_account_id, str) and ( + sticky_existing_is_legacy or (sticky_kind == StickySessionKind.CODEX_SESSION and not bare_session_key) ) if hard_sticky and required_account_id is not None and sticky_existing_account_id != required_account_id: return _direct_error( @@ -462,6 +543,67 @@ def _direct_error( traffic_class=traffic_class, ) probe_reservation: ProbeReservation | None = None + # Raw sticky rows are global, while account-assigned API keys and + # other authenticated policies narrow a request's mutation authority. + # Keep this check on the pre-health continuity pool: quota exhaustion + # may authorize retirement, but being outside policy scope never does. + legacy_owner_in_effective_policy_scope = ( + isinstance(sticky_existing_account_id, str) + and sticky_existing_account_id in selection_inputs.effective_sticky_mutation_authority_account_ids + ) + if ( + abandon_unavailable_legacy_owner + and hard_sticky + and sticky_existing_is_legacy + and sticky_source in {"session_header", "thread_header"} + and legacy_sticky_key is not None + and isinstance(sticky_existing_account_id, str) + and legacy_owner_in_effective_policy_scope + ): + async with owner._repo_factory() as repos: + owner_retired = await repos.sticky_sessions.abandon_legacy_session_header_owner_if_unavailable( + legacy_sticky_key, + kind=StickySessionKind.CODEX_SESSION, + expected_account_id=sticky_existing_account_id, + ) + authoritative_legacy_owner = None + if not owner_retired: + authoritative_legacy_owner = await repos.sticky_sessions.get_account_id_and_abandonment( + legacy_sticky_key, + kind=StickySessionKind.CODEX_SESSION, + continuity_source="session_header", + ) + # One guarded write is authoritative for this selection. Repeating + # it in capacity-wait retries would add write pressure and could + # reinterpret a later status transition as restart authorization. + abandon_unavailable_legacy_owner = False + if owner_retired: + # The raw compatibility row is now a tombstone. Drop only the + # selection loop's cached legacy owner and run the normal path + # again so namespaced affinity, leases, and admission checks + # are established through the existing selection path. + logger.info( + "Legacy Codex session-header owner abandoned for self-contained goal restart account_id=%s", + "" if redact_sensitive_details else sticky_existing_account_id, + ) + retired_legacy_owner_account_ids.add(sticky_existing_account_id) + legacy_existing_account_id = None + continue + # A failed compare-and-set means the cached owner is no longer + # authoritative: it may have recovered, another request may have + # rebound the raw row, or another worker may already have + # tombstoned it. Re-read under a fresh transaction and restart the + # loop so each outcome is handled by normal selection. Retaining + # the stale owner here would defeat the CAS and can fail a restart + # even though a concurrent operation already established a valid + # replacement. + assert authoritative_legacy_owner is not None + legacy_existing_account_id = authoritative_legacy_owner.account_id + if authoritative_legacy_owner.continuity_abandoned is True: + abandoned_account_id = authoritative_legacy_owner.abandoned_account_id + if isinstance(abandoned_account_id, str): + retired_legacy_owner_account_ids.add(abandoned_account_id) + continue sticky_outcome = _StickySelectionOutcome(selection=SelectionResult(None, None)) if fair_share_denial is not None: # Denial parks in the transport capacity-wait loop like a cap @@ -534,12 +676,18 @@ def _direct_error( relative_availability_top_k=relative_availability_top_k, sticky_repo=repos.sticky_sessions, sticky_existing_account_id=sticky_existing_account_id, + initial_preferred_account_id=( + sticky_seed_account_id + if not isinstance(sticky_existing_account_id, str) and not sticky_continuity_abandoned + else None + ), preserve_existing_mapping_on_fallback=preserve_existing_mapping, traffic_class=traffic_class, ignore_standard_quota=False, routing_costs_by_account_id=effective_routing_costs, allow_usage_exhaustion_error=allow_usage_exhaustion_error, usage_exhaustion_states=states, + sticky_refresh_skip_deadline=sticky_refresh_skip_deadline, ) result = sticky_outcome.selection if ( @@ -806,20 +954,35 @@ def _direct_error( assert sticky_kind is not None sticky_mutation = sticky_outcome.mutation assert sticky_mutation is not None - try: - async with owner._repo_factory() as repos: - await _persist_sticky_mutation( - sticky_repo=repos.sticky_sessions, - sticky_key=sticky_key, - sticky_kind=sticky_kind, - mutation=sticky_mutation, - ) - except BaseException: - await owner.release_account_lease(selected_lease) - selected_lease = None - async with owner._runtime_lock: - owner._release_due_probe_reservation_locked(probe_reservation) - raise + # A pure same-owner freshness rewrite may be omitted here exactly + # as on the non-probe path below (the row already holds this + # owner, so the rollback restores become no-ops and are skipped + # symmetrically). The probe path deliberately never initializes a + # seed, so only the deadline gates the skip. + probe_refresh_write_skipped = _sticky_refresh_write_skippable( + sticky_mutation, + initialize_seed_key=None, + ) + if not probe_refresh_write_skipped: + try: + async with owner._repo_factory() as repos: + # A recovery-probe reservation is still reversible until + # the runtime CAS below succeeds. Persist its thread row so + # existing rollback machinery can restore it, but do not + # publish an immutable process seed that cannot be safely + # deleted after a concurrent sibling observes it. + await _persist_sticky_mutation( + sticky_repo=repos.sticky_sessions, + sticky_key=sticky_key, + sticky_kind=sticky_kind, + mutation=sticky_mutation, + ) + except BaseException: + await owner.release_account_lease(selected_lease) + selected_lease = None + async with owner._runtime_lock: + owner._release_due_probe_reservation_locked(probe_reservation) + raise try: async with owner._runtime_lock: assert probe_reservation is not None @@ -837,14 +1000,15 @@ def _direct_error( selected_lease = None async with owner._runtime_lock: owner._release_due_probe_reservation_locked(probe_reservation) - async with owner._repo_factory() as repos: - await _restore_sticky_mutation( - sticky_repo=repos.sticky_sessions, - sticky_key=sticky_key, - sticky_kind=sticky_kind, - expected_account_id=sticky_mutation.account_id, - sticky_existing_account_id=sticky_existing_account_id, - ) + if not probe_refresh_write_skipped: + async with owner._repo_factory() as repos: + await _restore_sticky_mutation( + sticky_repo=repos.sticky_sessions, + sticky_key=sticky_key, + sticky_kind=sticky_kind, + expected_account_id=sticky_mutation.account_id, + sticky_existing_account_id=sticky_existing_account_id, + ) raise if not reservation_committed: # Runtime health changed while account-state persistence @@ -854,14 +1018,15 @@ def _direct_error( # runtime snapshot. await owner.release_account_lease(selected_lease) selected_lease = None - async with owner._repo_factory() as repos: - await _restore_sticky_mutation( - sticky_repo=repos.sticky_sessions, - sticky_key=sticky_key, - sticky_kind=sticky_kind, - expected_account_id=sticky_mutation.account_id, - sticky_existing_account_id=sticky_existing_account_id, - ) + if not probe_refresh_write_skipped: + async with owner._repo_factory() as repos: + await _restore_sticky_mutation( + sticky_repo=repos.sticky_sessions, + sticky_key=sticky_key, + sticky_kind=sticky_kind, + expected_account_id=sticky_mutation.account_id, + sticky_existing_account_id=sticky_existing_account_id, + ) selected_snapshot = None error_message = None selected_states = [] @@ -946,21 +1111,27 @@ def _direct_error( assert sticky_kind is not None sticky_mutation = sticky_outcome.mutation assert sticky_mutation is not None - try: - async with owner._repo_factory() as repos: - await _persist_sticky_mutation( - sticky_repo=repos.sticky_sessions, - sticky_key=sticky_key, - sticky_kind=sticky_kind, - mutation=sticky_mutation, - ) - except BaseException: - # Runtime admission may already be committed. Preserve - # its selection timestamp, but never leak the local - # concurrency lease when sticky persistence fails. - await owner.release_account_lease(selected_lease) - selected_lease = None - raise + initialize_seed_key = ( + sticky_seed_key if sticky_source == "thread_header" and sticky_seed_account_id is None else None + ) + if not _sticky_refresh_write_skippable(sticky_mutation, initialize_seed_key=initialize_seed_key): + try: + async with owner._repo_factory() as repos: + await _persist_sticky_mutation( + sticky_repo=repos.sticky_sessions, + sticky_key=sticky_key, + sticky_kind=sticky_kind, + mutation=sticky_mutation, + initialize_seed_key=initialize_seed_key, + initialize_seed_kind=sticky_seed_kind, + ) + except BaseException: + # Runtime admission may already be committed. Preserve + # its selection timestamp, but never leak the local + # concurrency lease when sticky persistence fails. + await owner.release_account_lease(selected_lease) + selected_lease = None + raise break return StickySelectionOutcome( @@ -991,11 +1162,13 @@ async def _select_with_stickiness( sticky_repo: StickySessionsRepository | None, routing_costs_by_account_id: RoutingCostsByAccount | None = None, sticky_existing_account_id: str | None | object = _STICKY_EXISTING_UNSET, + initial_preferred_account_id: str | None = None, preserve_existing_mapping_on_fallback: bool = False, traffic_class: TrafficClass = TRAFFIC_CLASS_FOREGROUND, ignore_standard_quota: bool = False, allow_usage_exhaustion_error: bool = True, usage_exhaustion_states: Iterable[AccountState] | None = None, + sticky_refresh_skip_deadline: datetime | None = None, ) -> _StickySelectionOutcome: if not sticky_key or not sticky_repo: return _StickySelectionOutcome( @@ -1023,10 +1196,14 @@ def finish_selection( selection: SelectionResult, *, persist_account_id: str | None = None, + refresh_skip_deadline: datetime | None = None, ) -> _StickySelectionOutcome: mutation = pending_mutation if persist_account_id is not None: - mutation = _StickyMutation(account_id=persist_account_id) + mutation = _StickyMutation( + account_id=persist_account_id, + refresh_skip_deadline=refresh_skip_deadline, + ) return _StickySelectionOutcome(selection=selection, mutation=mutation) if sticky_existing_account_id is _STICKY_EXISTING_UNSET: @@ -1035,6 +1212,10 @@ def finish_selection( kind=sticky_kind, max_age_seconds=sticky_max_age_seconds, ) + # The skip deadline is only valid for the lookup that produced the + # caller's ``sticky_existing_account_id``; this fresh lookup did not + # observe row freshness, so fall back to write-through refresh. + sticky_refresh_skip_deadline = None else: existing = sticky_existing_account_id if isinstance(sticky_existing_account_id, str) else None # When the pinned account is temporarily unavailable (rate-limited, @@ -1046,9 +1227,47 @@ def finish_selection( persist_fallback = not preserve_existing_mapping_on_fallback apply_sticky_secondary_budget_threshold = False + if not existing and initial_preferred_account_id is not None: + initial_preferred = next( + (state for state in states if state.account_id == initial_preferred_account_id), + None, + ) + if initial_preferred is not None: + initial_result = select_account( + [initial_preferred], + prefer_earlier_reset=prefer_earlier_reset_accounts, + prefer_earlier_reset_window=prefer_earlier_reset_window, + routing_strategy=routing_strategy, + allow_backoff_fallback=False, + relative_availability_power=relative_availability_power, + relative_availability_top_k=relative_availability_top_k, + traffic_class=traffic_class, + ignore_standard_quota=ignore_standard_quota, + routing_costs=routing_costs_by_account_id, + ) + if initial_result.account is not None: + # Persist only the new thread row. The process mapping supplied + # the preference but is deliberately outside this mutation. + return finish_selection( + initial_result, + persist_account_id=initial_preferred.account_id, + ) + if existing: pinned = next((state for state in states if state.account_id == existing), None) if pinned is not None: + # Retaining the pinned owner persists only to advance + # ``updated_at`` on TTL-based kinds. When this request's lookup + # already observed the row inside the repository's refresh-skip + # window, the persist site may skip that write after revalidating + # the observed deadline against the clock: concurrent requests on + # a hot session otherwise serialize on the same row's upsert + # lock. Rebinds and deletes never carry the deadline and always + # write immediately. + pinned_refresh_account_id = pinned.account_id if sticky_max_age_seconds is not None else None + pinned_refresh_skip_deadline = ( + sticky_refresh_skip_deadline if pinned_refresh_account_id is not None else None + ) # Proactively rebind session affinity for any sticky kind # once the pinned account is already above the configured # budget threshold. That preserves continuity below the @@ -1110,7 +1329,8 @@ def finish_selection( if pinned_result.account is not None: return finish_selection( pinned_result, - persist_account_id=pinned.account_id if sticky_max_age_seconds is not None else None, + persist_account_id=pinned_refresh_account_id, + refresh_skip_deadline=pinned_refresh_skip_deadline, ) else: # Reallocate only when a burn-first target exists and can @@ -1164,7 +1384,8 @@ def finish_selection( if pinned_result.account is not None: return finish_selection( pinned_result, - persist_account_id=(pinned.account_id if sticky_max_age_seconds is not None else None), + persist_account_id=pinned_refresh_account_id, + refresh_skip_deadline=pinned_refresh_skip_deadline, ) reallocate_sticky = True # Grace period: if the pinned account is rate-limited with a @@ -1191,7 +1412,8 @@ def finish_selection( if grace_result.account is not None: return finish_selection( grace_result, - persist_account_id=pinned.account_id if sticky_max_age_seconds is not None else None, + persist_account_id=pinned_refresh_account_id, + refresh_skip_deadline=pinned_refresh_skip_deadline, ) if reallocate_sticky: pending_mutation = _StickyMutation(account_id=None) @@ -1243,16 +1465,58 @@ def finish_selection( return finish_selection(chosen) +def _sticky_refresh_write_skippable( + mutation: _StickyMutation, + *, + initialize_seed_key: str | None, +) -> bool: + """Whether this mutation's write may be omitted at persist time. + + True only for a pure same-owner freshness rewrite whose observed skip + deadline still holds now, at the moment the statement would otherwise be + issued — admission and account-state persistence sit between selection + and this point, so the deadline computed at lookup time must be + revalidated to keep the mapping's effective expiry within the documented + skip-window bound. Deletes and seed-initializing writes are never + skippable. + """ + if mutation.account_id is None or initialize_seed_key is not None: + return False + deadline = mutation.refresh_skip_deadline + return isinstance(deadline, datetime) and utcnow() <= deadline + + async def _persist_sticky_mutation( *, sticky_repo: StickySessionsRepository, sticky_key: str, sticky_kind: StickySessionKind, mutation: _StickyMutation, + initialize_seed_key: str | None = None, + initialize_seed_kind: StickySessionKind | None = None, ) -> None: if mutation.account_id is None: await sticky_repo.delete(sticky_key, kind=sticky_kind) return + if initialize_seed_key is not None: + if initialize_seed_kind is None: + raise ValueError("initialize_seed_kind is required when initialize_seed_key is provided") + # Current Codex sends thread-id on the first root request, so a fresh + # process has no older bare-session request available to create its + # default. Initialize it exactly once from the first admitted thread. + # insert-if-absent is essential: failover or a later child may move its + # own bounded row but can never rewrite the process/sibling default. + # The repository operation is intentionally atomic; splitting it into + # the public insert/upsert methods would commit a process default even + # when persistence of the initiating thread fails. + await sticky_repo.upsert_with_seed_if_absent( + sticky_key, + mutation.account_id, + kind=sticky_kind, + seed_key=initialize_seed_key, + seed_kind=initialize_seed_kind, + ) + return await sticky_repo.upsert(sticky_key, mutation.account_id, kind=sticky_kind) diff --git a/app/modules/proxy/_service/api_key_usage.py b/app/modules/proxy/_service/api_key_usage.py index c9bbca703f..0acab92c97 100644 --- a/app/modules/proxy/_service/api_key_usage.py +++ b/app/modules/proxy/_service/api_key_usage.py @@ -26,6 +26,7 @@ from app.modules.proxy._service.support import ( _ApiKeyReservationTouchState, _consume_api_key_reservation_heartbeat_result, + _signal_propagated_responses_service_cleanup_ready, _StreamSettlement, _WebSocketRequestState, ) @@ -108,7 +109,7 @@ async def _reserve_websocket_api_key_usage( service = _service_api_keys_service()(repos.api_keys) try: return await service.enforce_limits_for_request( - api_key.id, + "", request_model=request_model, request_service_tier=request_service_tier, request_usage_budget=request_usage_budget, @@ -306,6 +307,7 @@ async def _settle_compact_api_key_usage( ) proxy = cast(_ApiKeyUsageServiceProtocol, self) + reservation_released = False with anyio.CancelScope(shield=True): try: async with proxy._repo_factory() as repos: @@ -321,10 +323,11 @@ async def _settle_compact_api_key_usage( ) else: await api_keys_service.release_usage_reservation(reservation_id) + reservation_released = True except Exception as exc: logger.warning( "Failed to settle compact API key reservation key_id=%s request_id=%s", - api_key.id, + "", get_request_id(), exc_info=True, ) @@ -332,6 +335,7 @@ async def _settle_compact_api_key_usage( async with proxy._repo_factory() as repos: api_keys_service = _service_api_keys_service()(repos.api_keys) await api_keys_service.release_usage_reservation(reservation_id) + reservation_released = True except Exception: logger.warning( "Failed to release compact API key reservation after settlement failure " @@ -350,7 +354,38 @@ async def _settle_compact_api_key_usage( failure_phase="usage_settlement", failure_detail="compact_api_key_usage_persistence_failed", failure_exception_type=type(exc).__name__, + reservation_released=reservation_released, ) from exc + finally: + _signal_propagated_responses_service_cleanup_ready() + + async def settle_image_api_key_usage( + self, + api_key: ApiKeyData | None, + reservation: ApiKeyUsageReservationData | None, + *, + model: str, + input_tokens: int | None, + output_tokens: int | None, + cached_input_tokens: int | None, + request_id: str, + ) -> bool: + """Transfer captured image usage to tracked reservation settlement.""" + has_usage = input_tokens is not None or output_tokens is not None + settlement = _StreamSettlement( + status="success" if has_usage else "failed", + model=model, + input_tokens=int(input_tokens or 0) if has_usage else None, + output_tokens=int(output_tokens or 0) if has_usage else None, + cached_input_tokens=int(cached_input_tokens or 0) if has_usage else None, + service_tier=None, + ) + return await self._settle_stream_api_key_usage( + api_key, + reservation, + settlement, + request_id=request_id, + ) async def _settle_stream_api_key_usage( self, diff --git a/app/modules/proxy/_service/codex_control.py b/app/modules/proxy/_service/codex_control.py index 6879c7c3c4..5c3fad98a2 100644 --- a/app/modules/proxy/_service/codex_control.py +++ b/app/modules/proxy/_service/codex_control.py @@ -229,6 +229,9 @@ async def _select_codex_control_account_without_budget( reallocate_sticky=affinity.reallocate_sticky, sticky_source=affinity.codex_session_source, legacy_sticky_key=affinity.legacy_selection_key, + legacy_continuity_source=affinity.legacy_continuity_source, + sticky_seed_key=affinity.seed_selection_key, + sticky_seed_kind=affinity.seed_selection_kind, sticky_max_age_seconds=affinity.max_age_seconds, account_ids=scoped_account_ids, prefer_earlier_reset_window=prefer_earlier_reset_window, @@ -396,6 +399,9 @@ async def _select_control_failover(excluded_account_ids: set[str]) -> AccountSel reallocate_sticky=affinity.reallocate_sticky, sticky_source=affinity.codex_session_source, legacy_sticky_key=affinity.legacy_selection_key, + legacy_continuity_source=affinity.legacy_continuity_source, + sticky_seed_key=affinity.seed_selection_key, + sticky_seed_kind=affinity.seed_selection_kind, sticky_max_age_seconds=affinity.max_age_seconds, prefer_earlier_reset_accounts=settings.prefer_earlier_reset_accounts, routing_strategy=routing_strategy, @@ -488,6 +494,9 @@ async def _select_control_failover(excluded_account_ids: set[str]) -> AccountSel reallocate_sticky=affinity.reallocate_sticky, sticky_source=affinity.codex_session_source, legacy_sticky_key=affinity.legacy_selection_key, + legacy_continuity_source=affinity.legacy_continuity_source, + sticky_seed_key=affinity.seed_selection_key, + sticky_seed_kind=affinity.seed_selection_kind, sticky_max_age_seconds=affinity.max_age_seconds, prefer_earlier_reset_accounts=settings.prefer_earlier_reset_accounts, prefer_earlier_reset_window=_prefer_earlier_reset_window(settings), diff --git a/app/modules/proxy/_service/compact.py b/app/modules/proxy/_service/compact.py index f6a87d32a7..682cfb4bb9 100644 --- a/app/modules/proxy/_service/compact.py +++ b/app/modules/proxy/_service/compact.py @@ -9,6 +9,7 @@ from typing import Any, NoReturn, Protocol, TypeVar, cast import aiohttp +from pydantic import ValidationError from app.core.auth.refresh import RefreshError, is_transient_refresh_contention, refresh_contention_kind from app.core.balancer import ResetPreferenceWindow, RoutingStrategy, failover_decision @@ -23,6 +24,7 @@ from app.core.config.settings import get_settings from app.core.config.settings_cache import get_settings_cache from app.core.errors import openai_error +from app.core.openai.exceptions import ClientPayloadError from app.core.openai.models import CompactResponsePayload from app.core.openai.requests import ResponsesCompactRequest from app.core.resilience.network_recovery import ProcessNetworkRecovery @@ -30,7 +32,7 @@ from app.core.upstream_proxy import ResolvedUpstreamRoute, UpstreamProxyRouteError from app.core.utils.request_id import ensure_request_id, get_request_id from app.core.utils.retry import backoff_seconds -from app.db.models import Account, DashboardSettings, StickySessionKind +from app.db.models import Account, AccountStatus, DashboardSettings, StickySessionKind from app.modules.api_keys.service import ( ApiKeyData, ApiKeyRequestUsageBudget, @@ -48,16 +50,30 @@ _resolve_prompt_cache_key, _sticky_key_from_session_header, _sticky_key_from_turn_state_header, + _thread_codex_session_affinity, ) from app.modules.proxy.api_key_usage import estimate_api_key_request_usage -from app.modules.proxy.continuity import resolve_required_account_id -from app.modules.proxy.helpers import _header_account_id, _normalize_error_code, _parse_openai_error +from app.modules.proxy.continuity import ( + resolve_required_account_id, + without_http_bridge_session_affinity_headers, +) +from app.modules.proxy.helpers import ( + _header_account_id, + _normalize_error_code, + _parse_openai_error, + classify_upstream_failure, +) from app.modules.proxy.load_balancer import ( AccountConcurrencyCaps, AccountLease, AccountSelection, effective_account_concurrency_caps, ) +from app.modules.proxy.replay_safety import ( + project_responses_input_for_account_neutral_fresh_replay, + responses_input_suffix_retains_prior_output, + responses_payload_is_account_neutral_fresh_replay, +) from app.modules.proxy.selection_errors import selection_failure_response from app.modules.proxy.work_admission import AdmissionLease, WorkAdmissionController @@ -97,6 +113,15 @@ async def _resolve_file_account_for_responses( self, payload: ResponsesCompactRequest, headers: Mapping[str, str] ) -> str | None: ... + async def _resolve_forwarded_file_account_for_responses( + self, + payload: ResponsesCompactRequest, + headers: Mapping[str, str], + *, + forwarded_file_owner_account_id: str | None, + require_forwarded_file_owner: bool = False, + ) -> str | None: ... + async def _acquire_account_response_create_lease_or_overload( self, *, account_id: str, request_id: str, surface: str, concurrency_caps: AccountConcurrencyCaps ) -> AccountLease: ... @@ -124,6 +149,8 @@ async def _resolve_compact_turn_state_owner( fail_on_missing: bool = True, ) -> str | None: ... + async def _compact_owner_selection_loss_is_quota_caused(self, account_id: str) -> bool: ... + async def _ensure_fresh_with_budget( self, account: Account, *, force: bool = False, timeout_seconds: float | None = None ) -> Account: ... @@ -427,6 +454,14 @@ def _sticky_key_for_compact_request( kind=StickySessionKind.CODEX_SESSION, codex_session_source="turn_state", ) + elif ( + thread_affinity := _thread_codex_session_affinity( + headers, + enabled=codex_session_affinity, + max_age_seconds=openai_cache_affinity_max_age_seconds, + ) + ) is not None: + policy = thread_affinity elif ( session_affinity := _bare_codex_session_affinity( headers, @@ -457,7 +492,160 @@ def _service_tier_from_compact_payload(payload: ResponsesCompactRequest) -> str return normalize(payload.service_tier) +# Account statuses that prove the pinned owner's selection-time loss is caused +# by upstream quota/rate-limit state rather than authentication, deactivation, +# or an operator pause. Only these authorize account-neutral replay recovery. +_COMPACT_OWNER_QUOTA_UNAVAILABLE_STATUSES = ( + AccountStatus.RATE_LIMITED, + AccountStatus.QUOTA_EXCEEDED, +) + + +def _compact_replay_history_retains_prior_output(input_items: list[JsonValue]) -> bool: + """Prove the carried history retains prior assistant output before new input. + + A self-contained account-neutral ``input`` is not by itself a full resend: + a client could send only the turns after ``previous_response_id`` (for + example two fresh user messages) and rely on the owner account to hold the + earlier conversation, so replaying without the anchor would compact a + truncated history. Without durable prefix metadata for the compact surface, + the strongest client-side evidence of a full resend is the same + retained-prior-output shape the HTTP bridge replay path trusts: the input + must parse as a clean transcript whose final segment is the previous + response's completed assistant output followed only by fresh client input. + The split is anchored at the last assistant message so the shared suffix + walk proves exactly that segment; anything it cannot prove stays + owner-bound. + + This is the evidence ceiling of the #1490 rescope: completeness relative to + the anchored conversation is not provable from the payload alone, and the + durable prefix metadata that could prove it is deliberately not consulted + here. A delta resend that itself carries a completed assistant exchange + ahead of the fresh input is indistinguishable from a full resend and is + recovered as the client's authoritative local history — the same trust the + shared account-neutral fresh-replay rules already grant a normal turn that + abandons an unavailable owner. The rejected shapes below are the ones the + transcript walk can actually refute. + """ + + last_assistant_index: int | None = None + for index in range(len(input_items) - 1, -1, -1): + item = input_items[index] + if isinstance(item, dict) and item.get("type") in (None, "message") and item.get("role") == "assistant": + last_assistant_index = index + break + # ``responses_input_suffix_retains_prior_output`` requires a non-empty + # stored prefix, so a history that opens with (or lacks) assistant output + # cannot be proven and stays owner-bound. + if last_assistant_index is None or last_assistant_index == 0: + return False + # The projection is an identity transform for input that already passed + # the account-neutral fresh-replay gate (no server-assigned ids, no + # reasoning or omitted bookkeeping types survive that gate), but it is the + # shared authority for recognizing the canonical Responses-Lite developer + # instruction behind an ``additional_tools`` bundle — without that index + # the suffix walk would reject every Lite full resend. + projection = project_responses_input_for_account_neutral_fresh_replay( + input_items, + stored_count=last_assistant_index, + ) + if projection is None: + return False + return responses_input_suffix_retains_prior_output( + projection.input_items, + stored_count=projection.stored_prefix_count, + canonical_lite_developer_index=projection.canonical_lite_developer_index, + ) + + +def _compact_account_neutral_replay_payload( + payload: ResponsesCompactRequest, +) -> ResponsesCompactRequest | None: + """Return the anchor-free replay payload for a verified full resend. + + A compact request pinned only by ``previous_response_id`` may move off an + unselectable owner account when the history it carries is provably + account-neutral: the upstream-bound payload without the anchor must pass + the shared fresh-replay validation, so no encrypted or compaction state, + server-assigned item ids, account-scoped file/container handles, + conversation/prompt handles, or hosted/MCP state can reach the replacement + account. + + Neutrality is checked on the serialized upstream-bound payload + (``to_payload``), never on the request model, matching how the HTTP bridge + replay paths apply the shared gate to pre-transport serializations. The + compact transport applies two further mutations after this serialization — + the Responses-Lite ``reasoning.context`` control and inline image + fetching — both proxy-injected, account-agnostic, and applied identically + to the owner send and the replay send, so they are not part of the client + payload being proven. The serialized history must additionally be a + complete resend. ``to_payload`` can still drop history on the wire: it + strips poisoned local-compact fallback messages together with their + trailing encrypted compaction item, and it trims oversized inputs down to a + head, a trim marker, and a tail. Both remain multi-item account-neutral + lists. Sending either to a replacement account without the anchor would + compact an incomplete conversation, because only the owner can resolve the + omitted context from the dropped anchor. So the wire input must be + item-for-item identical to the validated request input, must still carry + more than one item, and must retain prior assistant output ahead of the new + client input (see ``_compact_replay_history_retains_prior_output``). + """ + + previous_response_id = getattr(payload, "previous_response_id", None) + if not isinstance(previous_response_id, str) or not previous_response_id.strip(): + return None + if not isinstance(payload.input, list): + return None + replay_source = payload.model_dump(mode="json", exclude_none=True) + replay_source.pop("previous_response_id", None) + request_input = replay_source.get("input") + if not isinstance(request_input, list) or len(request_input) <= 1: + return None + try: + replay_payload = ResponsesCompactRequest.model_validate(replay_source) + replay_wire_payload = replay_payload.to_payload() + except (ValidationError, ClientPayloadError): + return None + replay_wire_input = replay_wire_payload.get("input") + if not isinstance(replay_wire_input, list) or len(replay_wire_input) <= 1: + return None + if replay_wire_input != request_input: + return None + if not responses_payload_is_account_neutral_fresh_replay(replay_wire_payload): + return None + if not _compact_replay_history_retains_prior_output(cast(list[JsonValue], replay_wire_input)): + return None + return replay_payload + + class _CompactMixin: + async def _compact_owner_selection_loss_is_quota_caused(self, account_id: str) -> bool: + """Return whether the pinned owner is unselectable because of quota state. + + Account-neutral replay off a pinned previous-response owner is legal + only for owner loss the owner's quota state caused. At selection time + that evidence is the owner's own persisted status: ``RATE_LIMITED`` or + ``QUOTA_EXCEEDED`` is the same upstream usage-exhaustion state the + selector consulted. Authentication loss (``REAUTH_REQUIRED``, + ``DEACTIVATED``), operator pauses, local capacity caps on an ``ACTIVE`` + account, and a failed lookup all stay owner-bound. + """ + + proxy = cast(_CompactServiceProtocol, self) + try: + async with proxy._repo_factory() as repos: + account = await repos.accounts.get_by_id_fresh(account_id) + # Read inside the repository scope: the session expires ORM + # attributes when it closes. + status = account.status if account is not None else None + except Exception: + logger.warning( + "Compact previous-response owner status lookup failed; keeping the request owner-bound", + exc_info=True, + ) + return False + return status in _COMPACT_OWNER_QUOTA_UNAVAILABLE_STATUSES + async def _resolve_compact_turn_state_owner( self, *, @@ -574,6 +762,8 @@ async def compact_responses( api_key: ApiKeyData | None = None, api_key_reservation: ApiKeyUsageReservationData | None = None, client_ip: str | None = None, + forwarded_request: bool = False, + forwarded_file_owner_account_id: str | None = None, ) -> CompactResponsePayload: proxy = cast(_CompactServiceProtocol, self) _maybe_log_proxy_request_payload("compact", payload, headers) @@ -597,8 +787,69 @@ async def compact_responses( route_endpoint_id: str | None = None route_fallback_used: bool | None = None route_fail_closed_reason: str | None = None + settlement_attempted = False + + async def settle_compact_usage( + *, + api_key: ApiKeyData | None, + api_key_reservation: ApiKeyUsageReservationData | None, + response: CompactResponsePayload | None, + request_service_tier: str | None, + ) -> None: + nonlocal settlement_attempted + if settlement_attempted: + return + if forwarded_request and response is None: + # A forwarded receiver has not transferred cleanup ownership + # until its successful HTTP 200. Every error before that + # acknowledgement remains the origin's single release path. + return + settlement_attempted = True + await proxy._settle_compact_api_key_usage( + api_key=api_key, + api_key_reservation=api_key_reservation, + response=response, + request_service_tier=request_service_tier, + ) + proxy._raise_for_unsupported_input_image_references(payload) - rewritten_file_account_id = await proxy._resolve_file_account_for_responses(payload, headers) + try: + rewritten_file_account_id = await proxy._resolve_forwarded_file_account_for_responses( + payload, + headers, + forwarded_file_owner_account_id=forwarded_file_owner_account_id, + require_forwarded_file_owner=forwarded_request, + ) + except ProxyResponseError: + if not forwarded_request and api_key is not None and api_key_reservation is not None: + try: + await settle_compact_usage( + api_key=api_key, + api_key_reservation=api_key_reservation, + response=None, + request_service_tier=_service_tier_from_compact_payload(payload), + ) + except Exception: + logger.warning( + "Failed to settle compact API key reservation after owner lookup failure", + exc_info=True, + ) + raise + except asyncio.CancelledError: + if not forwarded_request and api_key is not None and api_key_reservation is not None: + try: + await settle_compact_usage( + api_key=api_key, + api_key_reservation=api_key_reservation, + response=None, + request_service_tier=_service_tier_from_compact_payload(payload), + ) + except Exception: + logger.warning( + "Failed to settle compact API key reservation after cancelled owner lookup", + exc_info=True, + ) + raise settings = await _service_get_settings_cache().get() concurrency_caps = effective_account_concurrency_caps(settings) prefer_earlier_reset = settings.prefer_earlier_reset_accounts @@ -613,7 +864,11 @@ async def compact_responses( api_key=api_key, ) sticky_key_source = "none" - if affinity.kind == StickySessionKind.CODEX_SESSION: + if affinity.codex_session_source == "thread_header": + # The payload cache hint remains unchanged; diagnostics must not + # imply that it supplied the internal thread-local routing key. + sticky_key_source = "thread_header" + elif affinity.kind == StickySessionKind.CODEX_SESSION: if _sticky_key_from_turn_state_header(headers) is not None: sticky_key_source = "turn_state_header" elif _sticky_key_from_session_header(headers) is not None: @@ -686,6 +941,143 @@ async def compact_responses( ("previous response", previous_response_preferred_account_id), ("input file", rewritten_file_account_id), ) + deferred_stream_health: list[tuple[Account, Any, str, int | None]] = [] + deferred_http_500_health: list[tuple[Account, ProxyResponseError, int]] = [] + deferred_proxy_health: list[tuple[Account, ProxyResponseError]] = [] + settlement_attempted = False + + async def flush_deferred_health() -> None: + stream_pending = list(deferred_stream_health) + deferred_stream_health.clear() + http_500_pending = list(deferred_http_500_health) + deferred_http_500_health.clear() + proxy_pending = list(deferred_proxy_health) + deferred_proxy_health.clear() + for failed_account, failed_error, failed_code, failed_status in stream_pending: + try: + await proxy._handle_stream_error( + failed_account, + failed_error, + failed_code, + http_status=failed_status, + ) + except Exception: + logger.warning( + "Failed to flush deferred compact stream health account_id=%s request_id=%s", + failed_account.id, + request_id, + exc_info=True, + ) + for failed_account, failed_exc, extra_error_count in http_500_pending: + try: + await proxy._handle_proxy_error(failed_account, failed_exc) + await proxy._load_balancer.record_errors(failed_account, extra_error_count) + except Exception: + logger.warning( + "Failed to flush deferred compact HTTP 500 health account_id=%s request_id=%s", + failed_account.id, + request_id, + exc_info=True, + ) + for failed_account, failed_exc in proxy_pending: + try: + await proxy._handle_proxy_error(failed_account, failed_exc) + except Exception: + logger.warning( + "Failed to flush deferred compact proxy health account_id=%s request_id=%s", + failed_account.id, + request_id, + exc_info=True, + ) + + async def settle_compact_usage( + *, + api_key: ApiKeyData | None, + api_key_reservation: ApiKeyUsageReservationData | None, + response: CompactResponsePayload | None, + request_service_tier: str | None, + ) -> None: + nonlocal settlement_attempted + settlement_attempted = True + settlement_error: ProxyResponseError | None = None + try: + await proxy._settle_compact_api_key_usage( + api_key=api_key, + api_key_reservation=api_key_reservation, + response=response, + request_service_tier=request_service_tier, + ) + except ProxyResponseError as exc: + if exc.failure_phase != "usage_settlement" or not exc.reservation_released: + raise + settlement_error = exc + flush_task = asyncio.create_task( + flush_deferred_health(), + name=f"compact-deferred-health-{request_id}", + ) + cancellation_pending = False + while not flush_task.done(): + try: + await asyncio.shield(flush_task) + except asyncio.CancelledError: + cancellation_pending = True + except Exception: + break + try: + flush_task.result() + except Exception: + logger.warning( + "Failed to flush deferred compact account health request_id=%s", + request_id, + exc_info=True, + ) + if cancellation_pending: + raise asyncio.CancelledError() + if settlement_error is not None: + raise settlement_error + + async def settle_on_terminal_exit() -> None: + if settlement_attempted: + return + try: + await settle_compact_usage( + api_key=api_key, + api_key_reservation=api_key_reservation, + response=None, + request_service_tier=request_service_tier, + ) + except Exception: + logger.warning( + "Failed to settle compact reservation after unexpected exit request_id=%s", + request_id, + exc_info=True, + ) + + async def record_or_defer_proxy_health( + failed_account: Account, + failed_exc: ProxyResponseError, + ) -> None: + if api_key is not None and api_key_reservation is not None: + deferred_proxy_health.append((failed_account, failed_exc)) + return + await proxy._handle_proxy_error(failed_account, failed_exc) + + async def record_or_defer_stream_health( + failed_account: Account, + failed_error: Any, + failed_code: str, + failed_status: int | None = None, + ) -> None: + if api_key is not None and api_key_reservation is not None: + deferred_stream_health.append((failed_account, failed_error, failed_code, failed_status)) + return + await proxy._handle_stream_error( + failed_account, + failed_error, + failed_code, + http_status=failed_status, + ) + try: async def _call_compact( @@ -836,6 +1228,15 @@ async def _call_compact( last_exc: ProxyResponseError | None = None network_recovery = ProcessNetworkRecovery(transport="compact", request_id=request_id) excluded_account_ids: set[str] = set() + # Account-neutral replay off a pinned previous-response owner is only + # legal for owner loss the owner's quota state caused: either the + # owner was never usable at selection time, or it was excluded + # mid-request by a pre-visible quota / rate-limit failure. Post- + # selection authentication, refresh, transport, and transient + # failures also exclude the owner, and those keep their existing + # owner-bound handling instead of moving the history to another + # account. + owner_quota_failover_eligible = False require_security_work_authorized = False estimated_lease_tokens = _estimated_lease_tokens_from_request_usage_budget( estimate_api_key_request_usage(payload) @@ -891,6 +1292,136 @@ async def _call_compact( fallback_on_preferred_account_unavailable=preferred_account_id is None, ) account = selection.account + if ( + account is None + and previous_response_preferred_account_id is not None + and preferred_account_id == previous_response_preferred_account_id + ): + # Narrowed alias: the structural gate above proves the + # selection pin names the previous-response owner. + unavailable_owner_account_id = previous_response_preferred_account_id + # The pinned previous-response owner cannot be selected. + # A full resend that is provably account-neutral on the + # wire needs nothing from the owner, so the stale anchor + # can be dropped and the compact can move to a healthy + # account instead of wedging the session until the + # owner's quota window resets — the same selection-time + # escape normal turns already have. Turn-state and file + # pins keep the request owner-bound (the first blocked + # reason below), but their selection failure still + # records the fail-closed outcome on the common path. + recovery_blocked_reason: str | None = None + if turn_state_owner_account_id is not None or rewritten_file_account_id is not None: + # The previous-response owner is also pinned by a + # turn-state or input-file owner. Those pins are + # account ownership this recovery must never move, + # so the request stays owner-bound regardless of + # the owner's quota state — but the unavailable + # owner still fails closed and must be recorded. + recovery_blocked_reason = "additional_owner_pins" + elif previous_response_lookup_session_id is not None: + # A session/turn-state identity on the request can + # bind live or durable HTTP-bridge continuity rows + # that still name the lost owner. Without the + # rebinding machinery this recovery deliberately + # avoids, moving the history would strand that + # continuity, so session-scoped requests stay + # owner-bound. + recovery_blocked_reason = "session_scoped_continuity" + elif ( + affinity.kind == StickySessionKind.CODEX_SESSION + or affinity.legacy_selection_key is not None + or affinity.require_unambiguous_account + ): + # CODEX_SESSION affinity (turn-state, thread, or + # session-header keys, including raw legacy rows) + # is session ownership this recovery would have to + # rebind, so those requests stay owner-bound. + # PROMPT_CACHE / STICKY_THREAD keys are soft cache + # locality the sticky selection path already falls + # back from on an unavailable account — the compact + # routes derive one unconditionally — so they gate + # nothing here and the recovery reselection flows + # through that same existing sticky handling. + recovery_blocked_reason = "session_affinity" + elif ( + not owner_quota_failover_eligible + and selection.error_code == "preferred_account_unavailable" + ): + # The selector skipped the owner before evaluating + # its availability (API-key assignment scope, + # single-account routing, or an in-request + # exclusion that was not a pre-visible quota + # failover). Policy-caused loss must not become + # replay-eligible just because the owner's + # persisted status happens to be quota-exhausted. + recovery_blocked_reason = "owner_skipped_by_policy" + elif not ( + owner_quota_failover_eligible + or ( + unavailable_owner_account_id not in excluded_account_ids + and await proxy._compact_owner_selection_loss_is_quota_caused( + unavailable_owner_account_id + ) + ) + ): + recovery_blocked_reason = "non_quota_owner_loss" + replay_payload: ResponsesCompactRequest | None = None + if recovery_blocked_reason is None: + replay_payload = _compact_account_neutral_replay_payload(payload) + if replay_payload is None: + recovery_blocked_reason = "history_not_account_neutral" + if replay_payload is None: + logger.info( + "Compact previous-response owner unavailable; staying owner-bound " + "request_id=%s owner_account_id=%s blocked_reason=%s selection_error_code=%s", + request_id, + preferred_account_id, + recovery_blocked_reason, + selection.error_code, + ) + _record_continuity_fail_closed( + surface="compact", + reason="owner_account_unavailable", + previous_response_id=previous_response_id + if isinstance(previous_response_id, str) + else None, + session_id=previous_response_lookup_session_id, + upstream_error_code=selection.error_code, + ) + else: + logger.warning( + "Compact previous-response owner unavailable; replaying verified " + "account-neutral full resend request_id=%s owner_account_id=%s " + "selection_error_code=%s", + request_id, + preferred_account_id, + selection.error_code, + ) + excluded_account_ids.add(unavailable_owner_account_id) + payload = replay_payload + filtered = without_http_bridge_session_affinity_headers(filtered) + preferred_account_id = None + previous_response_preferred_account_id = None + selection = await proxy._select_account_with_budget_compatible( + deadline, + request_id=request_id, + kind="compact", + api_key=api_key, + affinity_policy=affinity, + prefer_earlier_reset_accounts=prefer_earlier_reset, + prefer_earlier_reset_window=_prefer_earlier_reset_window(settings), + routing_strategy=routing_strategy, + model=payload.model, + service_tier=payload.service_tier, + exclude_account_ids=excluded_account_ids, + preferred_account_id=None, + require_security_work_authorized=require_security_work_authorized, + lease_kind="response_create", + estimated_lease_tokens=estimated_lease_tokens, + fallback_on_preferred_account_unavailable=True, + ) + account = selection.account if account is not None: pass elif last_exc is not None: @@ -916,7 +1447,7 @@ async def _call_compact( # is the sole settler) the API-key reservation would leak held # quota. Settle BEFORE raising, mirroring the transport/permanent # preflight branches above. - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -934,7 +1465,7 @@ async def _call_compact( await proxy._load_balancer.release_account_lease(selected_account_response_create_lease) # Sole-settler leak guard (see above): settle the reservation # before this budget-exhausted terminal raise. - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -962,7 +1493,7 @@ async def _call_compact( # ensure_fresh_with_budget translates terminal process-network # recovery outcomes before the compact upstream settlement # branches run, so this boundary owns reservation cleanup. - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -983,7 +1514,7 @@ async def _call_compact( # reservation is finalized instead of leaking held # API-key quota (matching the post-401 permanent # branch, which settles before re-raising). - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1030,7 +1561,7 @@ async def _call_compact( # reservation. Settle it BEFORE raising so the # API-key reservation is finalized instead of leaking # held quota when the pinned refresh claim times out. - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1063,7 +1594,7 @@ async def _call_compact( # Settle BEFORE raising, mirroring the claim-contention and # post-401 transport branches. if not _should_retry_transient_stream_error("upstream_unavailable", message): - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1071,14 +1602,14 @@ async def _call_compact( ) _raise_proxy_unavailable(message) if preferred_account_id is not None: - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, request_service_tier=request_service_tier, ) _raise_proxy_unavailable(message) - await proxy._handle_stream_error( + await record_or_defer_stream_health( account, {"message": message}, "upstream_unavailable", @@ -1100,7 +1631,7 @@ async def _call_compact( await proxy._load_balancer.release_account_lease(selected_account_response_create_lease) # Sole-settler leak guard (see above): settle the reservation # before this budget-exhausted terminal raise. - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1121,7 +1652,7 @@ async def _call_compact( network_recovery.log_recovered() actual_service_tier = _service_tier_from_response(response) await proxy._load_balancer.record_success(account) - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=response, @@ -1134,7 +1665,7 @@ async def _call_compact( raise compact_continuity_error = _compact_previous_response_not_found_error(exc) if compact_continuity_error is not None: - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1151,9 +1682,9 @@ async def _call_compact( if exc.status_code == 401: if refresh_retry_used: try: - await proxy._handle_proxy_error(account, exc) + await record_or_defer_proxy_health(account, exc) except Exception: - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1179,7 +1710,7 @@ async def _call_compact( # the bridge/forwarded path (``owns_reservation`` # false) the reservation would leak held quota. # Settle BEFORE raising. - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1195,7 +1726,7 @@ async def _call_compact( # A translated refresh-recovery error escapes the # current upstream-error handler, so settle before # handing it to the request-level error boundary. - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1205,13 +1736,13 @@ async def _call_compact( except (RefreshError, aiohttp.ClientError, asyncio.TimeoutError) as refresh_exc: if isinstance(refresh_exc, RefreshError): if refresh_exc.is_permanent: - await proxy._load_balancer.mark_permanent_failure(account, refresh_exc.code) - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, request_service_tier=request_service_tier, ) + await proxy._load_balancer.mark_permanent_failure(account, refresh_exc.code) raise exc if is_transient_refresh_contention(refresh_exc): # Transient CROSS-REPLICA refresh contention @@ -1249,7 +1780,7 @@ async def _call_compact( exc_info=True, ) if preferred_account_id is not None: - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1266,7 +1797,7 @@ async def _call_compact( # Non-transport, non-permanent RefreshError # keeps its prior escalation: re-raise the # original 401 to the caller. - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1290,7 +1821,7 @@ async def _call_compact( exc_info=True, ) if not _should_retry_transient_stream_error("upstream_unavailable", message): - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1298,14 +1829,14 @@ async def _call_compact( ) _raise_proxy_unavailable(message) if preferred_account_id is not None: - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, request_service_tier=request_service_tier, ) _raise_proxy_unavailable(message) - await proxy._handle_stream_error( + await record_or_defer_stream_health( account, {"message": message}, "upstream_unavailable", @@ -1342,10 +1873,13 @@ async def _call_compact( account.id, transient_retries, ) - await proxy._handle_proxy_error(account, exc) - # Record remaining errors so total equals transient_retries, - # meeting the load balancer backoff threshold (error_count >= 3). - await proxy._load_balancer.record_errors(account, transient_retries - 1) + if api_key is not None and api_key_reservation is not None: + deferred_http_500_health.append((account, exc, transient_retries - 1)) + else: + await proxy._handle_proxy_error(account, exc) + # Record remaining errors so total equals transient_retries, + # meeting the load balancer backoff threshold (error_count >= 3). + await proxy._load_balancer.record_errors(account, transient_retries - 1) last_exc = exc excluded_account_ids.add(account.id) transient_exhausted = True @@ -1367,7 +1901,7 @@ async def _call_compact( if recovery_decision == "retry": continue if recovery_decision == "exhausted": - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1388,7 +1922,7 @@ async def _call_compact( require_security_work_authorized = True transient_exhausted = True break - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1401,7 +1935,7 @@ async def _call_compact( transient_exhausted = True break if _is_account_neutral_error_code(code): - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1409,7 +1943,7 @@ async def _call_compact( ) raise if code == "upstream_request_timeout": - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1459,11 +1993,11 @@ async def _call_compact( classified["failure_class"], ) raise - classified = await proxy._handle_stream_error( - account, - _upstream_error_from_openai(error), - code, + classified = classify_upstream_failure( + error_code=code, + error=_upstream_error_from_openai(error), http_status=exc.status_code, + phase="first_event", ) if getattr(base_settings, "deterministic_failover_enabled", True): action = failover_decision( @@ -1483,21 +2017,41 @@ async def _call_compact( action, ) if action == "failover_next": + if account.id == preferred_account_id and classified["failure_class"] in ( + "rate_limit", + "quota", + ): + # Only a pre-visible quota / rate-limit exclusion + # of the pinned owner makes account-neutral replay + # recovery eligible for the remaining attempts. + owner_quota_failover_eligible = True last_exc = exc excluded_account_ids.add(account.id) + await record_or_defer_stream_health( + account, + _upstream_error_from_openai(error), + code, + exc.status_code, + ) transient_exhausted = True break - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, request_service_tier=request_service_tier, ) + await proxy._handle_stream_error( + account, + _upstream_error_from_openai(error), + code, + http_status=exc.status_code, + ) raise if transient_exhausted: continue # outer loop: try different account # All account attempts exhausted — raise last error - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1510,6 +2064,7 @@ async def _call_compact( openai_error("upstream_unavailable", "All account attempts exhausted"), ) except ProxyResponseError as exc: + await settle_on_terminal_exit() failure_metadata = _request_log_failure_metadata(exc) error = _parse_openai_error(exc.payload) log_error_code = log_error_code or _normalize_error_code( @@ -1522,7 +2077,7 @@ async def _call_compact( route_fail_closed_reason = exc.reason log_error_code = "upstream_proxy_unavailable" log_error_message = exc.reason - await proxy._settle_compact_api_key_usage( + await settle_compact_usage( api_key=api_key, api_key_reservation=api_key_reservation, response=None, @@ -1532,6 +2087,9 @@ async def _call_compact( 502, openai_error("upstream_proxy_unavailable", f"Upstream proxy route unavailable: {exc.reason}"), ) from exc + except BaseException: + await settle_on_terminal_exit() + raise finally: usage = response.usage if response else None reasoning_effort = payload.reasoning.effort if payload.reasoning else None diff --git a/app/modules/proxy/_service/file_ops.py b/app/modules/proxy/_service/file_ops.py index acc0e39a13..411815427c 100644 --- a/app/modules/proxy/_service/file_ops.py +++ b/app/modules/proxy/_service/file_ops.py @@ -30,10 +30,14 @@ from app.db.models import Account from app.modules.api_keys.service import ApiKeyData from app.modules.proxy._service.support import ( - _FilePinEntry, _request_log_client_fields, _RequestLogFailureMetadata, ) +from app.modules.proxy.continuity import resolve_required_account_id +from app.modules.proxy.file_pin_repository import ( + FileAccountPinOwnershipConflict, + FileAccountPinRepository, +) from app.modules.proxy.helpers import _header_account_id, _normalize_error_code, _parse_openai_error from app.modules.proxy.load_balancer import AccountSelection from app.modules.proxy.selection_errors import selection_failure_response @@ -45,10 +49,9 @@ class _FileOpsServiceProtocol(Protocol): _encryptor: Any - _file_account_pin_lock: asyncio.Lock - _file_account_pins: dict[str, _FilePinEntry] + _file_pin_session_factory: Any _load_balancer: Any - _FILE_ACCOUNT_PIN_TTL_SECONDS: float + _FILE_ACCOUNT_PIN_TTL_SECONDS: int async def _select_account_with_budget_compatible(self, deadline: float, **kwargs: object) -> AccountSelection: ... async def _select_account_with_budget(self, deadline: float, **kwargs: Any) -> AccountSelection: ... @@ -65,8 +68,11 @@ async def _resolve_upstream_route_for_account( async def _proxy_files_call(self, **kwargs: Any) -> tuple[dict[str, JsonValue], str | None]: ... async def _pin_file_account(self, file_id: str, account_id: str) -> None: ... async def _resolve_file_account(self, file_id: str) -> str | None: ... - async def _lookup_file_pin(self, file_id: str) -> _FilePinEntry | None: ... - def _evict_expired_file_pins_locked(self) -> None: ... + async def _resolve_file_account_for_responses( + self, + payload: ResponsesRequest | ResponsesCompactRequest, + headers: Mapping[str, str], + ) -> str | None: ... def _service_core_create_file() -> Callable[..., Awaitable[dict[str, JsonValue]]]: @@ -157,17 +163,32 @@ def _routing_strategy(settings: Any) -> RoutingStrategy: _REQUEST_TRANSPORT_HTTP = "http" +class _FileOwnerPostSuccessError(RuntimeError): + def __init__(self, proxy_error: ProxyResponseError) -> None: + super().__init__("File owner persistence failed after a successful upstream call") + self.proxy_error = proxy_error + + +def _file_owner_unavailable_error() -> ProxyResponseError: + return ProxyResponseError( + 502, + openai_error( + "file_owner_unavailable", + "Input file owner metadata is unavailable; upload the file again and retry.", + error_type="server_error", + ), + ) + + class _FileOpsMixin: # File-account pin TTL: long enough to cover a slow client-side # PUT of a 512 MiB upload (the upstream limit) plus the finalize # poll loop and a follow-up ``/responses`` that references the - # file_id, while still bounding how long stale pins can sit in - # memory on long-lived workers. 30 minutes covers a 512 MiB - # upload at ~280 KiB/s -- well below typical broadband uplink -- - # while keeping the table size negligible (each pin is a short - # string tuple). Eviction runs opportunistically on every write, - # so this acts as an upper bound, not a fixed retention. - _FILE_ACCOUNT_PIN_TTL_SECONDS: float = 30 * 60.0 + # file_id, while still bounding how long stale pins remain in + # shared storage. 30 minutes covers a 512 MiB + # upload at ~280 KiB/s -- well below typical broadband uplink. + # The database clock defines both expiry and opportunistic cleanup. + _FILE_ACCOUNT_PIN_TTL_SECONDS: int = 30 * 60 async def _pin_file_account( self, @@ -177,74 +198,48 @@ async def _pin_file_account( """Remember that ``file_id`` was registered through ``account_id``. Used so a subsequent ``finalize_file`` can be routed to the same - account that created the file. Cross-instance handoff is - best-effort: if the finalize lands on a different replica with - no pin, we fall back to a fresh load-balancer selection. + account that created the file, including when another replica + handles the follow-up request. """ proxy = cast(_FileOpsServiceProtocol, self) if not file_id or not account_id: return - expires_at = time.monotonic() + proxy._FILE_ACCOUNT_PIN_TTL_SECONDS - async with proxy._file_account_pin_lock: - proxy._file_account_pins[file_id] = _FilePinEntry( - account_id=account_id, - expires_at=expires_at, - ) - proxy._evict_expired_file_pins_locked() + try: + async with proxy._file_pin_session_factory() as session: + await FileAccountPinRepository(session).claim( + file_id, + account_id, + ttl_seconds=proxy._FILE_ACCOUNT_PIN_TTL_SECONDS, + ) + except FileAccountPinOwnershipConflict as exc: + raise ProxyResponseError( + 502, + openai_error( + "continuity_owner_conflict", + "File ownership conflicts with an existing live upload.", + error_type="server_error", + ), + ) from exc + except Exception as exc: + raise _file_owner_unavailable_error() from exc async def _resolve_file_account(self, file_id: str) -> str | None: """Return the pinned account_id for ``file_id`` if still live.""" - proxy = cast(_FileOpsServiceProtocol, self) - entry = await proxy._lookup_file_pin(file_id) - return entry.account_id if entry is not None else None - - async def _lookup_file_pin(self, file_id: str) -> _FilePinEntry | None: proxy = cast(_FileOpsServiceProtocol, self) if not file_id: return None - async with proxy._file_account_pin_lock: - proxy._evict_expired_file_pins_locked() - entry = proxy._file_account_pins.get(file_id) - if entry is None: - return None - if entry.expires_at <= time.monotonic(): - proxy._file_account_pins.pop(file_id, None) - return None - return entry - - def _evict_expired_file_pins_locked(self) -> None: - """Drop pins past their TTL. Called under ``_file_account_pin_lock``.""" - proxy = cast(_FileOpsServiceProtocol, self) - now = time.monotonic() - expired = [file_id for file_id, entry in proxy._file_account_pins.items() if entry.expires_at <= now] - for file_id in expired: - proxy._file_account_pins.pop(file_id, None) + try: + async with proxy._file_pin_session_factory() as session: + return await FileAccountPinRepository(session).get_live_account_id(file_id) + except Exception as exc: + raise _file_owner_unavailable_error() from exc async def _resolve_file_account_for_responses( self, payload: ResponsesRequest | ResponsesCompactRequest, headers: Mapping[str, str], ) -> str | None: - """Resolve a ``preferred_account_id`` from ``input_file.file_id`` pins. - - Looks up the in-memory ``file_id -> account_id`` pin table built - by ``create_file``. Used by ``/responses`` flows so a request - carrying an ``{type: "input_file", file_id: "file_xxx"}`` part - is routed to the same upstream account that registered the - upload (the upstream contract is account-scoped via - ``chatgpt-account-id``). - - Live pins are account ownership evidence. Partial pin coverage and - files owned by different accounts fail closed; choosing a "best" - attachment would send at least one account-scoped object to the wrong - upstream account. When none of the IDs has a live pin, preserve the - established opaque-ID compatibility path and leave routing unpinned. - - Other hard sources are intentionally not inspected here. Callers merge - this result with previous-response, turn-state, or bridge ownership at - the shared continuity boundary so conflicts cannot hide behind source - precedence. - """ + """Resolve a ``preferred_account_id`` from durable ``input_file.file_id`` pins.""" proxy = cast(_FileOpsServiceProtocol, self) del headers @@ -255,18 +250,17 @@ async def _resolve_file_account_for_responses( if not file_ids: return None - async with proxy._file_account_pin_lock: - proxy._evict_expired_file_pins_locked() - entries = [proxy._file_account_pins.get(file_id) for file_id in file_ids] - - pinned_entries = [entry for entry in entries if entry is not None] - if not pinned_entries: - # A raw file_id may have been registered directly with upstream or - # before this replica observed the upload. With zero local proof, - # it remains an opaque compatibility reference rather than a hard - # owner; callers forward it verbatim under ordinary routing. + try: + async with proxy._file_pin_session_factory() as session: + account_ids_by_file_id = await FileAccountPinRepository(session).get_live_account_ids(file_ids) + except Exception as exc: + raise _file_owner_unavailable_error() from exc + resolved_account_ids = [account_ids_by_file_id.get(file_id) for file_id in file_ids] + + pinned_account_ids = [account_id for account_id in resolved_account_ids if account_id is not None] + if not pinned_account_ids: return None - if len(pinned_entries) != len(entries): + if len(pinned_account_ids) != len(resolved_account_ids): raise ProxyResponseError( 502, openai_error( @@ -275,7 +269,7 @@ async def _resolve_file_account_for_responses( error_type="server_error", ), ) - owner_account_ids = {entry.account_id for entry in pinned_entries} + owner_account_ids = set(pinned_account_ids) if len(owner_account_ids) != 1: raise ProxyResponseError( 502, @@ -287,6 +281,30 @@ async def _resolve_file_account_for_responses( ) return next(iter(owner_account_ids)) + async def _resolve_forwarded_file_account_for_responses( + self, + payload: ResponsesRequest | ResponsesCompactRequest, + headers: Mapping[str, str], + *, + forwarded_file_owner_account_id: str | None, + require_forwarded_file_owner: bool = False, + ) -> str | None: + """Revalidate signed bridge ownership against the shared database.""" + proxy = cast(_FileOpsServiceProtocol, self) + durable_owner_account_id = await proxy._resolve_file_account_for_responses(payload, headers) + if ( + require_forwarded_file_owner + and durable_owner_account_id is not None + and forwarded_file_owner_account_id is None + ): + raise _file_owner_unavailable_error() + if forwarded_file_owner_account_id is not None and durable_owner_account_id is None: + raise _file_owner_unavailable_error() + return resolve_required_account_id( + ("signed forwarding context", forwarded_file_owner_account_id), + ("durable file pin", durable_owner_account_id), + ) + def _raise_for_unsupported_input_image_references(self, payload: _ResponsesPayloadT) -> None: references = extract_input_image_file_references(payload.input) if not references: @@ -325,7 +343,13 @@ async def create_file( fail with not-found / unauthorized. """ proxy = cast(_FileOpsServiceProtocol, self) - result, account_id = await proxy._proxy_files_call( + + async def persist_file_owner(result: dict[str, JsonValue], account_id: str) -> None: + file_id = result.get("file_id") + if isinstance(file_id, str) and file_id: + await proxy._pin_file_account(file_id, account_id) + + result, _account_id = await proxy._proxy_files_call( log_model="files-create", kind="files-create", api_key=api_key, @@ -341,12 +365,8 @@ async def create_file( route_trace=route_trace, ) ), + on_success=persist_file_owner, ) - # Best-effort pin so finalize lands on the same account. - if isinstance(result, dict) and account_id: - file_id = result.get("file_id") - if isinstance(file_id, str) and file_id: - await proxy._pin_file_account(file_id, account_id) return result async def finalize_file( @@ -364,20 +384,26 @@ async def finalize_file( verbatim. Routes to the account that handled the matching ``create_file`` - (via the in-memory pin table) so the upstream finalize call + (via the durable pin table) so the upstream finalize call carries the same ``chatgpt-account-id`` that registered the file. Falls back to a fresh load-balancer selection when no - pin is found (unknown ``file_id`` or pin expired / missed across - a replica boundary). + pin is found (unknown ``file_id`` or an expired pin). """ proxy = cast(_FileOpsServiceProtocol, self) - pinned_account_id = await proxy._resolve_file_account(file_id) - result, account_id = await proxy._proxy_files_call( + + async def resolve_file_owner() -> str | None: + return await proxy._resolve_file_account(file_id) + + async def persist_file_owner(result: dict[str, JsonValue], account_id: str) -> None: + if result.get("status") == "success": + await proxy._pin_file_account(file_id, account_id) + + result, _account_id = await proxy._proxy_files_call( log_model="files-finalize", kind="files-finalize", api_key=api_key, headers=headers, - preferred_account_id=pinned_account_id, + resolve_preferred_account_id=resolve_file_owner, invoke=lambda access_token, upstream_account_id, filtered_headers, route, route_trace: ( _service_core_finalize_file()( file_id=file_id, @@ -389,11 +415,8 @@ async def finalize_file( route_trace=route_trace, ) ), + on_success=persist_file_owner, ) - if isinstance(result, dict) and account_id: - status = result.get("status") - if status == "success": - await proxy._pin_file_account(file_id, account_id) return result async def _proxy_files_call( @@ -408,6 +431,8 @@ async def _proxy_files_call( Awaitable[dict[str, JsonValue]], ], preferred_account_id: str | None = None, + resolve_preferred_account_id: Callable[[], Awaitable[str | None]] | None = None, + on_success: Callable[[dict[str, JsonValue], str], Awaitable[None]] | None = None, ) -> tuple[dict[str, JsonValue], str | None]: """Shared account-selection / refresh / 401-retry plumbing for `/files` calls. @@ -415,9 +440,11 @@ async def _proxy_files_call( ensure freshness, invoke upstream, on 401 force-refresh and retry once, translate ``FileProxyError`` -> ``ProxyResponseError``, and always write a request-log entry on the way out. When - ``preferred_account_id`` is provided (e.g. from the file_id pin - for ``finalize_file``), the call is strict to that account and - fails closed when the owner account is unavailable. + ``preferred_account_id`` is provided or resolved (e.g. from the file_id + pin for ``finalize_file``), the call is strict to that account and + fails closed when the owner account is unavailable. ``on_success`` runs + before the request is logged or returned so durable owner persistence + remains part of the route's success contract. """ proxy = cast(_FileOpsServiceProtocol, self) filtered = filter_inbound_headers(headers) @@ -437,10 +464,23 @@ async def _proxy_files_call( route_fallback_used: bool | None = None route_fail_closed_reason: str | None = None - settings = await _service_get_settings_cache().get() - prefer_earlier_reset = settings.prefer_earlier_reset_accounts - routing_strategy = _routing_strategy(settings) try: + if resolve_preferred_account_id is not None: + preferred_account_id = await resolve_preferred_account_id() + settings = await _service_get_settings_cache().get() + prefer_earlier_reset = settings.prefer_earlier_reset_accounts + routing_strategy = _routing_strategy(settings) + + async def _persist_success(result: dict[str, JsonValue], account_id: str) -> None: + if on_success is None: + return + try: + await on_success(result, account_id) + except ProxyResponseError as exc: + raise _FileOwnerPostSuccessError(exc) from exc + except Exception as exc: + raise _FileOwnerPostSuccessError(_file_owner_unavailable_error()) from exc + selection = await proxy._select_account_with_budget_compatible( deadline, request_id=request_id, @@ -530,6 +570,7 @@ async def _select_files_failover(excluded_account_ids: set[str]) -> AccountSelec account_id_value = account.id result = await _call(account) await proxy._load_balancer.record_success(account) + await _persist_success(result, account.id) log_status = "success" return result, account_id_value except RefreshError as refresh_exc: @@ -558,6 +599,7 @@ async def _select_files_failover(excluded_account_ids: set[str]) -> AccountSelec if failover is not None: account, result = failover account_id_value = account.id + await _persist_success(result, account.id) log_status = "success" return result, account_id_value failed_account = _proxy_response_failed_account(exc, account) @@ -612,6 +654,7 @@ async def _select_files_failover(excluded_account_ids: set[str]) -> AccountSelec # caller's pin is consistent with the upstream call. account_id_value = account.id await proxy._load_balancer.record_success(account) + await _persist_success(result, account.id) log_status = "success" return result, account_id_value except ProxyResponseError as retry_exc: @@ -639,12 +682,23 @@ async def _select_files_failover(excluded_account_ids: set[str]) -> AccountSelec try: result = await _call(account) await proxy._load_balancer.record_success(account) + await _persist_success(result, account.id) log_status = "success" return result, account_id_value except ProxyResponseError as failover_exc: await proxy._handle_proxy_error(account, failover_exc) raise raise + except _FileOwnerPostSuccessError as exc: + proxy_error = exc.proxy_error + failure_metadata = _request_log_failure_metadata(proxy_error) + error = _parse_openai_error(proxy_error.payload) + log_error_code = _normalize_error_code( + error.code if error else None, + error.type if error else None, + ) + log_error_message = error.message if error else None + raise proxy_error from exc except ProxyResponseError as exc: failed_account = getattr(exc, _FAILED_ACCOUNT_ATTR, None) if isinstance(failed_account, Account): diff --git a/app/modules/proxy/_service/http_bridge/account_sessions.py b/app/modules/proxy/_service/http_bridge/account_sessions.py index 69961ec8bf..6069c1cd3c 100644 --- a/app/modules/proxy/_service/http_bridge/account_sessions.py +++ b/app/modules/proxy/_service/http_bridge/account_sessions.py @@ -8,6 +8,7 @@ class _HTTPBridgeAccountSessionsMixin: async def close_http_bridge_sessions_for_account(self: _HTTPBridgeServiceProtocol, account_id: str) -> int: sessions_to_close: list[_HTTPBridgeSession] = [] + scheduled_session_ids: set[int] = set() async with self._http_bridge_lock: for key, session in tuple(self._http_bridge_sessions.items()): if session.account.id != account_id: @@ -24,6 +25,32 @@ async def close_http_bridge_sessions_for_account(self: _HTTPBridgeServiceProtoco model_class=_extract_model_class(session.request_model) if session.request_model else None, ) sessions_to_close.append(detached) + scheduled_session_ids.add(id(detached)) + # Detached predecessors still own authenticated sockets and account + # leases. Account invalidation must fence them even though a newer + # generation occupies (or has vacated) their canonical key. + for session in tuple(self._http_bridge_detached_sessions.values()): + if session.account.id != account_id or id(session) in scheduled_session_ids: + continue + close_task = session.resource_close_task + if close_task is not None and ( + not close_task.done() or (not close_task.cancelled() and close_task.exception() is None) + ): + # ``closed`` only rejects admission. A live close task (or a + # successfully completed one awaiting registry finalization) + # is the proof that this detached generation is already owned. + continue + session.closed = True + _log_http_bridge_event( + "evict_account_binding_changed", + session.key, + account_id=session.account.id, + model=session.request_model, + cache_key_family=session.key.affinity_kind, + model_class=_extract_model_class(session.request_model) if session.request_model else None, + ) + sessions_to_close.append(session) + scheduled_session_ids.add(id(session)) for session in sessions_to_close: await self._close_http_bridge_session_bounded(session, reason="account_binding_changed") diff --git a/app/modules/proxy/_service/http_bridge/activity.py b/app/modules/proxy/_service/http_bridge/activity.py index 10cce3d065..08f13a1eb4 100644 --- a/app/modules/proxy/_service/http_bridge/activity.py +++ b/app/modules/proxy/_service/http_bridge/activity.py @@ -1,9 +1,13 @@ from __future__ import annotations +import asyncio from typing import Any +from app.core.clients.proxy import ProxyResponseError +from app.core.resilience.overload import local_overload_error from app.modules.proxy._service.http_bridge.helpers import ( _close_http_bridge_session_bounded, + _http_bridge_capacity_generation_count, _http_bridge_pending_count_nowait, _http_bridge_pending_state_is_stale, _http_bridge_request_counts_against_queue, @@ -13,7 +17,11 @@ http_bridge_activity_snapshot_nowait, ) from app.modules.proxy._service.http_bridge.protocol import _HTTPBridgeServiceProtocol -from app.modules.proxy._service.support import _http_bridge_session_supports_service_tier, _HTTPBridgeSession +from app.modules.proxy._service.support import ( + _http_bridge_session_supports_service_tier, + _HTTPBridgeSession, + _HTTPBridgeSessionKey, +) from app.modules.proxy.affinity import _extract_model_class @@ -77,6 +85,75 @@ async def _close_http_bridge_session_bounded( ) -> None: await _close_http_bridge_session_bounded(self, session, reason=reason) + def _http_bridge_active_capacity_error( + self: _HTTPBridgeServiceProtocol, + *, + key: _HTTPBridgeSessionKey, + request_model: str | None, + ) -> ProxyResponseError: + _log_http_bridge_event( + "capacity_exhausted_active_sessions", + key, + account_id=None, + model=request_model, + pending_count=_http_bridge_capacity_generation_count(self), + cache_key_family=key.affinity_kind, + model_class=_extract_model_class(request_model) if request_model else None, + ) + return ProxyResponseError( + 429, + local_overload_error( + "HTTP responses session bridge has no idle capacity", + code="capacity_exhausted_active_sessions", + ), + ) + + def _http_bridge_forced_close_must_finish_before_create( + self: _HTTPBridgeServiceProtocol, + forced_replacement: bool, + max_sessions: int, + ) -> bool: + # Detachment retains capacity. A forced replacement at the cap must + # finish closing its idle predecessor before enforcing the same cap. + return forced_replacement and _http_bridge_capacity_generation_count(self) >= max_sessions + + async def _enforce_http_bridge_capacity_after_planned_closes( + self: _HTTPBridgeServiceProtocol, + *, + key: _HTTPBridgeSessionKey, + inflight_future: asyncio.Future[_HTTPBridgeSession] | None, + max_sessions: int, + request_model: str | None, + ) -> None: + assert inflight_future is not None + async with self._http_bridge_lock: + if ( + self._http_bridge_inflight_sessions.get(key) is not inflight_future + or _http_bridge_capacity_generation_count(self) <= max_sessions + ): + return + # Planned evictions are discounted only to reserve this creation + # slot. A bounded close may return on timeout while the detached + # socket and leases remain live, so registry ownership wins here. + _log_http_bridge_event( + "capacity_exhausted_after_lru_close", + key, + account_id=None, + model=request_model, + pending_count=_http_bridge_capacity_generation_count(self), + cache_key_family=key.affinity_kind, + model_class=_extract_model_class(request_model) if request_model else None, + ) + capacity_error = ProxyResponseError( + 429, + local_overload_error( + "HTTP responses session bridge has no idle capacity", + code="capacity_exhausted_active_sessions", + ), + ) + await self._fail_http_bridge_inflight_session_creation(key, inflight_future, capacity_error) + raise capacity_error + async def _http_bridge_pending_count( self: _HTTPBridgeServiceProtocol, session: _HTTPBridgeSession, diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index 44c7b758a6..bf19ab42f9 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -6,7 +6,7 @@ import sys import time from collections.abc import Callable, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from hashlib import sha256 from ipaddress import ip_address from typing import Any, Literal, Mapping, TypeVar, cast @@ -69,7 +69,6 @@ from app.db.models import ( AccountStatus, DashboardSettings, - HttpBridgeSessionState, StickySessionKind, ) from app.modules.api_keys.service import ( @@ -126,6 +125,8 @@ _REQUEST_TRANSPORT_HTTP, _WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS, # noqa: F401 _http_bridge_session_supports_service_tier, + _HTTPBridgeResponseCreateAttempt, + _HTTPBridgeRetryCircuitAttemptSelection, _HTTPBridgeSession, _HTTPBridgeSessionKey, _WebSocketRequestState, @@ -166,6 +167,7 @@ from app.modules.proxy.account_cache import is_account_routing_unavailable from app.modules.proxy.affinity import ( _AffinityPolicy, + _codex_backend_identity, _extract_model_class, _sticky_key_from_session_header, _sticky_key_from_turn_state_header, @@ -181,15 +183,16 @@ DurableBridgeAliasRegistration, DurableBridgeAliasRegistrationReceipt, ) +from app.modules.proxy.durable_bridge_runtime import http_bridge_owner_process_epoch from app.modules.proxy.helpers import ( _normalize_error_code, _parse_openai_error, ) -from app.modules.proxy.response_transition_manifest import ResponseTransitionManifest from app.modules.proxy.ring_membership import ( RING_STALE_THRESHOLD_SECONDS, RingMembershipService, ) +from app.modules.proxy.selection_errors import selection_failure_response logger = logging.getLogger("app.modules.proxy.service") _TASK_CANCEL_TIMEOUT_SECONDS = 1.0 @@ -424,17 +427,78 @@ def _http_bridge_inflight_creation_count(service: Any) -> int: ) +def _http_bridge_session_generation_count(service: Any) -> int: + return len(service._http_bridge_sessions) + len(service._http_bridge_detached_sessions) + + +def _http_bridge_capacity_generation_count(service: Any) -> int: + return _http_bridge_session_generation_count(service) + _http_bridge_inflight_creation_count(service) + + +def _http_bridge_capacity_after_planned_closes( + service: Any, + sessions_to_close_before_create: Sequence["_HTTPBridgeSession"], +) -> int: + return _http_bridge_capacity_generation_count(service) - len(sessions_to_close_before_create) + + +def _plan_http_bridge_lru_capacity_closes( + service: Any, + *, + max_sessions: int, + model_transition_parent_key: "_HTTPBridgeSessionKey | None", + sessions_to_close_before_create: list["_HTTPBridgeSession"], +) -> None: + while ( + _http_bridge_capacity_after_planned_closes(service, sessions_to_close_before_create) >= max_sessions + and service._http_bridge_sessions + ): + evictable_sessions: list[tuple[_HTTPBridgeSessionKey, _HTTPBridgeSession]] = [] + for candidate_key, candidate_session in service._http_bridge_sessions.items(): + if candidate_key == model_transition_parent_key: + continue + if getattr(candidate_session, "unanchored_reservation_id", None) is not None: + continue + pending_count = service._http_bridge_pending_count_nowait( + candidate_session, + context="capacity_evict_scan", + ) + if pending_count is None or pending_count: + continue + evictable_sessions.append((candidate_key, candidate_session)) + if not evictable_sessions: + break + lru_key, lru_session = min(evictable_sessions, key=lambda item: _http_bridge_eviction_priority(item[1])) + _log_http_bridge_event( + "evict_lru", + lru_key, + account_id=lru_session.account.id, + model=lru_session.request_model, + cache_key_family=lru_key.affinity_kind, + model_class=_extract_model_class(lru_session.request_model) if lru_session.request_model else None, + ) + detached = service._detach_http_bridge_session_locked(lru_key, expected_session=lru_session) + if detached is not None: + sessions_to_close_before_create.append(detached) + + def http_bridge_activity_snapshot_nowait(service: Any) -> dict[str, int | bool]: inflight_cleanup = _cleanup_http_bridge_inflight_sessions_nowait(service) live_sessions = 0 pending_or_queued_requests = 0 pending_unknown_sessions = 0 - for session in list(service._http_bridge_sessions.values()): - if session.closed and not _http_bridge_session_has_admission_waiter(session): - continue + # A canonical key names only the newest generation. Detached predecessors + # remain live work and must still block restart until their requests settle. + for session in [ + *service._http_bridge_sessions.values(), + *service._http_bridge_detached_sessions.values(), + ]: if not session.closed: live_sessions += 1 + # `closed` fences new admission; it does not prove that a detached + # request has finished queue/pending settlement. Drain must observe + # that work even after the generation stops counting as a live socket. pending_count = _http_bridge_pending_count_nowait(session, context="drain_status") if pending_count is None: pending_unknown_sessions += 1 @@ -752,6 +816,52 @@ def _http_bridge_eventless_precreated_deadline( ) +def _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + request_states: Sequence[_WebSocketRequestState], +) -> _HTTPBridgeRetryCircuitAttemptSelection: + eligible_attempts: list[_HTTPBridgeResponseCreateAttempt] = [] + recorded_attempts: list[_HTTPBridgeResponseCreateAttempt] = [] + settled_attempts: list[_HTTPBridgeResponseCreateAttempt] = [] + attempt_seen = False + for request_state in request_states: + attempt = getattr(request_state, "response_create_attempt", None) + if attempt is None: + continue + attempt_seen = True + if attempt.retry_circuit_failure_recorded: + recorded_attempts.append(attempt) + continue + if attempt.disarmed or attempt.response_observed: + settled_attempts.append(attempt) + continue + if ( + request_state.transport != _REQUEST_TRANSPORT_HTTP + or request_state.skip_request_log + or request_state.response_id is not None + or request_state.latency_response_created_ms is not None + or request_state.response_event_count != 0 + or request_state.downstream_visible + ): + continue + eligible_attempts.append(attempt) + + for kind, attempts in ( + ("eligible", eligible_attempts), + ("recorded", recorded_attempts), + ("settled", settled_attempts), + ): + unique_attempts: list[_HTTPBridgeResponseCreateAttempt] = [] + for attempt in attempts: + if not any(candidate is attempt for candidate in unique_attempts): + unique_attempts.append(attempt) + if unique_attempts: + return _HTTPBridgeRetryCircuitAttemptSelection( + kind=kind, + attempts=tuple(unique_attempts), + ) + return _HTTPBridgeRetryCircuitAttemptSelection(kind="ineligible" if attempt_seen else "absent") + + def _http_bridge_session_has_admission_waiter(session: object | None) -> bool: """Keep a closed bridge registered while an unsent request owns its handoff.""" return session is not None and bool(getattr(session, "admission_waiter_count", 0)) @@ -763,7 +873,100 @@ def _http_bridge_session_has_visible_requests(session: "_HTTPBridgeSession") -> ) -async def _close_http_bridge_session( +async def _raise_if_http_bridge_creation_superseded( + service: Any, + key: "_HTTPBridgeSessionKey", + *, + inflight_future: Any, +) -> None: + """Abort a creation whose inflight slot another creator already took. + + An evicted creator has lost the registry slot, so claiming would only + advance the durable epoch past the session that won — fencing that + winner's own renewals out of a row it legitimately owns (issue #1695). + Failing here leaves the row untouched; the caller's failure path then + closes this session without releasing the winner's row. + """ + async with service._http_bridge_lock: + superseded = service._http_bridge_inflight_sessions.get(key) is not inflight_future + if superseded: + raise _http_bridge_startup_wait_timeout_error( + "http_bridge_session_registration", + code="capacity_exhausted_active_sessions", + ) + + +async def _settle_failed_http_bridge_creation( + service: Any, + key: "_HTTPBridgeSessionKey", + *, + inflight_future: Any, + created_session: "_HTTPBridgeSession | None", + exc: BaseException, +) -> bool: + """Retire a failed creation and report whether another session replaced it. + + A rejected creator claimed the durable row last, so its epoch is the + current one and its fenced release WOULD succeed — closing the row out + from under the session that actually won the registry slot (issue #1695). + The caller uses the return value to skip the durable release in that case. + """ + async with service._http_bridge_lock: + current_future = service._http_bridge_inflight_sessions.get(key) + replacement_in_flight = current_future is not None and current_future is not inflight_future + if current_future is inflight_future: + service._http_bridge_inflight_sessions.pop(key, None) + if inflight_future is not None and not inflight_future.done(): + if isinstance(exc, asyncio.CancelledError): + inflight_future.cancel() + else: + inflight_future.set_exception(exc) + inflight_future.exception() + registered_session = service._http_bridge_sessions.get(key) + # A replacement that has claimed but not yet published its session is + # just as much the winner as a registered one: releasing here would + # close the row beneath it, and it would then register with an older + # epoch and be fenced out on its first renewal (issue #1695). + registered_winner = ( + registered_session if registered_session is not None and registered_session is not created_session else None + ) + # A registered winner on a DIFFERENT account no longer shares this row: + # our claim already rewrote its account binding and cleared its + # continuity aliases, so preserving the row would leave it bound to our + # account while that session keeps dispatching. Release it instead — + # the winner is then fenced promptly and retries cleanly. + winner_shares_row = registered_winner is not None and ( + created_session is None or registered_winner.account.id == created_session.account.id + ) + superseded = replacement_in_flight or winner_shares_row + if superseded and registered_session is not None and created_session is not None: + # Eviction can still land DURING the claim, so this creator may + # have advanced the shared row's epoch past the session that won + # the slot, fencing the winner's own renewals. Both sessions are + # this instance's and point at the same row, so hand the epoch we + # won over to the winner rather than leaving it stranded. + assert registered_session is not None + claimed_id = created_session.durable_session_id + claimed_epoch = created_session.durable_owner_epoch + registered_epoch = registered_session.durable_owner_epoch + if ( + claimed_id is not None + and claimed_epoch is not None + and registered_session.durable_session_id == claimed_id + and (registered_epoch is None or claimed_epoch > registered_epoch) + # Only when both sessions selected the same account: a claim + # rewrites the row's account_id and clears continuity aliases, + # so handing the epoch across an account change would leave the + # winner renewing and dispatching on a row bound to a different + # account. Fail closed there instead — the winner's renewal is + # fenced, it is evicted, and the request retries cleanly. + and created_session.account.id == registered_session.account.id + ): + registered_session.durable_owner_epoch = claimed_epoch + return superseded + + +async def _close_http_bridge_session_resources( service: Any, session: "_HTTPBridgeSession", *, @@ -836,6 +1039,59 @@ async def _close_http_bridge_session( ) +async def _close_http_bridge_session( + service: Any, + session: "_HTTPBridgeSession", + *, + turn_state_lock_held: bool = False, + release_durable_session: bool = True, +) -> None: + # Direct close callers can be cancelled just like the bounded background + # wrapper. Keep the resource owner alive until its reader, socket, and + # leases are actually finalized; only then may detached-capacity tracking + # disappear. Otherwise cancellation can turn a live predecessor into an + # unowned generation that neither shutdown nor capacity accounting sees. + def resource_close_task() -> asyncio.Task[None]: + existing = session.resource_close_task + if existing is not None and ( + not existing.done() or (not existing.cancelled() and existing.exception() is None) + ): + return existing + created = asyncio.create_task( + _close_http_bridge_session_resources( + service, + session, + turn_state_lock_held=turn_state_lock_held, + release_durable_session=release_durable_session, + ), + name=f"http-bridge-resource-close-{_hash_identifier(session.key.affinity_key)}", + ) + session.resource_close_task = created + return created + + # Close callers can race (reader retirement, account invalidation, and + # shutdown). Install one resource owner under the registry lock so leases + # and pending settlement are finalized exactly once. + if turn_state_lock_held: + close_task = resource_close_task() + else: + async with service._http_bridge_lock: + close_task = resource_close_task() + _, cancellation = await _await_task_deferring_cancellation(close_task) + # Detached generations remain capacity owners until resource closure ends. + # Finalize that ownership here so direct error-recovery closes and bounded + # background closes cannot drift into different lifecycles. + if turn_state_lock_held: + if service._http_bridge_detached_sessions.get(id(session)) is session: + service._http_bridge_detached_sessions.pop(id(session), None) + else: + async with service._http_bridge_lock: + if service._http_bridge_detached_sessions.get(id(session)) is session: + service._http_bridge_detached_sessions.pop(id(session), None) + if cancellation is not None: + raise cancellation + + async def _close_http_bridge_session_bounded( service: Any, session: "_HTTPBridgeSession", @@ -844,6 +1100,7 @@ async def _close_http_bridge_session_bounded( ) -> None: if session.upstream_reader is asyncio.current_task(): session.upstream_reader = None + close_task = asyncio.create_task( service._close_http_bridge_session(session), name=f"http-bridge-close-{_hash_identifier(session.key.affinity_key)}", @@ -961,6 +1218,7 @@ def _http_bridge_incompatible_model_fork_key( ) -> "_HTTPBridgeSessionKey | None": if key.affinity_kind not in { "session_header", + "thread_header", "turn_state_header", "internal_unanchored_parallel", "internal_model_parallel", @@ -1025,11 +1283,24 @@ def _http_bridge_parallel_fork_key( request_scope_id: str, allow_model_fork: bool = True, same_model_required: bool = False, + force_canonical_replacement: bool = False, ) -> "_HTTPBridgeSessionKey | None": """Give incompatible or concurrent requests an independent websocket lane.""" + if force_canonical_replacement: + if session is not None: + # A restart must replace the canonical session-header lane. Forking + # would leave the old owner reusable after the restart moved affinity. + session.upstream_control.reconnect_requested = True + session.upstream_control.retire_after_drain = True + return None + reason: str | None = None - if key.affinity_kind == "session_header" and incoming_turn_state is None and previous_response_id is None: + if ( + key.affinity_kind in {"session_header", "thread_header"} + and incoming_turn_state is None + and previous_response_id is None + ): if inflight_creation: reason = "session_creation_inflight" elif session is not None and not session.closed: @@ -1107,7 +1378,11 @@ def _http_bridge_request_needs_unanchored_handoff( ) -> bool: if forwarded_request: return forwarded_original_request_unanchored - return key.affinity_kind == "session_header" and incoming_turn_state is None and previous_response_id is None + return ( + key.affinity_kind in {"session_header", "thread_header"} + and incoming_turn_state is None + and previous_response_id is None + ) def _reserve_http_bridge_unanchored_handoff( @@ -1165,7 +1440,11 @@ async def _refresh_reused_http_bridge_session_with_handoff( def _http_bridge_session_retiring_with_visible_requests(session: "_HTTPBridgeSession") -> bool: - return session.upstream_control.retire_after_drain and _http_bridge_session_has_visible_requests(session) + # A reserved handoff is not queued yet, but it owns the lane just as a + # visible request does and must be allowed to submit before retirement. + return session.upstream_control.retire_after_drain and ( + _http_bridge_session_has_visible_requests(session) or session.unanchored_reservation_id is not None + ) def _http_bridge_payload_looks_like_full_resend(payload: ResponsesRequest) -> bool: @@ -1402,7 +1681,7 @@ def _record_http_bridge_handoff_compatibility_rejection( _hash_identifier_or_none(preferred_account_id), require_preferred_account, service_tier, - api_key_scope, + "", session.closed, getattr(session, "admission_waiter_count", 0), len(session.pending_requests), @@ -1475,14 +1754,19 @@ def _make_http_bridge_session_key( affinity_key = turn_state_key affinity_kind = "turn_state_header" strength: Literal["hard", "soft"] = "hard" + elif (thread_key := _codex_backend_identity(headers).thread_selection_key) is not None: + # prompt_cache_key is intentionally shared by current Codex root trees. + # The thread key is canonical identity; once a bridge exists it is hard + # transport continuity even though pre-bridge account locality is soft. + affinity_key = thread_key + affinity_kind = "thread_header" + strength = "hard" else: session_key = _sticky_key_from_session_header(headers) if session_key is not None: - # One Codex process session can host several independent agent - # threads. Codex keeps the process-level session header shared but - # gives every thread a stable explicit prompt_cache_key. Keying - # only by the header makes a later, non-overlapping child reuse the - # parent's upstream conversation and receive the wrong history. + # Compatibility path for clients that do not expose thread-id. + # Current Codex reaches the thread_header branch above; do not + # reintroduce prompt_cache_key as thread identity here. session_header_key = _make_http_bridge_session_header_fallback_key( headers=headers, api_key=api_key, @@ -1510,6 +1794,12 @@ def _make_http_bridge_session_header_fallback_key( api_key: ApiKeyData | None, explicit_prompt_cache_key: str | None, ) -> _HTTPBridgeSessionKey | None: + if _codex_backend_identity(headers).thread_id is not None: + # Never let a current thread attach to the legacy + # (session-id, prompt_cache_key) lane: both values are shared across + # siblings. Exact turn-state/previous-response aliases are handled by + # durable lookup independently and remain the only safe migration path. + return None session_key = _sticky_key_from_session_header(headers) if session_key is None: return None @@ -1525,6 +1815,39 @@ def _make_http_bridge_session_header_fallback_key( ) +def _turn_keys( + headers: Mapping[str, str], + api_key: ApiKeyData | None, + requested_key: _HTTPBridgeSessionKey, + fallback_key: _HTTPBridgeSessionKey | None, +) -> tuple[str | None, _HTTPBridgeSessionKey | None]: + thread_key = _codex_backend_identity(headers).thread_selection_key + thread_fallback_key = ( + _HTTPBridgeSessionKey("thread_header", thread_key, api_key.id if api_key is not None else None) + if thread_key is not None + else None + ) + incoming_session_key = None if thread_fallback_key is not None else _sticky_key_from_session_header(headers) + initial_session_key = ( + fallback_key + or thread_fallback_key + or (requested_key if requested_key.affinity_kind == "session_header" else None) + ) + return incoming_session_key, initial_session_key + + +def _alias_fallback_key( + incoming_session_key: str | None, + initial_session_key: _HTTPBridgeSessionKey | None, + api_key_id: str | None, +) -> _HTTPBridgeSessionKey | None: + if initial_session_key is not None: + return initial_session_key + if incoming_session_key is None: + return None + return _HTTPBridgeSessionKey("session_header", incoming_session_key, api_key_id) + + async def _http_bridge_should_wait_for_registration( self, key: _HTTPBridgeSessionKey, @@ -1576,15 +1899,19 @@ def _durable_bridge_lookup_allows_local_reuse( def _http_bridge_allow_durable_takeover(lookup: DurableBridgeLookup | None) -> bool: - owner_instance = _durable_bridge_lookup_active_owner(lookup) - if owner_instance is None: + return _http_bridge_durable_lookup_allows_turn_state_takeover(lookup) + + +def _http_bridge_claim_allows_takeover( + lookup: DurableBridgeLookup | None, + *, + force: bool, +) -> bool: + if _http_bridge_allow_durable_takeover(lookup): return True - if lookup is None: + if not force: return False - return lookup.state in { - HttpBridgeSessionState.DRAINING, - HttpBridgeSessionState.CLOSED, - } + return lookup is None or lookup.state != "draining" def _http_bridge_has_durable_recovery_anchor( @@ -1612,7 +1939,7 @@ def _http_bridge_can_local_recover_without_ring( ): return True return ( - key.affinity_kind == "session_header" + key.affinity_kind in {"session_header", "thread_header"} and previous_response_id is None and _sticky_key_from_turn_state_header(headers) is None ) @@ -1828,8 +2155,14 @@ async def _release_http_bridge_unanchored_handoffs_for_request( """Fail-safe cleanup for reservations published before request submission.""" async with service._http_bridge_lock: - for session in service._http_bridge_sessions.values(): + for session in (*service._http_bridge_sessions.values(), *service._http_bridge_detached_sessions.values()): _release_http_bridge_unanchored_handoff(session, request_scope_id=request_scope_id) + # Nested stream finalizers can clear their marker before this fail-safe + # sweep runs. Reconsider every detached generation so marker ordering + # cannot leave a fully drained predecessor owning a socket and cap slot. + detached_sessions = tuple(service._http_bridge_detached_sessions.values()) + for session in detached_sessions: + await service._retire_http_bridge_after_drain_if_ready(session) def _track_alias_registration(session: _HTTPBridgeSession, alias: str, *, turn_state: bool) -> int: @@ -1927,7 +2260,6 @@ async def _persist_http_bridge_previous_response_alias( input_item_count: int | None, input_full_fingerprint: str | None, pending_tool_calls: Mapping[str, str] | None, - response_transition_manifest: ResponseTransitionManifest | None, instance_id: str, lease_ttl_seconds: float, local_alias_was_published: bool = True, @@ -1944,7 +2276,6 @@ async def _persist_http_bridge_previous_response_alias( input_item_count=input_item_count, input_full_fingerprint=input_full_fingerprint, pending_tool_calls=pending_tool_calls, - response_transition_manifest=response_transition_manifest, ) except Exception: logger.warning("Failed to persist durable HTTP bridge previous_response_id alias", exc_info=True) @@ -2109,6 +2440,27 @@ async def _renew_durable_http_bridge_lease( return if lookup.owner_instance_id == current_instance and lookup.owner_epoch == session.durable_owner_epoch: return + if ( + lookup.owner_instance_id == current_instance + and lookup.owner_process_epoch == http_bridge_owner_process_epoch() + and lookup.owner_epoch > session.durable_owner_epoch + # A claim rewrites the row's account_id, so an advance that moved the + # row to another account is a real ownership change for this session, + # not a superseded creator: never adopt across it. + and lookup.account_id == session.account.id + and service._http_bridge_sessions.get(session.key) is session + ): + # THIS process advanced the epoch while this session still holds the + # registry slot for its key — a creator that was superseded mid claim, + # not a real ownership loss (issue #1695). Evicting here would 409 the + # session that legitimately owns the key, so adopt the epoch and keep + # renewing. The process-epoch check matters because two incarnations + # can share a configured instance ID across a graceful restart: the + # successor's claim must still fence the predecessor out. A DIFFERENT + # local session holding the slot also falls through: that one won, and + # this session must be evicted. + session.durable_owner_epoch = lookup.owner_epoch + return # Fenced out: another instance/epoch owns the durable session. Never adopt # the foreign epoch — evict the local session so its upstream websocket and # account lease are released, and fail the request with the retryable @@ -2283,7 +2635,10 @@ def _effective_http_bridge_idle_ttl_seconds( codex_idle_ttl_seconds: float, prompt_cache_idle_ttl_seconds: float | None = None, ) -> float: - if affinity.kind == StickySessionKind.CODEX_SESSION: + if affinity.kind == StickySessionKind.CODEX_SESSION or affinity.codex_session_source == "thread_header": + # The DB row is bounded soft locality, but a live thread bridge owns + # upstream socket history and therefore receives the Codex continuity + # lifetime once created. return max(idle_ttl_seconds, codex_idle_ttl_seconds) if affinity.kind == StickySessionKind.PROMPT_CACHE and prompt_cache_idle_ttl_seconds is not None: return prompt_cache_idle_ttl_seconds @@ -2387,6 +2742,27 @@ def _http_bridge_previous_response_owner_unavailable_error() -> ProxyResponseErr ) +def _http_bridge_reconnect_selection_failure( + selection: Any, + required_preferred_account_id: str | None, +) -> ProxyResponseError: + if required_preferred_account_id is not None: + return _http_bridge_previous_response_owner_unavailable_error() + status_code, error_payload = selection_failure_response(selection) + return ProxyResponseError(status_code, error_payload) + + +def _http_bridge_reconnect_connect_failure( + exc: BaseException, + required_preferred_account_id: str | None, +) -> ProxyResponseError: + if required_preferred_account_id is not None: + return _http_bridge_previous_response_owner_unavailable_error() + if isinstance(exc, ProxyResponseError): + return exc + raise exc + + def _http_bridge_should_attempt_local_previous_response_recovery(exc: ProxyResponseError) -> bool: payload = exc.payload if not isinstance(payload, dict): @@ -2398,6 +2774,10 @@ def _http_bridge_should_attempt_local_previous_response_recovery(exc: ProxyRespo raw_code = code_value.strip() if isinstance(code_value, str) and code_value.strip() else None type_value = error.get("type") error_type = type_value.strip() if isinstance(type_value, str) and type_value.strip() else None + # Normalize like the websocket rewrite path (#1818): upstream frames may + # carry the classifiable code only in ``type`` (or omit both code and + # param on the terse previous-response rejection), and a raw read would + # misclassify them into the ambiguous transport class below (issue #1830). code = _normalize_error_code(raw_code, error_type) if code in { "bridge_owner_unreachable", @@ -2406,6 +2786,14 @@ def _http_bridge_should_attempt_local_previous_response_recovery(exc: ProxyRespo "bridge_instance_mismatch", }: return True + if code in {"stream_incomplete", "stream_idle_timeout", "upstream_request_timeout"}: + # Recovery-first server mode permits exactly one anchored retry on a + # fresh upstream socket. This keeps Codex unchanged; delivery remains + # at-least-once because upstream acceptance is ambiguous. + return _service_get_settings().http_responses_session_bridge_ambiguous_continuation_recovery_mode in { + "server_anchored_replay_once", + "server_indefinite_recovery", + } param_value = error.get("param") param = param_value.strip() if isinstance(param_value, str) and param_value.strip() else None message_value = error.get("message") @@ -2451,6 +2839,15 @@ def _http_bridge_should_attempt_soft_affinity_reroute( } +def _persistent_http_bridge_affinity(affinity: _AffinityPolicy) -> _AffinityPolicy: + if not affinity.abandon_unavailable_legacy_owner: + return affinity + # Restart authority is proof attached to one canonical request body, not a + # property of the longer-lived process session. Never let a reused bridge + # grant a later ordinary request permission to retire hard ownership. + return replace(affinity, abandon_unavailable_legacy_owner=False) + + def _http_bridge_is_context_overflow_error(exc: ProxyResponseError) -> bool: payload = exc.payload if not isinstance(payload, dict): @@ -2485,7 +2882,7 @@ def _http_bridge_should_attempt_local_bootstrap_rebind( headers: Mapping[str, str], previous_response_id: str | None, ) -> bool: - if key.affinity_kind != "session_header": + if key.affinity_kind not in {"session_header", "thread_header"}: return False if previous_response_id is not None: return False @@ -2624,10 +3021,6 @@ def _log_http_bridge_event( upstream_close_code: int | None = None, response_events_seen: int | None = None, transport_classification: str | None = None, - admission_waiter_count: int | None = None, - idle_age_bucket: str | None = None, - retry_action: str | None = None, - circuit_action: str | None = None, ) -> None: level = logging.INFO if event in { @@ -2654,8 +3047,7 @@ def _log_http_bridge_event( "http_bridge_event event=%s bridge_kind=%s bridge_key=%s account_id=%s" " model=%s pending=%s detail=%s cache_key_family=%s model_class=%s" " key_strength=%s owner_check_applied=%s error_message=%s upstream_close_code=%s" - " response_events_seen=%s transport_classification=%s admission_waiters=%s" - " idle_age_bucket=%s retry_action=%s circuit_action=%s", + " response_events_seen=%s transport_classification=%s", event, key.affinity_kind, _hash_identifier(key.affinity_key), @@ -2671,10 +3063,6 @@ def _log_http_bridge_event( upstream_close_code, response_events_seen, transport_classification, - admission_waiter_count, - idle_age_bucket, - retry_action, - circuit_action, ) @@ -2714,6 +3102,7 @@ def _wrapper(*args: Any, **kwargs: Any) -> Any: "_durable_bridge_lookup_active_owner", "_durable_bridge_lookup_allows_local_reuse", "_http_bridge_allow_durable_takeover", + "_http_bridge_claim_allows_takeover", "_http_bridge_has_durable_recovery_anchor", "_http_bridge_can_local_recover_without_ring", "_http_bridge_can_single_instance_owner_takeover_without_anchor", @@ -2731,6 +3120,8 @@ def _wrapper(*args: Any, **kwargs: Any) -> Any: "_http_bridge_requires_cluster_registration", "_effective_http_bridge_idle_ttl_seconds", "_http_bridge_eviction_priority", + "_http_bridge_capacity_after_planned_closes", + "_plan_http_bridge_lru_capacity_closes", "_build_http_bridge_prewarm_text", "_http_bridge_prewarm_enabled", "_record_http_bridge_prewarm_outcome", diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index c28c6f131d..8d79de6d87 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -45,8 +45,8 @@ bridge_prompt_cache_locality_miss_total, bridge_soft_local_rebind_total, ) -from app.core.resilience.overload import local_overload_error from app.core.utils.request_id import ensure_request_scope_id +from app.core.utils.shared_future import wait_on_shared_future from app.db.models import ( AccountStatus, StickySessionKind, @@ -70,6 +70,7 @@ _HTTP_BRIDGE_BACKGROUND_CLOSE_TIMEOUT_SECONDS, _HTTP_BRIDGE_INFLIGHT_STARTED_AT_ATTR, _active_http_bridge_instance_ring, + _alias_fallback_key, _durable_bridge_lookup_active_owner, _durable_bridge_lookup_allows_local_reuse, _forwarded_http_bridge_session_key, @@ -79,10 +80,11 @@ _http_bridge_can_recover_during_drain, _http_bridge_can_single_instance_owner_takeover_without_anchor, _http_bridge_can_single_instance_prompt_cache_takeover_without_anchor, + _http_bridge_capacity_after_planned_closes, + _http_bridge_claim_allows_takeover, _http_bridge_compatible, _http_bridge_continuity_lost_error_envelope, _http_bridge_endpoint_matches_current_instance, - _http_bridge_eviction_priority, _http_bridge_has_durable_recovery_anchor, _http_bridge_incompatible_model_fork_key, _http_bridge_inflight_creation_count, @@ -95,10 +97,13 @@ _http_bridge_parallel_fork_key, _http_bridge_previous_response_alias_key, _http_bridge_previous_response_owner_unavailable_error, + _http_bridge_reconnect_connect_failure, + _http_bridge_reconnect_selection_failure, _http_bridge_request_budget_seconds, _http_bridge_request_needs_unanchored_handoff, _http_bridge_session_account_active, _http_bridge_session_allows_api_key, + _http_bridge_session_generation_count, _http_bridge_session_has_admission_waiter, _http_bridge_session_matches_preferred_account, _http_bridge_session_retiring_with_visible_requests, @@ -111,13 +116,18 @@ _log_http_bridge_startup_wait_timeout, _mark_http_bridge_reader_handoff_reconnect_failed, _persist_http_bridge_replacement_account, + _persistent_http_bridge_affinity, + _plan_http_bridge_lru_capacity_closes, _preferred_http_bridge_reconnect_turn_state, + _raise_if_http_bridge_creation_superseded, _record_bridge_drain_recovery_allowed, _record_bridge_first_turn_timeout, _refresh_reused_http_bridge_session_with_handoff, _register_http_bridge_turn_state_aliases_locked, _require_http_bridge_bound_account_not_excluded, _reserve_http_bridge_unanchored_handoff, + _settle_failed_http_bridge_creation, + _turn_keys, ) from app.modules.proxy._service.http_bridge.helpers import ( _close_http_bridge_session as _helpers_close_http_bridge_session, @@ -155,7 +165,6 @@ _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE, _HARD_HTTP_BRIDGE_AFFINITY_KINDS, # noqa: F401 _WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS, # noqa: F401 - _clear_http_bridge_session_response_checkpoint, _clear_websocket_precreated_replay_fallback, _complete_http_bridge_handoff, _copy_websocket_route_metadata_to_session, @@ -203,17 +212,15 @@ from app.modules.proxy.affinity import ( _AffinityPolicy, _extract_model_class, - _sticky_key_from_session_header, _sticky_key_from_turn_state_header, ) from app.modules.proxy.continuity import ( is_http_bridge_account_neutral_replay, + resolve_reconnect_preferred_account_id, resolve_required_account_id, without_http_bridge_session_affinity_headers, ) -from app.modules.proxy.durable_bridge_coordinator import ( - DurableBridgeLookup, -) +from app.modules.proxy.durable_bridge_coordinator import DurableBridgeLookup from app.modules.proxy.load_balancer import CONTINUITY_OWNER_UNAVAILABLE, AccountLease from app.modules.proxy.selection_errors import USAGE_LIMIT_REACHED, selection_failure_response @@ -427,16 +434,25 @@ async def _get_or_create_http_bridge_session( request_scope_id = ensure_request_scope_id() api_key_id = api_key.id if api_key is not None else None incoming_turn_state = _sticky_key_from_turn_state_header(headers) - incoming_session_key = _sticky_key_from_session_header(headers) - initial_session_key = session_header_fallback_key or (key if key.affinity_kind == "session_header" else None) + incoming_session_key, initial_session_key = _turn_keys(headers, api_key, key, session_header_fallback_key) original_request_unanchored = _http_bridge_request_needs_unanchored_handoff( key, incoming_turn_state, previous_response_id, forwarded_request, forwarded_original_request_unanchored ) + # Model-transition isolation intentionally drops the durable lookup as a + # routing input below. Preserve generation provenance first: the same + # replica id can still name an older socket/process whose late release + # must be fenced by a newly advanced owner epoch. + same_replica_durable_predecessor = bool( + durable_lookup and durable_lookup.owner_instance_id == settings.http_responses_session_bridge_instance_id + ) model_transition_rebind = bool( durable_lookup is not None and not _http_bridge_models_compatible(durable_lookup.model, request_model) ) if model_transition_rebind: durable_lookup = None + # Account selection consumes this one-shot capability; canonical creation + # also forces takeover so a prior-ring durable owner cannot reject it. + force_goal_restart_account_reselection = affinity.abandon_unavailable_legacy_owner if await _http_bridge_should_wait_for_registration(self, key, settings): skip_registration_gate = False async with self._http_bridge_lock: @@ -507,11 +523,12 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: allow_forward_to_owner = False inflight_future: asyncio.Future[_HTTPBridgeSession] | None = None capacity_wait_future: asyncio.Future[_HTTPBridgeSession] | None = None + capacity_error_after_planned_closes: ProxyResponseError | None = None owns_creation = False continuity_error: ProxyResponseError | None = None owner_mismatch_error: ProxyResponseError | None = None owner_forward: _HTTPBridgeOwnerForward | None = None - force_durable_takeover = force_durable_takeover_after_detach + force_durable_takeover = force_durable_takeover_after_detach or force_goal_restart_account_reselection missing_turn_state_alias = False sessions_to_close_before_create: list[_HTTPBridgeSession] = [] session_to_return_after_close: _HTTPBridgeSession | None = None @@ -660,10 +677,10 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: key=key.affinity_key, ): key = _HTTPBridgeSessionKey("turn_state_header", incoming_turn_state, api_key_id) - elif incoming_session_key is not None: - key = initial_session_key or _HTTPBridgeSessionKey( - "session_header", incoming_session_key, api_key_id - ) + elif ( + fallback_key := _alias_fallback_key(incoming_session_key, initial_session_key, api_key_id) + ) is not None: + key = fallback_key used_session_header_fallback = True else: key = _HTTPBridgeSessionKey("turn_state_header", incoming_turn_state, api_key_id) @@ -677,17 +694,21 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: retained_handoff = bool( existing and existing.closed and _http_bridge_session_has_admission_waiter(existing) ) - reusable = existing is not None and _http_bridge_session_reusable_for_lookup( - session=existing, - key=key, - api_key=api_key, - incoming_turn_state=incoming_turn_state, - previous_response_id=previous_response_id, - preferred_account_id=preferred_account_id, - require_preferred_account=require_preferred_account, - service_tier_supported=_http_bridge_compatible(existing, request_model, request_service_tier), - allow_closed_admission_handoff=retained_handoff, - session_key_quarantined=_http_bridge_session_key_quarantined(self, existing.key), + reusable = ( + not force_goal_restart_account_reselection + and existing is not None + and _http_bridge_session_reusable_for_lookup( + session=existing, + key=key, + api_key=api_key, + incoming_turn_state=incoming_turn_state, + previous_response_id=previous_response_id, + preferred_account_id=preferred_account_id, + require_preferred_account=require_preferred_account, + service_tier_supported=_http_bridge_compatible(existing, request_model, request_service_tier), + allow_closed_admission_handoff=retained_handoff, + session_key_quarantined=_http_bridge_session_key_quarantined(self, existing.key), + ) ) fork_key = _http_bridge_parallel_fork_key( key=key, @@ -700,6 +721,7 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: request_service_tier=request_service_tier, request_scope_id=request_scope_id, allow_model_fork=reusable or model_transition_rebind, + force_canonical_replacement=force_goal_restart_account_reselection, ) if fork_key is not None: if existing is not None: @@ -748,7 +770,10 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: force_durable_takeover = True self._schedule_http_bridge_session_closes([detached], reason="registry_detach") existing = None - if existing is not None and not existing.closed and existing.account.status == AccountStatus.ACTIVE: + if existing is not None and ( + force_goal_restart_account_reselection + or (not existing.closed and existing.account.status == AccountStatus.ACTIVE) + ): old_account_id = existing.account.id retiring_with_visible_requests = _http_bridge_session_retiring_with_visible_requests(existing) detached = self._detach_http_bridge_session_locked( @@ -759,7 +784,13 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: if detached is not None: force_durable_takeover = True if not retiring_with_visible_requests: - self._schedule_http_bridge_session_closes([detached], reason="registry_detach") + if self._http_bridge_forced_close_must_finish_before_create( + force_goal_restart_account_reselection, + max_sessions, + ): + sessions_to_close_before_create.append(detached) + else: + self._schedule_http_bridge_session_closes([detached], reason="registry_detach") existing = None if shutdown_state.is_bridge_drain_active() and not _http_bridge_can_recover_during_drain( key=key, @@ -1263,46 +1294,22 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: model_class=_extract_model_class(request_model) if request_model else None, owner_check_applied=owner_check_required, ) - elif inflight_future is None: - while ( - len(self._http_bridge_sessions) + _http_bridge_inflight_creation_count(self) >= max_sessions - and self._http_bridge_sessions + elif session_to_return_after_close is None and inflight_future is None: + # Detached generations remain globally capacity-owned + # until close finalization. This request may discount + # only the idle generations it has committed to close + # synchronously below, before its inflight reservation + # can create a replacement socket. + _plan_http_bridge_lru_capacity_closes( + self, + max_sessions=max_sessions, + model_transition_parent_key=model_transition_parent_key, + sessions_to_close_before_create=sessions_to_close_before_create, + ) + if ( + _http_bridge_capacity_after_planned_closes(self, sessions_to_close_before_create) + >= max_sessions ): - evictable_sessions: list[tuple[_HTTPBridgeSessionKey, _HTTPBridgeSession]] = [] - for candidate_key, candidate_session in self._http_bridge_sessions.items(): - if candidate_key == model_transition_parent_key: - continue - if getattr(candidate_session, "unanchored_reservation_id", None) is not None: - continue - pending_count = self._http_bridge_pending_count_nowait( - candidate_session, - context="capacity_evict_scan", - ) - if pending_count is None: - continue - if pending_count: - continue - evictable_sessions.append((candidate_key, candidate_session)) - if not evictable_sessions: - break - lru_key, lru_session = min( - evictable_sessions, - key=lambda item: _http_bridge_eviction_priority(item[1]), - ) - _log_http_bridge_event( - "evict_lru", - lru_key, - account_id=lru_session.account.id, - model=lru_session.request_model, - cache_key_family=lru_key.affinity_kind, - model_class=_extract_model_class(lru_session.request_model) - if lru_session.request_model - else None, - ) - detached = self._detach_http_bridge_session_locked(lru_key, expected_session=lru_session) - if detached is not None: - sessions_to_close_before_create.append(detached) - if len(self._http_bridge_sessions) + _http_bridge_inflight_creation_count(self) >= max_sessions: if _http_bridge_inflight_creation_count(self): capacity_wait_future = next( future @@ -1310,24 +1317,19 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: if not getattr(future, "_http_bridge_handoff", False) ) else: - _log_http_bridge_event( - "capacity_exhausted_active_sessions", - key, - account_id=None, - model=request_model, - pending_count=( - len(self._http_bridge_sessions) + _http_bridge_inflight_creation_count(self) - ), - cache_key_family=key.affinity_kind, - model_class=_extract_model_class(request_model) if request_model else None, - ) - raise ProxyResponseError( - 429, - local_overload_error( - "HTTP responses session bridge has no idle capacity", - code="capacity_exhausted_active_sessions", - ), + capacity_error = self._http_bridge_active_capacity_error( + key=key, + request_model=request_model, ) + if not sessions_to_close_before_create: + raise capacity_error + # Detachment already transferred these LRU + # generations out of the canonical registry. + # Give each one a bounded-close owner before + # rejecting admission; otherwise this early 429 + # leaves its live socket and leases stranded in + # the detached registry until unrelated cleanup. + capacity_error_after_planned_closes = capacity_error else: inflight_future = asyncio.get_running_loop().create_future() setattr( @@ -1344,6 +1346,15 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: if owns_creation: await self._fail_http_bridge_inflight_session_creation(key, inflight_future, exc) raise + if capacity_error_after_planned_closes is not None: + raise capacity_error_after_planned_closes + if owns_creation and sessions_to_close_before_create: + await self._enforce_http_bridge_capacity_after_planned_closes( + key=key, + inflight_future=inflight_future, + max_sessions=max_sessions, + request_model=request_model, + ) if session_to_return_after_close is not None: return session_to_return_after_close if owner_forward is not None: @@ -1355,8 +1366,11 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: if capacity_wait_future is not None: wait_timeout_seconds = _proxy_admission_wait_timeout_seconds(settings) try: - await asyncio.wait_for( - asyncio.shield(capacity_wait_future), + # Not wait_for(shield(...)): shield attaches per-waiter + # callbacks to the shared registry future, which livelocks + # the event loop under mass timeout (see shared_future.py). + await wait_on_shared_future( + capacity_wait_future, timeout=wait_timeout_seconds, ) except asyncio.CancelledError: @@ -1374,7 +1388,7 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: timeout_seconds=wait_timeout_seconds, key=stale_key or key, request_model=request_model, - pending_count=len(self._http_bridge_sessions), + pending_count=_http_bridge_session_generation_count(self), inflight_count=len(self._http_bridge_inflight_sessions), ) raise timeout_error from exc @@ -1386,8 +1400,11 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: if inflight_future is not None and not owns_creation: wait_timeout_seconds = _proxy_admission_wait_timeout_seconds(settings) try: - session = await asyncio.wait_for( - asyncio.shield(inflight_future), + # Not wait_for(shield(...)): shield attaches per-waiter + # callbacks to the shared registry future, which livelocks + # the event loop under mass timeout (see shared_future.py). + session = await wait_on_shared_future( + inflight_future, timeout=wait_timeout_seconds, ) except asyncio.CancelledError: @@ -1405,7 +1422,7 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: timeout_seconds=wait_timeout_seconds, key=key, request_model=request_model, - pending_count=len(self._http_bridge_sessions), + pending_count=_http_bridge_session_generation_count(self), inflight_count=len(self._http_bridge_inflight_sessions), ) raise timeout_error from exc @@ -1423,6 +1440,7 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: request_service_tier=request_service_tier, request_scope_id=request_scope_id, same_model_required=True, + force_canonical_replacement=force_goal_restart_account_reselection, ) if fork_key is not None: bind_account_neutral_recovery_owner(session) @@ -1434,7 +1452,8 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: ) continue if ( - not session.closed + not force_goal_restart_account_reselection + and not session.closed and _http_bridge_session_account_active(session) and _http_bridge_session_allows_api_key(session, api_key) and _http_bridge_compatible(session, request_model, request_service_tier, True) @@ -1458,7 +1477,9 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: session.request_service_tier = request_service_tier session.last_used_at = _service_time().monotonic() return session - if not session.closed and session.account.status == AccountStatus.ACTIVE: + if force_goal_restart_account_reselection or ( + not session.closed and session.account.status == AccountStatus.ACTIVE + ): old_account_id = session.account.id retiring_with_visible_requests = _http_bridge_session_retiring_with_visible_requests(session) async with self._http_bridge_lock: @@ -1525,29 +1546,23 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: if optional_kwarg not in create_signature.parameters: create_kwargs.pop(optional_kwarg, None) created_session = await create_session(key, **create_kwargs) - # Reader failure detaches locally before bounded close releases the durable row. - same_instance_orphan = bool( - durable_lookup is not None - and durable_lookup.canonical_kind == key.affinity_kind - and durable_lookup.canonical_key == key.affinity_key - and _durable_bridge_lookup_active_owner(durable_lookup) - == settings.http_responses_session_bridge_instance_id - ) - expected_owner_instance = ( - settings.http_responses_session_bridge_instance_id if same_instance_orphan else None - ) - expected_owner_process = ( - durable_lookup.owner_process_epoch if same_instance_orphan and durable_lookup is not None else None - ) - force_durable_takeover = force_durable_takeover or same_instance_orphan - await self._claim_durable_http_bridge_session( - created_session, - allow_takeover=force_durable_takeover or _http_bridge_allow_durable_takeover(durable_lookup), - force_owner_epoch_advance=force_durable_takeover, - # Row-lock check permits released-row reclaim but fences foreign/restarted owners. - expected_takeover_owner_instance_id=expected_owner_instance, - expected_takeover_owner_process_epoch=expected_owner_process, - ) + await _raise_if_http_bridge_creation_superseded(self, key, inflight_future=inflight_future) + claim_kwargs: dict[str, Any] = { + "allow_takeover": _http_bridge_claim_allows_takeover( + durable_lookup, + force=force_durable_takeover, + ), + "force_owner_epoch_advance": (force_durable_takeover or same_replica_durable_predecessor), + } + restart_takeover = durable_lookup is not None and _http_bridge_allow_durable_takeover(durable_lookup) + if restart_takeover: + # restart_takeover means recovering a row whose previous + # owner is genuinely gone. Every claim now advances the + # epoch, so epoch > 1 alone would also count ordinary + # local successor claims (no pre-claim lookup, or a + # forced replace of a live local session). + claim_kwargs["record_restart_takeover"] = True + await self._claim_durable_http_bridge_session(created_session, **claim_kwargs) async with self._http_bridge_lock: current_future = self._http_bridge_inflight_sessions.get(key) if current_future is inflight_future: @@ -1564,18 +1579,18 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: code="capacity_exhausted_active_sessions", ) except BaseException as exc: - async with self._http_bridge_lock: - current_future = self._http_bridge_inflight_sessions.get(key) - if current_future is inflight_future: - self._http_bridge_inflight_sessions.pop(key, None) - if inflight_future is not None and not inflight_future.done(): - if isinstance(exc, asyncio.CancelledError): - inflight_future.cancel() - else: - inflight_future.set_exception(exc) - inflight_future.exception() + superseded = await _settle_failed_http_bridge_creation( + self, + key, + inflight_future=inflight_future, + created_session=created_session, + exc=exc, + ) if created_session is not None and not session_registered: - await self._close_http_bridge_session(created_session) + await self._close_http_bridge_session( + created_session, + release_durable_session=not superseded, + ) raise assert created_session is not None _log_http_bridge_event( @@ -1607,30 +1622,6 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: ) return created_session - async def close_all_http_bridge_sessions(self) -> None: - async with self._http_bridge_lock: - sessions_to_close = list(self._http_bridge_sessions.values()) - inflight_futures = list(self._http_bridge_inflight_sessions.values()) - self._http_bridge_sessions.clear() - self._http_bridge_inflight_sessions.clear() - self._http_bridge_previous_response_index.clear() - shutdown_error = ProxyResponseError( - 503, - openai_error( - "upstream_unavailable", - "HTTP responses session bridge is shutting down", - error_type="server_error", - ), - ) - for inflight_future in inflight_futures: - if inflight_future.done(): - continue - inflight_future.set_exception(shutdown_error) - inflight_future.exception() - for session in sessions_to_close: - await self._close_http_bridge_session(session) - await self._drain_http_bridge_background_cleanup_tasks(reason="shutdown") - async def mark_http_bridge_draining(self) -> None: try: await self._durable_bridge.mark_instance_draining( @@ -1961,7 +1952,7 @@ async def _create_http_bridge_session( session = _HTTPBridgeSession( key=key, headers=connect_headers, - affinity=affinity, + affinity=_persistent_http_bridge_affinity(affinity), api_key=api_key, request_model=request_model, request_service_tier=request_service_tier, @@ -1975,7 +1966,7 @@ async def _create_http_bridge_session( lifecycle_lock=anyio.Lock(), last_used_at=_service_time().monotonic(), idle_ttl_seconds=idle_ttl_seconds, - codex_session=affinity.kind == StickySessionKind.CODEX_SESSION, + codex_session=(affinity.kind == StickySessionKind.CODEX_SESSION or key.affinity_kind == "thread_header"), prewarm_lock=anyio.Lock(), upstream_turn_state=_upstream_turn_state_from_socket(upstream), downstream_turn_state=None, @@ -1999,11 +1990,14 @@ async def _reconnect_http_bridge_session( selection_affinity: _AffinityPolicy | None = None, ) -> None: request_state.response_create_sent_at = None + goal_restart = request_state.affinity_policy.abandon_unavailable_legacy_owner + if selection_affinity is None and goal_restart: + # Storage drops this bit; its request retains reconnect and account-switch authority. + selection_affinity = request_state.affinity_policy account_neutral_recovery = is_http_bridge_account_neutral_replay( - kind=session.key.affinity_kind, - key=session.key.affinity_key, + kind=session.key.affinity_kind, key=session.key.affinity_key ) - require_same_account = require_same_account or account_neutral_recovery + require_same_account = account_neutral_recovery or (require_same_account and not goal_restart) old_upstream = session.upstream old_reader = session.upstream_reader if restart_reader else None session.handoff_in_progress = True @@ -2044,16 +2038,15 @@ async def _reconnect_http_bridge_session( session.api_key = request_state.api_key forced_refresh_account_id = request_state.force_refresh_account_id excluded_account_ids: set[str] = set(request_state.excluded_account_ids) - requested_preferred_account_id = ( - request_state.preferred_account_id if require_preferred_account or account_neutral_recovery else None + requested_preferred_account_id = resolve_reconnect_preferred_account_id( + request_state, session.account.id, require_preferred_account, account_neutral_recovery ) - close_skips_account = session.last_upstream_close_code in _UPSTREAM_CLOSE_CODES_SKIP_SAME_ACCOUNT_RETRY - reconnect_account_bound = require_same_account or (session.key.strength == "hard" and close_skips_account) required_preferred_account_id = resolve_required_account_id( ("requested reconnect owner", requested_preferred_account_id), ("account-neutral recovery", session.account.id if account_neutral_recovery else None), - ("same-account reconnect", session.account.id if reconnect_account_bound else None), ) + close_skips_account = session.last_upstream_close_code in _UPSTREAM_CLOSE_CODES_SKIP_SAME_ACCOUNT_RETRY + hard_close_account_bound = session.key.strength == "hard" and (close_skips_account or require_same_account) skip_same_account = ( session.key.strength != "hard" and close_skips_account and required_preferred_account_id is None ) @@ -2062,7 +2055,7 @@ async def _reconnect_http_bridge_session( _complete_http_bridge_handoff(session, self._http_bridge_inflight_sessions) raise _http_bridge_previous_response_owner_unavailable_error() _require_http_bridge_bound_account_not_excluded( - reconnect_account_bound, session.account.id, excluded_account_ids + hard_close_account_bound, session.account.id, excluded_account_ids ) except BaseException: session.closed = True @@ -2073,7 +2066,7 @@ async def _reconnect_http_bridge_session( retry_same_account_once = not skip_same_account and session.account.id not in excluded_account_ids if skip_same_account: preferred_candidate_id: str | None = None - elif reconnect_account_bound and session.account.id not in excluded_account_ids: + elif hard_close_account_bound and session.account.id not in excluded_account_ids: preferred_candidate_id = session.account.id elif required_preferred_account_id is not None: preferred_candidate_id = required_preferred_account_id @@ -2113,7 +2106,7 @@ async def release_selected_account_lease() -> None: async def abandon_selected_account_retry(selected_account: Any) -> None: nonlocal preferred_candidate_id - if reconnect_account_bound or selected_account_model_replacement: + if hard_close_account_bound or selected_account_model_replacement: await release_selected_account_lease() complete_failed_handoff() raise @@ -2141,7 +2134,7 @@ def complete_failed_handoff() -> None: def require_bound_account() -> None: try: _require_http_bridge_bound_account_not_excluded( - reconnect_account_bound, session.account.id, excluded_account_ids + hard_close_account_bound, session.account.id, excluded_account_ids ) except BaseException: complete_failed_handoff() @@ -2172,7 +2165,7 @@ def require_bound_account() -> None: ), fallback_on_preferred_account_unavailable=( not reuse_current_account_lease - and not reconnect_account_bound + and not hard_close_account_bound and required_preferred_account_id is None ), ) @@ -2191,14 +2184,14 @@ def require_bound_account() -> None: raise _http_bridge_previous_response_owner_unavailable_error() if ( reuse_current_account_lease - and not reconnect_account_bound + and not hard_close_account_bound and required_preferred_account_id is None and _remaining_budget_seconds(deadline) > 0 ): preferred_candidate_id = None continue if selection.error_code == USAGE_LIMIT_REACHED and ( - required_preferred_account_id is not None or reconnect_account_bound + required_preferred_account_id is not None or hard_close_account_bound ): complete_failed_handoff() raise _http_bridge_previous_response_owner_unavailable_error() @@ -2231,7 +2224,7 @@ def require_bound_account() -> None: retry_same_account_once = not skip_same_account and session.account.id not in excluded_account_ids if skip_same_account: preferred_candidate_id = None - elif reconnect_account_bound and session.account.id not in excluded_account_ids: + elif hard_close_account_bound and session.account.id not in excluded_account_ids: preferred_candidate_id = session.account.id elif required_preferred_account_id is not None: preferred_candidate_id = required_preferred_account_id @@ -2245,9 +2238,8 @@ def require_bound_account() -> None: preferred_candidate_id = None continue record_selected_account_takeover(None) - status_code, error_payload = selection_failure_response(selection) complete_failed_handoff() - raise ProxyResponseError(status_code, error_payload) + raise _http_bridge_reconnect_selection_failure(selection, required_preferred_account_id) if required_preferred_account_id is not None and account.id != required_preferred_account_id: if selection.lease is not None: selected_account_lease = selection.lease @@ -2292,7 +2284,7 @@ def require_bound_account() -> None: if exc.status_code != 401 or _remaining_budget_seconds(deadline) <= 0: await release_selected_account_lease() complete_failed_handoff() - raise + raise _http_bridge_reconnect_connect_failure(exc, required_preferred_account_id) from exc try: account = await self._ensure_fresh_with_budget( account, @@ -2315,7 +2307,7 @@ def require_bound_account() -> None: if retry_exc.status_code != 401: await release_selected_account_lease() complete_failed_handoff() - raise + raise _http_bridge_reconnect_connect_failure(retry_exc, required_preferred_account_id) await self._handle_proxy_error(account, retry_exc) await abandon_selected_account_retry(account) continue @@ -2336,8 +2328,8 @@ def require_bound_account() -> None: continue await release_selected_account_lease() complete_failed_handoff() - raise - except (aiohttp.ClientError, asyncio.TimeoutError): + raise _http_bridge_reconnect_connect_failure(exc, required_preferred_account_id) + except (aiohttp.ClientError, asyncio.TimeoutError) as transport_exc: if selected_is_preferred and _remaining_budget_seconds(deadline) > 0: if retry_same_account_once: retry_same_account_once = False @@ -2347,7 +2339,7 @@ def require_bound_account() -> None: continue await release_selected_account_lease() complete_failed_handoff() - raise + raise _http_bridge_reconnect_connect_failure(transport_exc, required_preferred_account_id) except asyncio.CancelledError: session.closed = True await release_selected_account_lease() @@ -2390,9 +2382,13 @@ async def abort_selected_handoff() -> None: if owner_rebind_affinity is not None or account.id != session.account.id: await self._unregister_http_bridge_turn_states(session) await self._unregister_http_bridge_previous_response_ids(session) - _clear_http_bridge_session_response_checkpoint(session) - session.affinity = selection_affinity or session.affinity - session.codex_session = False + session.last_completed_response_id = None + session.last_completed_response_account_id = None + session.last_completed_input_count = 0 + session.last_completed_input_prefix_fingerprint = None + session.last_pending_tool_calls.clear() + session.affinity = _persistent_http_bridge_affinity(selection_affinity or session.affinity) + session.codex_session = session.key.affinity_kind == "thread_header" session.upstream_turn_state = None session.downstream_turn_state = None session.headers = { diff --git a/app/modules/proxy/_service/http_bridge/owner_forwarding.py b/app/modules/proxy/_service/http_bridge/owner_forwarding.py index 2155d8ebed..ef095e5f8f 100644 --- a/app/modules/proxy/_service/http_bridge/owner_forwarding.py +++ b/app/modules/proxy/_service/http_bridge/owner_forwarding.py @@ -2,6 +2,7 @@ import asyncio import logging +from enum import StrEnum from typing import Any, AsyncIterator, Mapping, TypeVar import aiohttp @@ -103,6 +104,9 @@ _HTTPBridgeSessionKey, _signal_propagated_capacity_startup_ready, _signal_propagated_capacity_startup_wait, + _signal_propagated_responses_owner_forward_dispatched, + _signal_propagated_responses_owner_forward_rejected, + _signal_propagated_responses_service_cleanup_ready, ) from app.modules.proxy._service.support import ( _websocket_route_log_kwargs as _websocket_route_log_kwargs, @@ -157,6 +161,43 @@ T = TypeVar("T") +class _OwnerForwardOutcome(StrEnum): + NOT_DISPATCHED = "not_dispatched" + DISPATCH_AMBIGUOUS = "dispatch_ambiguous" + RECEIVER_ACKNOWLEDGED = "receiver_acknowledged" + RECEIVER_REJECTED = "receiver_rejected" + + +class _OwnerForwardRequestError(ProxyResponseError): + def __init__( + self, + source: ProxyResponseError, + *, + outcome: _OwnerForwardOutcome, + ) -> None: + super().__init__( + source.status_code, + source.payload, + failure_phase=source.failure_phase, + retryable_same_contract=source.retryable_same_contract, + failure_detail=source.failure_detail, + failure_exception_type=source.failure_exception_type, + upstream_status_code=source.upstream_status_code, + upstream_error_code=source.upstream_error_code, + failed_session=source.failed_session, + ) + self.outcome = outcome + + +def _owner_forward_failure_allows_local_recovery(exc: ProxyResponseError) -> bool: + if not isinstance(exc, _OwnerForwardRequestError): + return True + return exc.outcome in { + _OwnerForwardOutcome.NOT_DISPATCHED, + _OwnerForwardOutcome.RECEIVER_REJECTED, + } + + def _durable_recovery_supersedes_local_session( durable_lookup: DurableBridgeLookup | None, session: _HTTPBridgeSession, @@ -381,7 +422,8 @@ async def _forward_http_bridge_request_to_owner( original_request_unanchored=( recovery_forward or ( - owner_forward.key.affinity_kind in {"session_header", "internal_unanchored_parallel"} + owner_forward.key.affinity_kind + in {"session_header", "thread_header", "internal_unanchored_parallel"} and incoming_turn_state is None and payload.previous_response_id is None ) @@ -407,8 +449,32 @@ async def _forward_http_bridge_request_to_owner( owner_check_applied=True, ) + forward_outcome = _OwnerForwardOutcome.NOT_DISPATCHED forwarded_any = False forwarded_response_id: str | None = None + + def owner_response_ready() -> None: + nonlocal forward_outcome + forward_outcome = _OwnerForwardOutcome.RECEIVER_ACKNOWLEDGED + _signal_propagated_capacity_startup_ready() + if api_key_reservation is not None: + # A receiver carrying the origin reservation delays its 200 + # response until its settlement finalizer is active. Mirror + # that explicit handoff into the origin's startup guard. + _signal_propagated_responses_service_cleanup_ready() + + def owner_request_dispatched() -> None: + nonlocal forward_outcome + forward_outcome = _OwnerForwardOutcome.DISPATCH_AMBIGUOUS + if api_key_reservation is not None: + _signal_propagated_responses_owner_forward_dispatched() + + def owner_response_rejected() -> None: + nonlocal forward_outcome + forward_outcome = _OwnerForwardOutcome.RECEIVER_REJECTED + if api_key_reservation is not None: + _signal_propagated_responses_owner_forward_rejected() + try: async for event_block in self._http_bridge_owner_client.stream_responses( owner_endpoint=owner_forward.owner_endpoint, @@ -416,8 +482,10 @@ async def _forward_http_bridge_request_to_owner( headers=forward_headers, context=forward_context, request_started_at=request_started_at, + on_request_dispatched=owner_request_dispatched, + on_response_rejected=owner_response_rejected, on_response_wait=_signal_propagated_capacity_startup_wait, - on_response_ready=_signal_propagated_capacity_startup_ready, + on_response_ready=owner_response_ready, ): forwarded_any = True event_payload = parse_sse_data_json(event_block) @@ -445,7 +513,7 @@ async def _forward_http_bridge_request_to_owner( if forwarded_any: yield exc.event_block return - raise ProxyResponseError( + error = ProxyResponseError( 503, openai_error( "bridge_owner_unreachable", @@ -455,7 +523,8 @@ async def _forward_http_bridge_request_to_owner( failure_phase="owner_forward", failure_detail="relay_timeout", failure_exception_type=type(exc).__name__, - ) from exc + ) + raise _OwnerForwardRequestError(error, outcome=forward_outcome) from exc except ProxyResponseError as exc: if PROMETHEUS_AVAILABLE and bridge_owner_forward_total is not None: bridge_owner_forward_total.labels(outcome="fail").inc() @@ -480,7 +549,7 @@ async def _forward_http_bridge_request_to_owner( default_message="HTTP bridge owner request failed", ) return - raise + raise _OwnerForwardRequestError(exc, outcome=forward_outcome) from exc except (aiohttp.ClientError, asyncio.TimeoutError) as exc: if PROMETHEUS_AVAILABLE and bridge_owner_forward_total is not None: bridge_owner_forward_total.labels(outcome="fail").inc() @@ -506,7 +575,7 @@ async def _forward_http_bridge_request_to_owner( ) ) return - raise ProxyResponseError( + error = ProxyResponseError( 503, openai_error( "bridge_owner_unreachable", @@ -516,7 +585,8 @@ async def _forward_http_bridge_request_to_owner( failure_phase="owner_forward", failure_detail=str(exc) or "owner_forward_request_failed", failure_exception_type=type(exc).__name__, - ) from exc + ) + raise _OwnerForwardRequestError(error, outcome=forward_outcome) from exc else: if PROMETHEUS_AVAILABLE and bridge_owner_forward_total is not None: bridge_owner_forward_total.labels(outcome="success").inc() diff --git a/app/modules/proxy/_service/http_bridge/protocol.py b/app/modules/proxy/_service/http_bridge/protocol.py index 42ecbc16a6..83ccb59d5d 100644 --- a/app/modules/proxy/_service/http_bridge/protocol.py +++ b/app/modules/proxy/_service/http_bridge/protocol.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio from collections.abc import Mapping from typing import Any, Protocol @@ -9,7 +10,6 @@ from app.modules.proxy._service.support import _HTTPBridgeSession, _HTTPBridgeSessionKey from app.modules.proxy.durable_bridge_repository import DurableBridgeAliasRegistrationReceipt from app.modules.proxy.load_balancer import AccountSelection -from app.modules.proxy.response_transition_manifest import ResponseTransitionManifest class _HTTPBridgeServiceProtocol(Protocol): @@ -22,6 +22,7 @@ class _HTTPBridgeServiceProtocol(Protocol): _durable_bridge_coordinator: Any _http_bridge_owner_client: Any _http_bridge_sessions: Any + _http_bridge_detached_sessions: Any _http_bridge_inflight_sessions: Any _http_bridge_turn_state_index: Any _http_bridge_previous_response_index: Any @@ -44,10 +45,27 @@ def _raise_for_unsupported_input_image_references(self, payload: ResponsesReques async def _resolve_file_account_for_responses( self, payload: ResponsesRequest, headers: Mapping[str, str] ) -> str | None: ... - async def _fail_pending_websocket_requests(self, *args: Any, **kwargs: Any) -> None: ... + async def _resolve_forwarded_file_account_for_responses( + self, + payload: ResponsesRequest, + headers: Mapping[str, str], + *, + forwarded_file_owner_account_id: str | None, + require_forwarded_file_owner: bool = False, + ) -> str | None: ... + async def _fail_pending_websocket_requests(self, *args: Any, **kwargs: Any) -> bool: ... async def _finalize_websocket_request_state(self, *args: Any, **kwargs: Any) -> None: ... async def _next_websocket_receive_timeout(self, *args: Any, **kwargs: Any) -> Any: ... async def _close_http_bridge_session_bounded(self, session: _HTTPBridgeSession, *, reason: str) -> None: ... + async def _close_http_bridge_session(self, session: _HTTPBridgeSession) -> None: ... + async def _drain_http_bridge_background_cleanup_tasks(self, *, reason: str) -> None: ... + async def _fail_http_bridge_inflight_session_creation( + self, + key: _HTTPBridgeSessionKey, + inflight_future: asyncio.Future[_HTTPBridgeSession] | None, + exc: BaseException, + ) -> bool: ... + async def _retire_http_bridge_after_drain_if_ready(self, session: _HTTPBridgeSession) -> bool: ... async def _refresh_durable_http_bridge_session(self, session: _HTTPBridgeSession) -> None: ... def _http_bridge_pending_count_nowait(self, session: _HTTPBridgeSession, *, context: str) -> int | None: ... def _detach_http_bridge_session_locked( @@ -57,6 +75,9 @@ def _detach_http_bridge_session_locked( expected_session: _HTTPBridgeSession | None = None, mark_closed: bool = True, ) -> _HTTPBridgeSession | None: ... + def _take_all_http_bridge_sessions_locked( + self, + ) -> tuple[list[_HTTPBridgeSession], list[asyncio.Future[_HTTPBridgeSession]]]: ... def _unregister_http_bridge_turn_states_locked(self, session: _HTTPBridgeSession) -> None: ... def _unregister_http_bridge_previous_response_ids_locked(self, session: _HTTPBridgeSession) -> None: ... async def _register_http_bridge_turn_state_impl( @@ -79,7 +100,6 @@ async def _register_http_bridge_previous_response_id_impl( input_item_count: int | None = None, input_full_fingerprint: str | None = None, pending_tool_calls: Mapping[str, str] | None = None, - response_transition_manifest: ResponseTransitionManifest | None = None, ) -> bool: ... def _schedule_http_bridge_session_closes( self, diff --git a/app/modules/proxy/_service/http_bridge/quarantine.py b/app/modules/proxy/_service/http_bridge/quarantine.py index 991a2211f8..ff47ea8b9f 100644 --- a/app/modules/proxy/_service/http_bridge/quarantine.py +++ b/app/modules/proxy/_service/http_bridge/quarantine.py @@ -18,10 +18,8 @@ # Quarantine is a bounded, in-memory, session-scoped (never account-scoped) # marker for HTTP bridge session keys that have proven silent/wedged: a later -# request must not re-attach to them. Only a durable-owner-bound complete -# resend proof may take the existing fresh session/no-anchor path (#1534); -# merely full-resend-shaped or delta payloads retain the anchor and fail -# closed. It complements — and never replaces +# request must not re-attach to them and must take the existing fresh +# session/no-anchor path instead (#1534). It complements — and never replaces # — the in-flight recovery machinery: the eventless watchdog and bounded # replay (#1394) recover the request that is currently stuck, the fenced # durable-anchor clear (#1563) stops a *fully eventless* full-resend anchor @@ -37,7 +35,6 @@ _HTTP_BRIDGE_QUARANTINE_WEDGED_REATTACH_REASON = "reattach_missing_response_created" _HTTP_BRIDGE_QUARANTINE_REPEATED_EVENTLESS_REASON = "repeated_eventless_timeout" -_HTTP_BRIDGE_QUARANTINE_REJECTED_STALE_ANCHOR_REASON = "proxy_injected_previous_response_rejected" @dataclass(slots=True) diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 54d2c799f1..e3412b33f9 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -6,7 +6,8 @@ import math import random from collections import deque -from dataclasses import replace +from collections.abc import Callable +from dataclasses import dataclass, replace from typing import Any, Literal, Mapping, cast from uuid import uuid4 @@ -14,8 +15,9 @@ from app.core.clients.files import create_file as core_create_file # noqa: F401 from app.core.clients.files import finalize_file as core_finalize_file # noqa: F401 -from app.core.clients.proxy import CodexControlResponse as CodexControlResponse from app.core.clients.proxy import ( # noqa: F401 + CODEX_INSTALLATION_ID_HEADER, + CODEX_TURN_METADATA_HEADER, ImageFetchSession, ProxyResponseError, UpstreamProxyRouteTrace, @@ -35,18 +37,16 @@ push_stream_timeout_overrides, push_transcribe_timeout_overrides, ) +from app.core.clients.proxy import CodexControlResponse as CodexControlResponse from app.core.clients.proxy import codex_control_request as core_codex_control_request # noqa: F401 from app.core.clients.proxy import compact_responses as core_compact_responses # noqa: F401 from app.core.clients.proxy import transcribe_audio as core_transcribe_audio # noqa: F401 from app.core.clients.proxy_websocket import ( - UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE, UpstreamWebSocketTransportError, is_account_neutral_websocket_error_code, ) -from app.core.errors import ( - openai_error, -) +from app.core.errors import OpenAIErrorEnvelope, openai_error from app.core.openai.parsing import parse_sse_event from app.core.openai.requests import ( ResponsesRequest, @@ -61,8 +61,7 @@ set_request_id, ) from app.core.utils.sse import format_sse_event, parse_sse_data_json -from app.db.models import HttpBridgeRowlessRecoveryState -from app.db.session import SessionLocal +from app.db.models import StickySessionKind from app.modules.api_keys.service import ( ApiKeyData, ApiKeyUsageReservationData, @@ -80,11 +79,13 @@ _await_task_deferring_cancellation, _build_http_bridge_prewarm_text, _http_bridge_durable_lease_ttl_seconds, + _http_bridge_is_previous_response_owner_unavailable, _http_bridge_key_strength, _http_bridge_precreated_retry_failure_error, _http_bridge_prewarm_enabled, _http_bridge_request_budget_seconds, _http_bridge_request_counts_against_queue, + _http_bridge_retry_circuit_attempt_selection_for_pending_requests, _log_http_bridge_event, _record_continuity_fail_closed, _record_http_bridge_prewarm_outcome, @@ -96,7 +97,6 @@ ) from app.modules.proxy._service.http_bridge.retry_circuit import ( _http_bridge_anchor_poison_detail, - _http_bridge_retry_circuit_error_message, ) from app.modules.proxy._service.http_bridge.service_stubs import ( _call_with_supported_optional_kwargs, @@ -125,6 +125,7 @@ _upstream_response_create_max_bytes, _websocket_auth_failure_permanent_code, _websocket_auth_failure_requires_reauth, + _websocket_request_text_is_account_neutral_fresh_replay, ) from app.modules.proxy._service.http_bridge.upstream_events import ( _abandon_durable_http_bridge_continuity, @@ -153,6 +154,8 @@ _clear_websocket_request_error_overrides, _copy_websocket_route_metadata_from_session, _event_type_from_payload, + _HTTPBridgeResponseCreateAttempt, + _HTTPBridgeRetryCircuitAttemptSelection, _HTTPBridgeSession, _request_log_client_fields, _websocket_request_can_replay_before_visible_output, @@ -202,7 +205,10 @@ from app.modules.proxy.durable_bridge_repository import ( DurableBridgeAliasRegistration, DurableBridgeAliasRegistrationReceipt, + durable_bridge_api_key_scope, durable_bridge_hash, + durable_bridge_operation_fingerprint, + durable_bridge_operation_id, ) from app.modules.proxy.fair_share import ( API_KEY_STREAM_FAIR_SHARE_ERROR_CODE, @@ -213,8 +219,6 @@ _parse_openai_error, ) from app.modules.proxy.load_balancer import effective_account_concurrency_caps -from app.modules.proxy.rowless_recovery import rowless_actual_wire_fingerprint -from app.modules.proxy.rowless_recovery_repository import RowlessRecoveryRepository from app.modules.proxy.tool_call_dedupe import ( dedupe_replayed_side_effect_input_items, ) @@ -234,6 +238,119 @@ ) +@dataclass(frozen=True, slots=True) +class _HTTPBridgeStaleGateSnapshot: + pending_states: list[_WebSocketRequestState] + queued_count: int + threshold_seconds: float + stale_request_states: list[_WebSocketRequestState] + should_retire: bool + retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection + + +def _http_bridge_client_full_history_recovery_enabled(request_state: _WebSocketRequestState) -> bool: + """Return whether an ambiguous send failure may ask the client to replay.""" + settings = _service_get_settings() + return ( + request_state.propagate_http_errors + and getattr(settings, "http_responses_session_bridge_ambiguous_continuation_recovery_mode", "fail_closed") + == "client_full_history_once" + and request_state.previous_response_id is not None + and request_state.response_id is None + and request_state.response_event_count == 0 + ) + + +def _http_bridge_server_anchored_replay_enabled(request_state: _WebSocketRequestState) -> bool: + settings = _service_get_settings() + return ( + getattr(settings, "http_responses_session_bridge_ambiguous_continuation_recovery_mode", "fail_closed") + in {"server_anchored_replay_once", "server_indefinite_recovery"} + and request_state.previous_response_id is not None + and request_state.response_id is None + and request_state.response_event_count == 0 + and ( + request_state.replay_count == 0 + or getattr(settings, "http_responses_session_bridge_ambiguous_continuation_recovery_mode", "") + == "server_indefinite_recovery" + ) + ) + + +def _http_bridge_operation_fence_for_hard_continuity_enabled(request_state: _WebSocketRequestState) -> bool: + """Return whether a hard turn-state request may use the durable replay fence.""" + if not request_state.hard_continuity_anchor: + return False + return getattr( + _service_get_settings(), + "http_responses_session_bridge_ambiguous_continuation_recovery_mode", + "fail_closed", + ) in {"server_anchored_replay_once", "server_indefinite_recovery"} + + +def _http_bridge_operation_fingerprint( + *, + session_id: str, + api_key_scope: str, + request_state: _WebSocketRequestState, + text_data: str, +) -> str: + fingerprint_text = _text_without_account_installation_id(text_data) + if request_state.previous_response_id is None and _http_bridge_operation_fence_for_hard_continuity_enabled( + request_state + ): + # Hard turn-state requests do not carry previous_response_id. Scope + # their operation identity to the durable session so identical prompts + # in two conversations cannot collide in the global fingerprint fence. + fingerprint_text = f"session:{session_id}\n{fingerprint_text}" + return durable_bridge_operation_fingerprint( + api_key_scope=api_key_scope, + request_text=fingerprint_text, + ) + + +def _http_bridge_terminal_hard_turn_response_id( + request_state: _WebSocketRequestState, + operation: Any, + *, + allow_anchored_continuation: bool = False, +) -> str | None: + """Return a completed hard-turn anchor when this is a new client turn. + + Hard turn-state requests can omit ``previous_response_id``. Their durable + operation fingerprint is therefore otherwise identical for repeated + prompts. A terminal operation with a response id represents the prior turn, + not an in-flight retry, so the next request must advance from its response + rather than replaying that transcript. Recovery/rebind states retain the + operation identity and are intentionally excluded here. Spool completeness + is required only when replaying the stored transcript. + """ + if ( + (request_state.previous_response_id is not None and not allow_anchored_continuation) + or not request_state.hard_continuity_anchor + or request_state.operation_id is not None + or request_state.operation_rebind_required + or request_state.replay_count != 0 + ): + return None + operation_state = getattr(operation, "state", None) + operation_state = getattr(operation_state, "value", operation_state) + if operation_state != "completed": + return None + response_id = getattr(operation, "response_id", None) + return response_id if isinstance(response_id, str) and response_id else None + + +def _http_bridge_client_full_history_recovery_error() -> OpenAIErrorEnvelope: + payload = openai_error( + "previous_response_not_found", + "Previous response was not found; retry without previous_response_id.", + error_type="invalid_request_error", + ) + payload["error"]["param"] = "previous_response_id" + return payload + + async def _rollback_http_bridge_recovery_turn_state_registration( service: Any, receipt: DurableBridgeAliasRegistrationReceipt, @@ -248,9 +365,21 @@ async def _send_http_bridge_request_text_with_archive_id( session: "_HTTPBridgeSession", request_state: _WebSocketRequestState, text_data: str, + *, + on_send_started: Callable[[], None] | None = None, ) -> None: + text_data = _text_with_operation_id(text_data, request_state.operation_id) + # Operation metadata is added after the initial payload sizing pass. Check + # the exact frame that will cross the websocket so the metadata cannot + # push an otherwise-valid response.create over the upstream limit. + _enforce_http_bridge_response_create_text_size(request_state, text_data) + if on_send_started is not None: + on_send_started() token = set_request_id(request_state.archive_request_id) try: + request_state.response_create_attempt_count += 1 + attempt = _HTTPBridgeResponseCreateAttempt(ordinal=request_state.response_create_attempt_count) + request_state.response_create_attempt = attempt request_state.response_create_sent_at = _service_time().monotonic() session.upstream_reader_wakeup.set() try: @@ -259,7 +388,9 @@ async def _send_http_bridge_request_text_with_archive_id( # A failed or cancelled send is settled by its caller. Disarm the # owner watchdog before lifecycle ownership is released so the # reader cannot race that cleanup and settle the request twice. - request_state.response_create_sent_at = None + attempt.disarmed = True + if request_state.response_create_attempt is attempt: + request_state.response_create_sent_at = None session.upstream_reader_wakeup.set() raise finally: @@ -294,6 +425,89 @@ def _text_with_account_installation_id(text_data: str, codex_installation_id: st return json.dumps(payload, ensure_ascii=True, separators=(",", ":")) +def _text_with_operation_id(text_data: str, operation_id: str | None) -> str: + """Attach a stable operation identity without changing the request contract.""" + if not operation_id: + return text_data + try: + payload = json.loads(text_data) + except (TypeError, json.JSONDecodeError): + return text_data + if not isinstance(payload, dict): + return text_data + raw_metadata = payload.get("client_metadata") + metadata = dict(raw_metadata) if isinstance(raw_metadata, dict) else {} + # This namespace is reserved by the bridge; never trust a caller-supplied + # value to stand in for the durable operation identity. + metadata["codex_lb_operation_id"] = operation_id + payload["client_metadata"] = metadata + return json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + + +def _text_without_operation_id(text_data: str) -> str: + """Remove caller-supplied bridge identity before durable fingerprinting.""" + try: + payload = json.loads(text_data) + except (TypeError, json.JSONDecodeError): + return text_data + if not isinstance(payload, dict): + return text_data + raw_metadata = payload.get("client_metadata") + if not isinstance(raw_metadata, dict) or "codex_lb_operation_id" not in raw_metadata: + return text_data + metadata = dict(raw_metadata) + metadata.pop("codex_lb_operation_id", None) + if metadata: + payload["client_metadata"] = metadata + else: + payload.pop("client_metadata", None) + return json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + + +def _text_without_account_installation_id(text_data: str) -> str: + """Normalize account-specific installation metadata out of a fingerprint.""" + try: + payload = json.loads(text_data) + except (TypeError, json.JSONDecodeError): + return text_data + if not isinstance(payload, dict): + return text_data + raw_metadata = payload.get("client_metadata") + if not isinstance(raw_metadata, dict): + return text_data + metadata: dict[str, JsonValue] = {} + for key, value in raw_metadata.items(): + if not isinstance(key, str) or key.lower() == CODEX_INSTALLATION_ID_HEADER: + continue + if key.lower() == CODEX_TURN_METADATA_HEADER and isinstance(value, str): + try: + turn_metadata = json.loads(value) + except json.JSONDecodeError: + turn_metadata = None + if isinstance(turn_metadata, dict) and "installation_id" in turn_metadata: + turn_metadata.pop("installation_id", None) + value = json.dumps(turn_metadata, ensure_ascii=True, separators=(",", ":")) + metadata[key] = value + if metadata: + payload["client_metadata"] = metadata + else: + payload.pop("client_metadata", None) + return json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + + +def _text_with_previous_response_id(text_data: str, response_id: str | None) -> str: + if not response_id: + return text_data + try: + payload = json.loads(text_data) + except (TypeError, json.JSONDecodeError): + return text_data + if not isinstance(payload, dict) or not response_id: + return text_data + payload["previous_response_id"] = response_id + return json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + + def _enforce_http_bridge_response_create_text_size( request_state: _WebSocketRequestState, text_data: str, @@ -614,6 +828,7 @@ async def _submit_http_bridge_request( recovery_turn_state: str | None = None, ) -> None: request_scope_id = ensure_request_scope_id() + owned_unanchored_handoff = session.unanchored_reservation_id == request_scope_id try: await self._submit_http_bridge_request_with_handoff( session, @@ -621,20 +836,70 @@ async def _submit_http_bridge_request( text_data=text_data, queue_limit=queue_limit, request_scope_id=request_scope_id, + owned_unanchored_handoff=owned_unanchored_handoff, recovery_turn_state=recovery_turn_state, ) finally: - with anyio.CancelScope(shield=True): - await self._rollback_marker_recovery_claim_before_dispatch( - request_state, - api_key_id=session.key.api_key_id, - durable_session_id=session.durable_session_id, - durable_owner_epoch=session.durable_owner_epoch, - ) - _release_http_bridge_unanchored_handoff( - session, - request_scope_id=request_scope_id, - ) + _release_http_bridge_unanchored_handoff( + session, + request_scope_id=request_scope_id, + ) + # Inner pre-submit cleanup may clear the reservation before control + # returns here, so ownership must be captured before awaiting it. + # Only that request can make detached-session retirement newly + # ready; an ordinary send/reader failure already owns terminal + # settlement, and closing again would run that funnel twice. + if ( + owned_unanchored_handoff + and session.upstream_control.retire_after_drain + and not session.upstream_close_attempted + ): + await self._retire_http_bridge_after_drain_if_ready(session) + + async def _http_bridge_operation_fenced_continuity_replay_allowed( + self: Any, + session: "_HTTPBridgeSession", + *, + request_state: _WebSocketRequestState, + text_data: str, + ) -> bool: + """Allow a cooldown bypass only for an already-fenced hard turn.""" + if ( + not _http_bridge_operation_fence_for_hard_continuity_enabled(request_state) + or request_state.previous_response_id is not None + or session.durable_session_id is None + or session.durable_owner_epoch is None + ): + return False + get_operation_by_fingerprint = getattr(self._durable_bridge, "get_operation_by_fingerprint", None) + if not callable(get_operation_by_fingerprint): + return False + api_key_scope = durable_bridge_api_key_scope(session.key.api_key_id) + request_fingerprint = _http_bridge_operation_fingerprint( + session_id=session.durable_session_id, + api_key_scope=api_key_scope, + request_state=request_state, + text_data=text_data, + ) + try: + operation = await _call_with_supported_optional_kwargs( + get_operation_by_fingerprint, + optional_kwargs={"api_key_scope": api_key_scope}, + request_fingerprint=request_fingerprint, + ) + except Exception: + logger.warning( + "Failed to inspect hard-continuity operation fence before retry request_id=%s", + request_state.request_id, + exc_info=True, + ) + return False + if operation is None or operation.session_id != session.durable_session_id: + return False + operation_state = getattr(operation.state, "value", operation.state) + return operation_state == "unknown" or ( + operation_state in {"completed", "incomplete"} and bool(getattr(operation, "event_spool_complete", False)) + ) async def _submit_http_bridge_request_with_handoff( self: Any, @@ -644,8 +909,21 @@ async def _submit_http_bridge_request_with_handoff( text_data: str, queue_limit: int, request_scope_id: str, + owned_unanchored_handoff: bool, recovery_turn_state: str | None = None, ) -> None: + recovery_attempt_consumed = False + allow_operation_fenced_continuity_replay = False + if _http_bridge_operation_fence_for_hard_continuity_enabled(request_state): + retry_cooldown_seconds = await self._http_bridge_precreated_retry_cooldown_seconds(session) + if retry_cooldown_seconds > 0: + allow_operation_fenced_continuity_replay = ( + await self._http_bridge_operation_fenced_continuity_replay_allowed( + session, + request_state=request_state, + text_data=text_data, + ) + ) # Eventless upstream timeouts retire the current socket. A client # reconnect can otherwise create a fresh socket for the same hard key # and submit the identical request repeatedly while the retry circuit @@ -661,21 +939,22 @@ async def _submit_http_bridge_request_with_handoff( and request_state.response_event_count == 0 and request_state.replay_count == 0 ) - retry_decision = await self._http_bridge_precreated_retry_decision( + allow_server_anchored_replay = _http_bridge_server_anchored_replay_enabled(request_state) + if not await self._http_bridge_precreated_retry_allowed( session, - allow_proof_gated_continuity_replay=allow_proof_gated_continuity_replay, - ) - if not retry_decision.allowed: + allow_proof_gated_continuity_replay=allow_proof_gated_continuity_replay or allow_server_anchored_replay, + allow_operation_fenced_continuity_replay=allow_operation_fenced_continuity_replay, + ): retry_after_seconds = max( 1, - math.ceil(retry_decision.retry_after_seconds), + math.ceil(await self._http_bridge_precreated_retry_cooldown_seconds(session)), ) _log_http_bridge_event( "submit_retry_circuit_suppressed", session.key, account_id=session.account.id, model=session.request_model, - detail=retry_decision.last_detail or "hard_key_cooldown", + detail="hard_key_cooldown", cache_key_family=session.key.affinity_kind, model_class=_extract_model_class(session.request_model) if session.request_model else None, ) @@ -683,10 +962,7 @@ async def _submit_http_bridge_request_with_handoff( 503, openai_error( "upstream_request_timeout", - _http_bridge_retry_circuit_error_message( - retry_decision.last_detail, - retry_after_seconds=retry_after_seconds, - ), + "HTTP responses session bridge is cooling down after repeated upstream timeouts; retry shortly.", ), retry_after_seconds=retry_after_seconds, ) @@ -695,78 +971,26 @@ async def _submit_http_bridge_request_with_handoff( # cooldown must not create or refresh a journal entry for a request # that was never dispatched upstream. if ( - request_state.replay_count == 0 - and not request_state.recovery_attempt_claimed - and ( - request_state.recovery_attempt_fingerprint is not None - or (request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text) - ) + request_state.fresh_upstream_request_is_retry_safe + and request_state.fresh_upstream_request_text + and request_state.replay_count == 0 + and request_state.recovery_attempt_fingerprint is None and session.durable_session_id is not None and session.durable_owner_epoch is not None ): - attempt_fingerprint = request_state.recovery_attempt_fingerprint - if attempt_fingerprint is None: - fresh_request_text = request_state.fresh_upstream_request_text - if fresh_request_text is None: - raise RuntimeError("replay-safe recovery request lost its serialized body") - attempt_fingerprint = durable_bridge_hash(fresh_request_text) + attempt_fingerprint = durable_bridge_hash(request_state.fresh_upstream_request_text) try: - if request_state.marker_recovery_terminal_settlement_required: - rejected_response_id = request_state.marker_recovery_rejected_response_id - claim_request_id = request_state.marker_recovery_claim_request_id - if rejected_response_id is None or claim_request_id is None: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The marker recovery generation lost its durable identity.", - ), - ) - if request_state.marker_recovery_claimed: - attempt = await self._durable_bridge.record_recovery_attempt( - session_id=session.durable_session_id, - api_key_id=session.key.api_key_id, - instance_id=_service_get_settings().http_responses_session_bridge_instance_id, - owner_epoch=session.durable_owner_epoch, - request_fingerprint=attempt_fingerprint, - request_id=request_state.request_id, - account_id=session.account.id, - model=request_state.model, - replay_safe=True, - ) - else: - attempt = await self._durable_bridge.claim_and_record_live_session_recovery_attempt( - session_id=session.durable_session_id, - instance_id=_service_get_settings().http_responses_session_bridge_instance_id, - owner_epoch=session.durable_owner_epoch, - account_id=session.account.id, - rejected_response_id=rejected_response_id, - attempt_fingerprint=attempt_fingerprint, - claim_request_id=claim_request_id, - journal_request_id=request_state.request_id, - model=request_state.model, - ) - if attempt is None: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "Another complete-context request already owns this recovery generation.", - ), - ) - request_state.marker_recovery_claimed = True - else: - attempt = await self._durable_bridge.record_recovery_attempt( - session_id=session.durable_session_id, - api_key_id=session.key.api_key_id, - instance_id=_service_get_settings().http_responses_session_bridge_instance_id, - owner_epoch=session.durable_owner_epoch, - request_fingerprint=attempt_fingerprint, - request_id=request_state.request_id, - account_id=session.account.id, - model=request_state.model, - replay_safe=True, - ) + attempt = await self._durable_bridge.record_recovery_attempt( + session_id=session.durable_session_id, + api_key_id=session.key.api_key_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + request_fingerprint=attempt_fingerprint, + request_id=request_state.request_id, + account_id=session.account.id, + model=request_state.model, + replay_safe=True, + ) if attempt is None: # ``None`` is the durable owner fence rejecting this # worker, not an unavailable journal (which raises and @@ -789,14 +1013,24 @@ async def _submit_http_bridge_request_with_handoff( ), ) if getattr(attempt.state, "value", attempt.state) != "unknown": - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The recovery checkpoint was already consumed; retry the request.", - ), - ) - if getattr(attempt, "request_id", request_state.request_id) != request_state.request_id: + if getattr(attempt.state, "value", attempt.state) != "replayed": + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "The recovery checkpoint was already consumed; retry the request.", + ), + ) + # A REPLAYED checkpoint may belong to a completed + # operation whose finalized transcript is safe to replay. + # Defer rejection until the operation ledger lookup below + # can decide that terminal-spool case; nonterminal rows + # remain fail-closed after that lookup. + recovery_attempt_consumed = True + if ( + not recovery_attempt_consumed + and getattr(attempt, "request_id", request_state.request_id) != request_state.request_id + ): raise ProxyResponseError( 502, openai_error( @@ -831,8 +1065,377 @@ async def _submit_http_bridge_request_with_handoff( "Recovered response continuity could not be persisted; retry the request.", ), ) from exc + # Account installation metadata is part of the final upstream frame. + # Apply and size-check it before recording the operation so a local + # payload-too-large rejection cannot leave a submitted retry fence. + text_data = self._http_bridge_text_with_account_installation_id(session, request_state, text_data) + operation_ledger_enabled = bool( + getattr(_service_get_settings(), "http_responses_session_bridge_operation_ledger_enabled", True) + ) + operation_ledger_for_hard_continuity = _http_bridge_operation_fence_for_hard_continuity_enabled(request_state) + record_operation = getattr(self._durable_bridge, "record_operation", None) + if ( + operation_ledger_enabled + and callable(record_operation) + and ( + request_state.previous_response_id is not None + or operation_ledger_for_hard_continuity + or request_state.operation_rebind_required + or recovery_attempt_consumed + ) + and (request_state.operation_id is None or request_state.operation_rebind_required) + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + text_data = _text_without_operation_id(text_data) + api_key_scope = durable_bridge_api_key_scope(session.key.api_key_id) + operation_fingerprint = ( + request_state.operation_fingerprint + if request_state.operation_rebind_required and request_state.operation_fingerprint is not None + else _http_bridge_operation_fingerprint( + session_id=session.durable_session_id, + api_key_scope=api_key_scope, + request_state=request_state, + text_data=text_data, + ) + ) + operation_id = ( + request_state.operation_id + if request_state.operation_rebind_required and request_state.operation_id is not None + else durable_bridge_operation_id(session.durable_session_id, operation_fingerprint) + ) + operation_parent_response_id = ( + request_state.operation_parent_response_id + if request_state.operation_rebind_required + else request_state.previous_response_id + ) + # The operation row must not be committed until the exact + # operation-tagged frame is known to fit. Otherwise a local size + # rejection before ``send_text`` leaves a submitted ledger row + # that fences every identical retry as an unknown in-flight turn. + operation_tagged_text = _text_with_operation_id(text_data, operation_id) + _enforce_http_bridge_response_create_text_size(request_state, operation_tagged_text) + try: + get_operation_by_fingerprint = getattr(self._durable_bridge, "get_operation_by_fingerprint", None) + get_operation = getattr(self._durable_bridge, "get_operation", None) + + async def lookup_operation() -> Any: + operation = None + if callable(get_operation_by_fingerprint): + operation = await _call_with_supported_optional_kwargs( + get_operation_by_fingerprint, + optional_kwargs={"api_key_scope": api_key_scope}, + request_fingerprint=operation_fingerprint, + ) + if operation is None and callable(get_operation): + operation = await get_operation(operation_id=operation_id) + return operation + + existing_operation = await lookup_operation() + if recovery_attempt_consumed and existing_operation is None: + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "The recovery checkpoint was already consumed; retry the request.", + ), + ) + hard_turn_chain_advanced = False + seen_hard_turn_response_ids: set[str] = set() + while not recovery_attempt_consumed: + terminal_hard_turn_response_id = _http_bridge_terminal_hard_turn_response_id( + request_state, + existing_operation, + allow_anchored_continuation=hard_turn_chain_advanced, + ) + if ( + terminal_hard_turn_response_id is not None + and terminal_hard_turn_response_id not in seen_hard_turn_response_ids + ): + # A completed operation with the same body is the + # prior hard turn, not a replay request: advance from + # its response instead of replaying that transcript. + # Keep walking the chain because repeated identical + # turns can have several terminal operations with + # successive response anchors. + seen_hard_turn_response_ids.add(terminal_hard_turn_response_id) + hard_turn_chain_advanced = True + text_data = _text_with_previous_response_id(text_data, terminal_hard_turn_response_id) + request_state.request_text = text_data + request_state.previous_response_id = terminal_hard_turn_response_id + request_state.proxy_injected_previous_response_id = True + request_state.hard_continuity_anchor = True + operation_parent_response_id = terminal_hard_turn_response_id + operation_fingerprint = durable_bridge_operation_fingerprint( + api_key_scope=api_key_scope, + request_text=_text_without_account_installation_id(text_data), + ) + operation_id = durable_bridge_operation_id( + session.durable_session_id, + operation_fingerprint, + ) + operation_tagged_text = _text_with_operation_id(text_data, operation_id) + _enforce_http_bridge_response_create_text_size(request_state, operation_tagged_text) + existing_operation = await lookup_operation() + continue + + # If another worker durably observed the previous turn's + # completion, advance a new continuation to that response + # anchor instead of replaying the timed-out turn. Re-run + # the operation lookup after this race-path advancement so + # two completions observed back-to-back are both walked. + if existing_operation is None: + get_latest_completed = getattr(self._durable_bridge, "get_latest_completed_operation", None) + if callable(get_latest_completed): + completed_operation = await _call_with_supported_optional_kwargs( + get_latest_completed, + optional_kwargs={"request_fingerprint": operation_fingerprint}, + session_id=session.durable_session_id, + parent_response_id=operation_parent_response_id, + ) + if completed_operation is None: + get_latest_completed_any_session = getattr( + self._durable_bridge, + "get_latest_completed_operation_any_session", + None, + ) + if callable(get_latest_completed_any_session): + completed_operation = await _call_with_supported_optional_kwargs( + get_latest_completed_any_session, + optional_kwargs={ + "api_key_scope": api_key_scope, + "request_fingerprint": operation_fingerprint, + }, + parent_response_id=request_state.previous_response_id, + ) + completed_response_id = getattr(completed_operation, "response_id", None) + if completed_response_id and completed_response_id != request_state.previous_response_id: + text_data = _text_with_previous_response_id(text_data, completed_response_id) + request_state.request_text = text_data + request_state.previous_response_id = completed_response_id + request_state.proxy_injected_previous_response_id = True + operation_parent_response_id = completed_response_id + hard_turn_chain_advanced = True + seen_hard_turn_response_ids.add(completed_response_id) + request_state.hard_continuity_anchor = True + operation_fingerprint = durable_bridge_operation_fingerprint( + api_key_scope=api_key_scope, + request_text=_text_without_account_installation_id(text_data), + ) + operation_id = durable_bridge_operation_id( + session.durable_session_id, + operation_fingerprint, + ) + operation_tagged_text = _text_with_operation_id(text_data, operation_id) + _enforce_http_bridge_response_create_text_size(request_state, operation_tagged_text) + existing_operation = await lookup_operation() + continue + break + operation = await _call_with_supported_optional_kwargs( + record_operation, + optional_kwargs={ + "recovery_attempt_session_id": request_state.recovery_attempt_session_id + if request_state.recovery_attempt_claimed + else None, + "recovery_attempt_owner_epoch": request_state.recovery_attempt_owner_epoch + if request_state.recovery_attempt_claimed + else None, + "recovery_attempt_fingerprint": request_state.recovery_attempt_fingerprint + if request_state.recovery_attempt_claimed + else None, + "recovery_attempt_consumed": recovery_attempt_consumed, + }, + operation_id=operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + request_fingerprint=operation_fingerprint, + api_key_scope=api_key_scope, + account_id=session.account.id, + model=request_state.model, + parent_response_id=operation_parent_response_id, + request_text=text_data, + ) + except Exception as exc: + session.closed = True + session.upstream_control.reconnect_requested = True + session.upstream_control.retire_after_drain = True + _record_continuity_fail_closed( + surface="http_bridge", + reason="operation_persistence_failed", + previous_response_id=request_state.previous_response_id, + session_id=request_state.session_id, + upstream_error_code="bridge_continuity_persistence_failed", + ) + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "Response operation continuity could not be persisted; retry the request.", + ), + ) from exc + if operation is None: + session.closed = True + session.upstream_control.reconnect_requested = True + session.upstream_control.retire_after_drain = True + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP responses session ownership changed; retry the request.", + ), + ) + if recovery_attempt_consumed and operation.created: + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "The recovery checkpoint was already consumed; retry the request.", + ), + ) + if not operation.created: + if operation.state in {"completed", "incomplete"}: + if getattr(operation, "event_spool_complete", False): + get_operation_events = getattr(self._durable_bridge, "get_operation_events", None) + replay_events = ( + await get_operation_events(operation_id=operation.operation_id) + if callable(get_operation_events) + else [] + ) + if replay_events and request_state.event_queue is not None: + request_state.operation_replay = True + request_state.operation_id = operation.operation_id + request_state.operation_fingerprint = operation_fingerprint + request_state.operation_registered = True + for replay_event in replay_events: + await request_state.event_queue.put(replay_event) + await request_state.event_queue.put(None) + return + if recovery_attempt_consumed: + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "The recovery checkpoint was already consumed; retry the request.", + ), + ) + recovery_mode = getattr( + _service_get_settings(), + "http_responses_session_bridge_ambiguous_continuation_recovery_mode", + "fail_closed", + ) + indefinite_recovery = recovery_mode == "server_indefinite_recovery" + one_shot_recovery = recovery_mode == "server_anchored_replay_once" and request_state.replay_count == 0 + async with session.pending_lock: + same_operation_pending = any( + pending_request is not request_state + and getattr(pending_request, "operation_id", None) == operation.operation_id + for pending_request in session.pending_requests + ) + if ( + (indefinite_recovery or one_shot_recovery) + and operation.state == "unknown" + and not same_operation_pending + ): + # A previous owner may have persisted a partial sequence + # before its socket died. Claim UNKNOWN atomically with + # the transcript reset so concurrent reconnects cannot + # both pass admission and submit the same operation. + claim_unknown_operation = getattr( + self._durable_bridge, + "claim_unknown_operation_for_recovery", + None, + ) + if not callable(claim_unknown_operation): + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP response recovery could not claim the previous operation; retry the request.", + ), + ) + claimed = await _call_with_supported_optional_kwargs( + claim_unknown_operation, + optional_kwargs={"max_recovery_dispatches": 1} if one_shot_recovery else {}, + operation_id=operation.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + ) + if not claimed: + session.closed = True + session.upstream_control.reconnect_requested = True + session.upstream_control.retire_after_drain = True + raise ProxyResponseError( + 503, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP response recovery ownership changed; retry the request.", + ), + ) + request_state.operation_recovery_claimed = True + request_state.operation_attempt_generation = getattr(operation, "recovery_dispatch_count", 0) + 1 + # The operation remains fenced to one durable identity. + # One-shot mode consumes its existing replay-count budget; + # indefinite mode may make further serialized attempts + # after cooldown because upstream has no idempotency or + # status endpoint. + request_state.operation_id = operation.operation_id + request_state.operation_fingerprint = operation_fingerprint + request_state.operation_registered = True + else: + # A prior dispatch with the same parent and body may have + # been accepted by upstream, or another request may still + # be using the same operation in this session. Without + # upstream idempotency/status proof, never submit it a + # second time. + _record_continuity_fail_closed( + surface="http_bridge", + reason="operation_already_recorded_no_status_proof", + previous_response_id=request_state.previous_response_id, + session_id=request_state.session_id, + upstream_error_code="upstream_operation_status_unknown", + ) + retry_after_seconds = max( + 1, + math.ceil(await self._http_bridge_precreated_retry_cooldown_seconds(session)), + ) + raise ProxyResponseError( + 503, + openai_error( + "upstream_operation_status_unknown", + "The previous response operation may still be running; retry after the cooldown.", + ), + retry_after_seconds=retry_after_seconds, + ) + request_state.operation_id = operation.operation_id + request_state.operation_fingerprint = operation_fingerprint + request_state.operation_parent_response_id = operation_parent_response_id + request_state.operation_registered = True + request_state.operation_rebind_required = False + request_state.operation_created = operation.created + request_state.operation_persisted_response_id = ( + None if request_state.operation_recovery_claimed else getattr(operation, "response_id", None) + ) + if not request_state.operation_recovery_claimed: + request_state.operation_attempt_generation = getattr(operation, "recovery_dispatch_count", 0) + + async def _cleanup_unsubmitted_recovery_claim() -> None: + if ( + not request_state.operation_recovery_claimed and not request_state.operation_created + ) or request_state.operation_dispatched: + return + await self._cleanup_http_bridge_submit_interruption( + session, + request_state=request_state, + gate_acquired=False, + request_enqueued=False, + counted_in_queue=False, + ) + text_data = self._http_bridge_text_with_account_installation_id(session, request_state, text_data) if request_state.response_id is not None or request_state.response_event_count > 0: + await _cleanup_unsubmitted_recovery_claim() _log_http_bridge_event( "submit_after_response_event", session.key, @@ -853,7 +1456,8 @@ async def _submit_http_bridge_request_with_handoff( error_type="server_error", ), ) - if session.upstream_control.retire_after_drain: + if session.upstream_control.retire_after_drain and not owned_unanchored_handoff: + await _cleanup_unsubmitted_recovery_claim() if not session.upstream_close_attempted: await self._retire_http_bridge_after_drain_if_ready(session) raise ProxyResponseError( @@ -873,6 +1477,7 @@ async def _submit_http_bridge_request_with_handoff( elif http_bridge_sessions is not None: current_session = http_bridge_sessions.get(session.key) if current_session is None and _http_bridge_key_strength(session.key) == "hard": + await _cleanup_unsubmitted_recovery_claim() _log_http_bridge_event( "submit_on_closed", session.key, @@ -904,16 +1509,21 @@ async def _submit_http_bridge_request_with_handoff( # receiving 400 previous_response_not_found (which causes the # CLI to drop previous_response_id and resend the full # conversation history, inflating per-turn context by ~20x). - recovered = await self._retry_http_bridge_request_on_fresh_upstream( - session, - request_state=request_state, - text_data=text_data, - send_request=False, - require_same_account=_http_bridge_key_strength(session.key) == "hard", - ) + try: + recovered = await self._retry_http_bridge_request_on_fresh_upstream( + session, + request_state=request_state, + text_data=text_data, + send_request=False, + require_same_account=_http_bridge_key_strength(session.key) == "hard", + ) + except BaseException: + await _cleanup_unsubmitted_recovery_claim() + raise if recovered: session.closed = False else: + await _cleanup_unsubmitted_recovery_claim() _log_http_bridge_event( "submit_on_closed", session.key, @@ -931,14 +1541,34 @@ async def _submit_http_bridge_request_with_handoff( gate_acquired = False request_enqueued = False admission_waiter_registered = False - async with session.pending_lock: - await self._ensure_http_bridge_session_stream_lease_locked(session, request_state=request_state) - # Register the submit as an admission waiter atomically with the - # reacquire so a previous turn's finalizer unwinding concurrently - # cannot see an apparently idle session and release this lease - # before the turn is counted into the session queue. - session.admission_waiter_count += 1 - admission_waiter_registered = True + try: + async with session.pending_lock: + await self._ensure_http_bridge_session_stream_lease_locked(session, request_state=request_state) + # Register the submit as an admission waiter atomically with the + # reacquire so a previous turn's finalizer unwinding concurrently + # cannot see an apparently idle session and release this lease + # before the turn is counted into the session queue. + session.admission_waiter_count += 1 + admission_waiter_registered = True + except BaseException: + # Recovery claims are made before admission. If reacquiring an + # idle session's stream lease fails, no upstream frame can have + # been sent; restore that claim before propagating the admission + # error so a later reconnect is not fenced as already dispatched. + if getattr(session, "unanchored_reservation_id", None) == request_scope_id: + session.unanchored_reservation_id = None + cleanup_task = asyncio.create_task( + self._cleanup_http_bridge_submit_interruption( + session, + request_state=request_state, + gate_acquired=False, + request_enqueued=False, + counted_in_queue=False, + admission_waiter_registered=admission_waiter_registered, + ) + ) + await _await_task_deferring_cancellation(cleanup_task) + raise try: await self._maybe_prewarm_http_bridge_session( session, @@ -1002,35 +1632,6 @@ async def _submit_http_bridge_request_with_handoff( try: text_data = await self._inline_http_bridge_image_urls(text_data, request_state) text_data = self._http_bridge_text_with_account_installation_id(session, request_state, text_data) - if ( - request_state.rowless_recovery_authority_id is not None - and request_state.rowless_recovery_generation is not None - and request_state.rowless_recovery_wire_fingerprint is not None - and rowless_actual_wire_fingerprint(text_data) != request_state.rowless_recovery_wire_fingerprint - ): - async with SessionLocal() as rowless_session: - restored = await RowlessRecoveryRepository(rowless_session).rollback_preflight_setup_failure( - authority_id=request_state.rowless_recovery_authority_id, - generation=request_state.rowless_recovery_generation, - request_id=request_state.request_id, - wire_request_fingerprint=request_state.rowless_recovery_wire_fingerprint, - ) - if not restored: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The changed semantic-rebase wire could not be restored safely.", - ), - ) - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_actual_wire_changed", - "The exact upstream wire changed before dispatch; the semantic rebase remains unsent.", - error_type="invalid_request_error", - ), - ) self._start_request_state_api_key_reservation_heartbeat( request_state, api_key=request_state.api_key, @@ -1066,6 +1667,12 @@ async def _submit_http_bridge_request_with_handoff( current_session = http_bridge_sessions.get(session.key) session_unregistered = current_session is None and _http_bridge_key_strength(session.key) == "hard" session_replaced = current_session is not None and current_session is not session + # Queue publication clears the mutable reservation marker. The + # proof captured before the first await still authorizes exactly + # that request to submit on its detached, draining generation. + detached_handoff_can_submit = ( + owned_unanchored_handoff and session.upstream_control.retire_after_drain and not session.closed + ) if session.closed and current_session is session and not session.upstream_control.retire_after_drain: recovered = await self._retry_http_bridge_request_on_fresh_upstream( session, @@ -1076,7 +1683,7 @@ async def _submit_http_bridge_request_with_handoff( ) if recovered: session.closed = False - if session.closed or session_unregistered or session_replaced: + if session.closed or ((session_unregistered or session_replaced) and not detached_handoff_can_submit): _log_http_bridge_event( "submit_on_closed", session.key, @@ -1280,269 +1887,32 @@ async def _submit_http_bridge_request_with_handoff( "The recovery checkpoint was consumed before dispatch; retry the request.", ), ) - rowless_wire_fingerprint: str | None = None - if request_state.rowless_recovery_authority_id is not None: - if ( - request_state.rowless_recovery_generation is None - or session.durable_session_id is None - or request_state.rowless_recovery_task_authority_digest is None - or request_state.rowless_recovery_wire_fingerprint is None - ): - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The semantic-rebase replacement session is not durable.", - ), - ) - rowless_wire_fingerprint = request_state.rowless_recovery_wire_fingerprint - async with SessionLocal() as rowless_session: - rowless_repository = RowlessRecoveryRepository(rowless_session) - await rowless_repository.claim_dispatch( - authority_id=request_state.rowless_recovery_authority_id, - generation=request_state.rowless_recovery_generation, - replacement_session_id=session.durable_session_id, - request_id=request_state.request_id, - wire_request_fingerprint=rowless_wire_fingerprint, - model=request_state.model, - task_authority_digest=request_state.rowless_recovery_task_authority_digest, - ) async with session.pending_lock: session.pending_requests.append(request_state) session.admission_waiter_count = max(0, session.admission_waiter_count - 1) admission_waiter_registered = False request_enqueued = True - if rowless_wire_fingerprint is not None: - assert request_state.rowless_recovery_authority_id is not None - assert request_state.rowless_recovery_generation is not None - async with SessionLocal() as rowless_session: - rowless_repository = RowlessRecoveryRepository(rowless_session) - send_started = await rowless_repository.mark_dispatch_send_started( - authority_id=request_state.rowless_recovery_authority_id, - generation=request_state.rowless_recovery_generation, - request_id=request_state.request_id, - wire_request_fingerprint=rowless_wire_fingerprint, - ) - if not send_started: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The semantic-rebase send fence could not be persisted.", - ), - ) - request_state.rowless_recovery_send_primitive_reached = True - upstream_send_started = True + + def mark_upstream_send_started() -> None: + nonlocal upstream_send_started + # The helper invokes this only after the final frame + # size preflight. A payload_too_large rejection must + # therefore remain proven pre-dispatch so cleanup can + # roll back a newly-created operation. + upstream_send_started = True + try: - await _send_http_bridge_request_text_with_archive_id(session, request_state, text_data) - except UpstreamWebSocketTransportError as exc: - if exc.error_code == UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE: - if request_state.rowless_recovery_authority_id is not None: - request_state.rowless_recovery_first_send_proven_unsent = True - _log_http_bridge_event( - "retry_pre_dispatch_closed", - session.key, - account_id=session.account.id, - model=session.request_model, - detail="closed_before_send", - cache_key_family=session.key.affinity_kind, - model_class=( - _extract_model_class(session.request_model) if session.request_model else None - ), - ) - try: - recovered = await self._retry_http_bridge_request_on_fresh_upstream( - session, - request_state=request_state, - text_data=text_data, - send_request=True, - # Replacing the physical socket preserves - # the current logical bridge key and its - # session/turn-state handshake. Until this - # path performs an explicit account-neutral - # fork with stripped affinity headers, it - # must stay on the current owner for both - # hard and soft keys. - require_same_account=True, - ) - except UpstreamWebSocketTransportError as retry_exc: - # A second pre-dispatch close is still proven - # unsent, but the one-shot recovery has been - # consumed. Any other replacement-send failure - # is ambiguous and must retain the original - # send-side settlement ownership semantics. - second_pre_dispatch_close = ( - retry_exc.error_code == UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE - ) - if not second_pre_dispatch_close: - request_state.recovery_attempt_dispatched = True - session.closed = True - session.upstream_control.reconnect_requested = True - session.upstream_control.retire_after_drain = True - if ( - second_pre_dispatch_close - and request_state.rowless_recovery_authority_id is not None - and request_state.rowless_recovery_generation is not None - and request_state.rowless_recovery_wire_fingerprint is not None - ): - rowless_rollback_task = asyncio.create_task( - self._rollback_rowless_cancelled_before_fresh_send(request_state) - ) - ( - rolled_back_rowless, - rowless_rollback_cancellation, - ) = await _await_task_deferring_cancellation(rowless_rollback_task) - if not rolled_back_rowless: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The proven-unsent semantic rebase could not be restored safely.", - ), - ) from retry_exc - else: - rowless_rollback_cancellation = None - if second_pre_dispatch_close and recovery_receipt is not None: - # Both physical sockets rejected the send - # before any bytes were dispatched. The - # reversible turn-state alias therefore - # still belongs to the predecessor and - # must be restored before surfacing the - # one-shot recovery failure. Leaving the - # alias on this retiring session would - # fence the next safe retry onto a request - # that upstream never observed. - rollback_cancellation: asyncio.CancelledError | None = None - async with session.recovery_alias_lock: - try: - ( - rolled_back, - rollback_cancellation, - ) = await _rollback_http_bridge_recovery_turn_state_registration( - self, - recovery_receipt, - ) - except Exception: - rolled_back = False - logger.warning( - "Failed to roll back unsent HTTP bridge recovery alias", - exc_info=True, - ) - recovery_receipt = None - if not rolled_back: - _record_continuity_fail_closed( - surface="http_bridge", - reason="recovery_alias_rollback_failed", - previous_response_id=request_state.previous_response_id, - session_id=request_state.session_id, - upstream_error_code="bridge_continuity_persistence_failed", - ) - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - ( - "The unsent recovery alias could not be restored safely; " - "retry the request." - ), - ), - ) from retry_exc - if rollback_cancellation is not None: - raise rollback_cancellation - if rowless_rollback_cancellation is not None: - raise rowless_rollback_cancellation - if retry_exc.error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE: - session.claim_liveness_settlement() - raise - if recovered: - request_state.recovery_attempt_dispatched = True - session.last_used_at = _service_time().monotonic() - else: - session.closed = True - session.upstream_control.reconnect_requested = True - session.upstream_control.retire_after_drain = True - if ( - recovery_receipt is not None - and not request_state.fresh_upstream_send_primitive_reached - ): - # Reconnect/setup failed before the - # replacement send primitive was reached. - # No physical socket observed this turn, - # so restore the predecessor alias instead - # of fencing the next retry onto an - # undispatched request. - rollback_cancellation: asyncio.CancelledError | None = None - async with session.recovery_alias_lock: - try: - ( - rolled_back, - rollback_cancellation, - ) = await _rollback_http_bridge_recovery_turn_state_registration( - self, - recovery_receipt, - ) - except Exception: - rolled_back = False - logger.warning( - "Failed to roll back HTTP bridge recovery alias after setup failure", - exc_info=True, - ) - recovery_receipt = None - if not rolled_back: - _record_continuity_fail_closed( - surface="http_bridge", - reason="recovery_alias_rollback_failed", - previous_response_id=request_state.previous_response_id, - session_id=request_state.session_id, - upstream_error_code="bridge_continuity_persistence_failed", - ) - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - ( - "The unsent recovery alias could not be restored safely; " - "retry the request." - ), - ), - ) from exc - if rollback_cancellation is not None: - raise rollback_cancellation - if ( - not request_state.fresh_upstream_send_primitive_reached - and request_state.rowless_recovery_authority_id is not None - and request_state.rowless_recovery_generation is not None - and request_state.rowless_recovery_wire_fingerprint is not None - ): - async with SessionLocal() as rowless_session: - rolled_back_rowless = await RowlessRecoveryRepository( - rowless_session - ).rollback_physically_unsent_after_send_marker( - authority_id=request_state.rowless_recovery_authority_id, - generation=request_state.rowless_recovery_generation, - request_id=request_state.request_id, - wire_request_fingerprint=(request_state.rowless_recovery_wire_fingerprint), - transport_proof_code=UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - ) - if not rolled_back_rowless: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The proven-unsent semantic rebase could not be restored safely.", - ), - ) from exc - raise - else: - request_state.recovery_attempt_dispatched = True - session.closed = True - session.upstream_control.reconnect_requested = True - session.upstream_control.retire_after_drain = True - if exc.error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE: - session.claim_liveness_settlement() - raise + await _send_http_bridge_request_text_with_archive_id( + session, + request_state, + text_data, + on_send_started=mark_upstream_send_started, + ) except BaseException as exc: - request_state.recovery_attempt_dispatched = True + request_state.recovery_attempt_dispatched = upstream_send_started + request_state.operation_dispatched = ( + request_state.operation_id is not None and upstream_send_started + ) # Publish retirement while lifecycle ownership is still # held; a gate waiter must never reuse an ambiguously sent # response.create socket between unlock and cleanup. @@ -1560,12 +1930,10 @@ async def _submit_http_bridge_request_with_handoff( session.claim_liveness_settlement() raise request_state.recovery_attempt_dispatched = True + request_state.operation_dispatched = request_state.operation_id is not None session.last_used_at = _service_time().monotonic() except asyncio.CancelledError: - recovery_alias_is_proven_unsent = not upstream_send_started or ( - request_state.replay_count > 0 and not request_state.fresh_upstream_send_primitive_reached - ) - if recovery_receipt is not None and recovery_alias_is_proven_unsent: + if recovery_receipt is not None and not upstream_send_started: session.closed = True session.upstream_control.reconnect_requested = True session.upstream_control.retire_after_drain = True @@ -1592,32 +1960,6 @@ async def _submit_http_bridge_request_with_handoff( session_id=request_state.session_id, upstream_error_code="bridge_continuity_persistence_failed", ) - rowless_retry_is_proven_unsent = ( - upstream_send_started - and request_state.rowless_recovery_first_send_proven_unsent - and not request_state.fresh_upstream_send_primitive_reached - and request_state.rowless_recovery_authority_id is not None - ) - if rowless_retry_is_proven_unsent: - rowless_rollback_task = asyncio.create_task( - self._rollback_rowless_cancelled_before_fresh_send(request_state) - ) - try: - rolled_back_rowless, _ = await _await_task_deferring_cancellation(rowless_rollback_task) - except Exception: - rolled_back_rowless = False - logger.warning( - "Failed to roll back cancelled rowless recovery before fresh send", - exc_info=True, - ) - if not rolled_back_rowless: - _record_continuity_fail_closed( - surface="http_bridge", - reason="rowless_cancelled_fresh_send_rollback_failed", - previous_response_id=request_state.previous_response_id, - session_id=request_state.session_id, - upstream_error_code="bridge_continuity_persistence_failed", - ) raise except ProxyResponseError: await self._cleanup_http_bridge_submit_interruption( @@ -1667,6 +2009,7 @@ async def _submit_http_bridge_request_with_handoff( # handed to the kernel. Never reconnect-and-resend from this path; # only failures proven to precede dispatch may be replayed. error_code = exc.error_code if isinstance(exc, UpstreamWebSocketTransportError) else "stream_incomplete" + failure_error_message = str(exc) or "Upstream websocket closed before response.completed" # Liveness expiry and local network loss are transport failures, # not evidence against the selected account. Keep this in sync # with the reader path's shared provenance classification. @@ -1691,6 +2034,48 @@ async def _submit_http_bridge_request_with_handoff( if settlement_cancellation is not None: raise settlement_cancellation else: + # Once the operation-tagged frame has been handed to the + # socket, the transport exception is ambiguous: upstream may + # have accepted it even though this worker saw no + # acknowledgement. Persist UNKNOWN under the owner fence + # before cleanup can retire the closed session and release + # that fence. + if ( + request_state.operation_dispatched + and request_state.operation_registered + and request_state.operation_id is not None + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + mark_operation_unknown = getattr(self._durable_bridge, "mark_operation_unknown", None) + marked_unknown = False + if callable(mark_operation_unknown): + try: + marked_unknown = await mark_operation_unknown( + operation_id=request_state.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + ) + except Exception: + logger.warning( + "Failed to mark ambiguous HTTP bridge operation UNKNOWN operation_id=%s", + request_state.operation_id, + exc_info=True, + ) + if not marked_unknown: + request_state.operation_registered = False + error_code = "bridge_continuity_persistence_failed" + failure_error_message = ( + "Ambiguous response operation could not be persisted; retry the request." + ) + _record_continuity_fail_closed( + surface="http_bridge", + reason="ambiguous_operation_unknown_persistence_failed", + previous_response_id=request_state.previous_response_id, + session_id=request_state.session_id, + upstream_error_code=error_code, + ) await self._cleanup_http_bridge_submit_interruption( session, request_state=request_state, @@ -1705,7 +2090,7 @@ async def _submit_http_bridge_request_with_handoff( pending_requests=deque([request_state]), pending_lock=anyio.Lock(), error_code=error_code, - error_message=str(exc) or "Upstream websocket closed before response.completed", + error_message=failure_error_message, api_key=None, response_create_gate=session.response_create_gate, penalize_account=not account_neutral, @@ -1720,9 +2105,14 @@ async def _submit_http_bridge_request_with_handoff( # previous_response_not_found causes the client to drop # previous_response_id and resend the full conversation # history, inflating per-turn context by ~20x. + if _http_bridge_client_full_history_recovery_enabled(request_state): + raise ProxyResponseError( + 400, + _http_bridge_client_full_history_recovery_error(), + ) from exc raise ProxyResponseError( 502, - openai_error(error_code, str(exc) or "Upstream websocket closed"), + openai_error(error_code, failure_error_message), ) from exc async def _maybe_prewarm_http_bridge_session( @@ -1953,7 +2343,6 @@ async def _cleanup_http_bridge_submit_interruption( counted_in_queue: bool, admission_waiter_registered: bool = False, ) -> None: - await self._rollback_rowless_preflight_setup_failure_if_unbound(request_state) retire_closed_session = False async with session.pending_lock: if request_enqueued and request_state in session.pending_requests: @@ -1963,6 +2352,92 @@ async def _cleanup_http_bridge_submit_interruption( if admission_waiter_registered: session.admission_waiter_count = max(0, session.admission_waiter_count - 1) retire_closed_session = session.closed and session.admission_waiter_count == 0 + if ( + request_state.recovery_attempt_fingerprint is not None + and not request_state.recovery_attempt_claimed + and not request_state.recovery_attempt_dispatched + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + rollback_recovery_attempt = getattr(self._durable_bridge, "rollback_recovery_attempt_before_dispatch", None) + if callable(rollback_recovery_attempt): + try: + await _call_with_supported_optional_kwargs( + rollback_recovery_attempt, + optional_kwargs={}, + session_id=session.durable_session_id, + api_key_id=session.key.api_key_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + request_fingerprint=request_state.recovery_attempt_fingerprint, + ) + except Exception: + logger.warning( + "Failed to roll back pre-dispatch HTTP bridge recovery checkpoint request_id=%s", + request_state.request_id, + exc_info=True, + ) + if ( + request_state.operation_recovery_claimed + and request_state.operation_registered + and request_state.operation_id is not None + and not request_state.operation_dispatched + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + mark_operation_unknown = getattr(self._durable_bridge, "mark_operation_unknown", None) + restored = False + if callable(mark_operation_unknown): + try: + restored = await _call_with_supported_optional_kwargs( + mark_operation_unknown, + optional_kwargs={"restore_recovery_dispatch_claim": True}, + operation_id=request_state.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + ) + except Exception: + logger.warning( + "Failed to restore pre-dispatch HTTP bridge recovery operation UNKNOWN operation_id=%s", + request_state.operation_id, + exc_info=True, + ) + if restored: + request_state.operation_recovery_claimed = False + request_state.operation_id = None + request_state.operation_fingerprint = None + request_state.operation_parent_response_id = None + elif ( + request_state.operation_created + and request_state.operation_registered + and request_state.operation_id is not None + and not request_state.operation_dispatched + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + rollback_operation = getattr(self._durable_bridge, "rollback_operation_before_dispatch", None) + if callable(rollback_operation): + try: + rolled_back = await rollback_operation( + operation_id=request_state.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + ) + except Exception: + rolled_back = False + logger.warning( + "Failed to roll back pre-dispatch HTTP bridge operation operation_id=%s", + request_state.operation_id, + exc_info=True, + ) + if rolled_back: + request_state.operation_registered = False + request_state.operation_created = False + request_state.operation_id = None + request_state.operation_fingerprint = None + request_state.operation_parent_response_id = None self._cancel_request_state_api_key_reservation_heartbeat(request_state) if request_state.response_create_gate is not None: if gate_acquired or request_state.response_create_gate_acquired: @@ -1993,122 +2468,6 @@ async def _cleanup_http_bridge_submit_interruption( ) await self._maybe_release_idle_http_bridge_session_lease(session) - async def _rollback_rowless_preflight_setup_failure_if_unbound( - self, - request_state: _WebSocketRequestState, - ) -> None: - authority_id = request_state.rowless_recovery_authority_id - generation = request_state.rowless_recovery_generation - wire_request_fingerprint = request_state.rowless_recovery_wire_fingerprint - if authority_id is None or generation is None or wire_request_fingerprint is None: - return - if request_state.rowless_recovery_send_primitive_reached: - return - async with SessionLocal() as rowless_session: - repository = RowlessRecoveryRepository(rowless_session) - authority = await repository.get(authority_id) - if ( - authority is None - or authority.state != HttpBridgeRowlessRecoveryState.UNKNOWN - or authority.dispatch_request_id != request_state.request_id - or authority.wire_request_fingerprint != wire_request_fingerprint - ): - return - if authority.replacement_session_id is None: - if authority.dispatch_send_started_at is not None: - restored = False - else: - restored = await repository.rollback_preflight_setup_failure( - authority_id=authority_id, - generation=generation, - request_id=request_state.request_id, - wire_request_fingerprint=wire_request_fingerprint, - ) - else: - restored = await repository.rollback_before_send_primitive( - authority_id=authority_id, - generation=generation, - request_id=request_state.request_id, - wire_request_fingerprint=wire_request_fingerprint, - ) - if not restored: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The unsent semantic-rebase preflight could not be restored safely.", - ), - ) - - async def _rollback_marker_recovery_claim_before_dispatch( - self: Any, - request_state: _WebSocketRequestState, - *, - api_key_id: str | None, - durable_session_id: str | None, - durable_owner_epoch: int | None, - ) -> None: - if not (request_state.marker_recovery_terminal_settlement_required and request_state.marker_recovery_claimed): - return - if request_state.rowless_recovery_send_primitive_reached or request_state.recovery_attempt_dispatched: - return - session_id = request_state.recovery_attempt_session_id - account_id = request_state.preferred_account_id - attempt_fingerprint = request_state.recovery_attempt_fingerprint - claim_request_id = request_state.marker_recovery_claim_request_id - if ( - session_id is None - or durable_session_id != session_id - or durable_owner_epoch is None - or account_id is None - or attempt_fingerprint is None - or claim_request_id is None - ): - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The unsent marker recovery claim lost its durable identity.", - ), - ) - restored = await self._durable_bridge.rollback_live_session_recovery_attempt_before_dispatch( - session_id=session_id, - api_key_id=api_key_id, - instance_id=_service_get_settings().http_responses_session_bridge_instance_id, - owner_epoch=durable_owner_epoch, - account_id=account_id, - attempt_fingerprint=attempt_fingerprint, - request_id=claim_request_id, - journal_request_id=request_state.request_id, - ) - if not restored: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The unsent marker recovery claim could not be restored safely.", - ), - ) - request_state.marker_recovery_claimed = False - - async def _rollback_rowless_cancelled_before_fresh_send( - self, - request_state: _WebSocketRequestState, - ) -> bool: - authority_id = request_state.rowless_recovery_authority_id - generation = request_state.rowless_recovery_generation - wire_request_fingerprint = request_state.rowless_recovery_wire_fingerprint - if authority_id is None or generation is None or wire_request_fingerprint is None: - return False - async with SessionLocal() as rowless_session: - return await RowlessRecoveryRepository(rowless_session).rollback_physically_unsent_after_send_marker( - authority_id=authority_id, - generation=generation, - request_id=request_state.request_id, - wire_request_fingerprint=wire_request_fingerprint, - transport_proof_code=UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - ) - async def _ensure_http_bridge_session_stream_lease_locked( self: Any, session: "_HTTPBridgeSession", @@ -2264,6 +2623,15 @@ async def _detach_http_bridge_request( request_state.event_queue = None await _release_websocket_response_create_gate(request_state, session.response_create_gate) if not detached: + if request_state.operation_replay: + # Replay requests are delivered from the durable transcript + # without entering pending ownership, so the normal detach + # branch cannot settle their API-key reservation. + self._cancel_request_state_api_key_reservation_heartbeat(request_state) + await self._release_websocket_request_state_reservation(request_state) + request_state.api_key_reservation = None + request_state.operation_replay = False + return False if request_state.terminal_settlement_phase == "abandoned": # Belt-and-braces for issue #1594: terminal bookkeeping # claimed this request out of pending ownership, aborted, and @@ -2287,7 +2655,15 @@ async def _fail_stale_http_bridge_pending_requests( request_states: list[_WebSocketRequestState], *, detail: str, + retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None, ) -> None: + if retry_circuit_attempt_selection is None: + # Capture the physical sends before waiting for pending ownership. + # A concurrent recovery may replace request_state.response_create_attempt + # while this task is suspended on pending_lock. + retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + request_states + ) stale_requests: deque[_WebSocketRequestState] = deque() response_events_seen = 0 async with session.pending_lock: @@ -2314,7 +2690,11 @@ async def _fail_stale_http_bridge_pending_requests( # even when the session itself survives with other active requests. _record_http_bridge_quarantine_wedged_pending(self, session, stale_requests) if response_events_seen == 0: - await self._record_http_bridge_retry_circuit_failure(session, detail=detail) + await self._record_http_bridge_retry_circuit_failure_for_attempt_selection( + session, + detail=detail, + selection=retry_circuit_attempt_selection, + ) await self._fail_pending_websocket_requests( account=session.account, account_id_value=session.account.id, @@ -2357,6 +2737,37 @@ def _classify_http_bridge_stale_gate_holders( return stale_states, False return [], bool(stale_states) + async def _snapshot_http_bridge_stale_gate_state( + self: Any, + session: "_HTTPBridgeSession", + *, + now: float, + ) -> _HTTPBridgeStaleGateSnapshot: + threshold_seconds = float( + getattr(_service_get_settings(), "http_responses_session_bridge_stuck_gate_retire_after_seconds", 300.0) + ) + async with session.pending_lock: + pending_states = list(session.pending_requests) + stale_request_states, should_retire = self._classify_http_bridge_stale_gate_holders( + pending_states, + now=now, + threshold_seconds=threshold_seconds, + session_closed=session.closed, + ) + retry_circuit_request_states = ( + stale_request_states if stale_request_states else (pending_states if should_retire else ()) + ) + return _HTTPBridgeStaleGateSnapshot( + pending_states=pending_states, + queued_count=session.queued_request_count, + threshold_seconds=threshold_seconds, + stale_request_states=stale_request_states, + should_retire=should_retire, + retry_circuit_attempt_selection=( + _http_bridge_retry_circuit_attempt_selection_for_pending_requests(retry_circuit_request_states) + ), + ) + async def _retire_http_bridge_after_drain_if_ready(self: Any, session: "_HTTPBridgeSession") -> bool: if not (session.upstream_control.reconnect_requested and session.upstream_control.retire_after_drain): return False @@ -2365,7 +2776,10 @@ async def _retire_http_bridge_after_drain_if_ready(self: Any, session: "_HTTPBri _http_bridge_request_counts_against_queue(request_state) for request_state in session.pending_requests ) should_reconnect = ( - not has_visible_pending and session.queued_request_count == 0 and not session.upstream_close_attempted + not has_visible_pending + and session.queued_request_count == 0 + and session.unanchored_reservation_id is None + and not session.upstream_close_attempted ) if should_reconnect: session.pending_requests.clear() @@ -2382,61 +2796,104 @@ async def _retire_stale_pending_http_bridge_session( *, detail: str, retry_circuit_detail: str | None = None, - retry_circuit_already_recorded: bool = False, response_events_seen: int | None = None, + retired_request_count: int | None = None, + retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None, ) -> None: async with session.pending_lock: retired_request_states = list(session.pending_requests) + if retired_request_count is None: + retired_request_count = sum( + 1 + for request_state in retired_request_states + if _http_bridge_request_counts_against_queue(request_state) + ) + if response_events_seen is None: + # Direct retirement must derive event evidence from the same + # locked ownership snapshot as the pending count. Otherwise an + # eventful stale-gate owner looks eventless merely because its + # caller omitted this optional handoff, creating a false + # circuit strike. Explicit values remain authoritative for + # reader-failure callers whose pending deque was already + # drained before entering this shared boundary. + response_events_seen = max( + ( + max( + request_state.response_event_count, + int( + request_state.response_id is not None + or request_state.latency_response_created_ms is not None + or request_state.downstream_visible + ), + ) + for request_state in retired_request_states + ), + default=0, + ) + if retry_circuit_attempt_selection is None: + retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + retired_request_states + ) # Direct retirement (for example the all-stale stuck-gate path, where # the wedged reattach is the only pending request) cancels the reader # and fails the pendings without passing the partial-cleanup hook or # the reader-failure funnel, so evaluate the wedge shape (#1534) here # too; recording is idempotent for callers that already quarantined. _record_http_bridge_quarantine_wedged_pending(self, session, retired_request_states) - # A terminal upstream error removes its request from pending ownership - # before the websocket's normal close arrives. That trailing close is - # transport cleanup, not another failed request. Recording it with an - # empty retired set double-counts one semantic failure and can open the - # durable retry circuit before the verified recovery turn is admitted. - if retired_request_states and (response_events_seen is None or response_events_seen == 0): - failure_detail = retry_circuit_detail or detail - if retry_circuit_already_recorded: - circuit_snapshot = await self._http_bridge_retry_circuit_snapshot(session) - consecutive_failures = circuit_snapshot.consecutive_failures - else: - consecutive_failures = await self._record_http_bridge_retry_circuit_failure( - session, - detail=failure_detail, - ) - poison_detail = _http_bridge_anchor_poison_detail(failure_detail) + # This circuit measures failed request lifecycles, not upstream socket + # churn. ``response_events_seen == 0`` is also true when an idle reader + # closes with an empty pending deque. Charging that idle close creates a + # phantom first strike, so one later response-create timeout opens the + # nominally "repeated" 60-second cooldown and interrupts the client. + # Keep the ownership proof at this shared retirement boundary unless a + # caller already claimed and drained the deque. The reader-failure + # funnel must pass its pre-drain count because terminal notification + # deliberately empties ``pending_requests`` before retirement. Without + # that handoff, genuine pre-response failures disappear from circuit + # accounting while idle closes and request failures look identical. + if retired_request_count > 0 and response_events_seen == 0: + consecutive_failures = await self._record_http_bridge_retry_circuit_failure_for_attempt_selection( + session, + detail=retry_circuit_detail or detail, + selection=retry_circuit_attempt_selection, + ) + poison_detail = _http_bridge_anchor_poison_detail(retry_circuit_detail or detail) if ( poison_detail is not None and consecutive_failures is not None and consecutive_failures >= _service_get_settings().http_responses_session_bridge_anchor_poison_failure_threshold ): - durable_cleared = await _abandon_durable_http_bridge_continuity( - self, - session, - detail=poison_detail, - ) + # Consecutive eventless failures on one bridge key are + # same-anchor failures (the anchor only advances on a + # completed response, which resets the circuit). Clear the + # poisoned durable anchor while this session still owns the + # lease so the next attempt is not re-anchored into the same + # failure. Without this, only the admission-waiter reader + # path could ever poison an anchor, and an anchored session + # failing without waiters cooled down forever (issue #1830). + durable_cleared = await _abandon_durable_http_bridge_continuity(self, session, detail=poison_detail) if not durable_cleared and session.durable_session_id is not None: + # Keep failed waiterless clears visible in the same + # poison-clear telemetry the admission-waiter path emits; + # the next threshold failure re-attempts the clear. _log_http_bridge_event( "durable_anchor_poison_clear_failed", session.key, account_id=session.account.id, model=session.request_model, - pending_count=len(retired_request_states), + pending_count=retired_request_count, detail=poison_detail, cache_key_family=session.key.affinity_kind, - model_class=(_extract_model_class(session.request_model) if session.request_model else None), + model_class=_extract_model_class(session.request_model) if session.request_model else None, ) session.closed = True async with self._http_bridge_lock: - if self._http_bridge_sessions.get(session.key) is session: - self._http_bridge_sessions.pop(session.key, None) - self._unregister_http_bridge_turn_states_locked(session) - self._unregister_http_bridge_previous_response_ids_locked(session) + # Bounded close may return while resource finalization is still + # running. Detachment transfers ownership instead of freeing the + # capacity slot at canonical removal, and leaves a failed close + # discoverable by shutdown/account invalidation for a later retry. + self._detach_http_bridge_session_locked(session.key, expected_session=session) async with session.pending_lock: should_close = not session.upstream_close_attempted if should_close: @@ -2463,12 +2920,10 @@ async def _retry_http_bridge_request_on_fresh_upstream( send_request: bool = True, require_same_account: bool = False, ) -> bool: - # This helper replaces only the physical WebSocket. It does not create - # a new account-neutral logical key or strip the existing session and - # turn-state handshake. Consequently every replacement remains pinned - # to the current account. Keep the argument for the internal call - # contract, but fail closed if any caller attempts to loosen it. - require_same_account = True + require_same_account = require_same_account or is_http_bridge_account_neutral_replay( + kind=session.key.affinity_kind, + key=session.key.affinity_key, + ) retry_text_data = text_data using_fresh_replay = False if request_state.previous_response_id is not None and send_request: @@ -2487,17 +2942,11 @@ async def _retry_http_bridge_request_on_fresh_upstream( return False retry_text_data = request_state.fresh_upstream_request_text using_fresh_replay = True - if ( - _http_bridge_key_strength(session.key) == "hard" - or not request_state.fresh_upstream_request_is_account_neutral - ): - require_same_account = True if request_state.replay_count >= 1: return False if request_state.response_event_count > 0: return False request_state.replay_count += 1 - request_state.fresh_upstream_send_primitive_reached = False _log_http_bridge_event( "retry_fresh_upstream", session.key, @@ -2513,6 +2962,7 @@ async def _retry_http_bridge_request_on_fresh_upstream( request_state=request_state, restart_reader=True, require_same_account=require_same_account, + require_preferred_account=request_state.file_required_preferred_account, ) if send_request: retry_text_data = self._http_bridge_text_with_account_installation_id( @@ -2524,8 +2974,6 @@ async def _retry_http_bridge_request_on_fresh_upstream( request_state.previous_response_id = None request_state.proxy_injected_previous_response_id = False request_state.request_text = retry_text_data - request_state.rowless_recovery_first_send_proven_unsent = False - request_state.fresh_upstream_send_primitive_reached = True await _send_http_bridge_request_text_with_archive_id(session, request_state, retry_text_data) _clear_websocket_request_error_overrides(request_state) session.last_used_at = _service_time().monotonic() @@ -2535,6 +2983,11 @@ async def _retry_http_bridge_request_on_fresh_upstream( # owner retire the whole session with the typed, non-replayable # failure instead of falling back to the earlier close reason. raise + except ProxyResponseError as exc: + if _http_bridge_is_previous_response_owner_unavailable(exc): + raise + logger.warning("HTTP bridge retry on fresh upstream failed", exc_info=True) + return False except Exception: logger.warning("HTTP bridge retry on fresh upstream failed", exc_info=True) return False @@ -2574,6 +3027,7 @@ def request_is_retryable(request_state: _WebSocketRequestState) -> bool: fresh_hard_request_account_switch_candidate = False proof_gated_continuity_replay_candidate = False + server_anchored_replay_candidate = False if session.key.strength == "hard": async with session.pending_lock: retryable_candidates = [ @@ -2598,10 +3052,13 @@ def request_is_retryable(request_state: _WebSocketRequestState) -> bool: and candidate.response_event_count == 0 and candidate.replay_count == 0 ) + server_anchored_replay_candidate = _http_bridge_server_anchored_replay_enabled(candidate) if not await self._http_bridge_precreated_retry_allowed( session, allow_fresh_hard_account_switch=fresh_hard_request_account_switch_candidate, - allow_proof_gated_continuity_replay=proof_gated_continuity_replay_candidate, + allow_proof_gated_continuity_replay=( + proof_gated_continuity_replay_candidate or server_anchored_replay_candidate + ), ): return False @@ -2674,54 +3131,58 @@ def request_is_retryable(request_state: _WebSocketRequestState) -> bool: ) if request_state.replay_count >= 1 and not additional_clean_close_retry: return False + account_bound_replay = False if request_state.previous_response_id is not None: require_preferred_reconnect = False - if request_state.rowless_recovery_authority_id is not None: - captured_input_item_count = request_state.input_item_count - captured_input_fingerprint = request_state.input_full_fingerprint - request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state) - # Rowless settlement binds the complete client checkpoint, - # not the normalized anchor-free projection sent upstream. - request_state.input_item_count = captured_input_item_count - request_state.input_full_fingerprint = captured_input_fingerprint + if account_neutral_recovery: + request_state.preferred_account_id = session.account.id + switch_text = None + else: + switch_text = _prepare_websocket_request_state_for_account_switch(request_state) + if switch_text is None: + # The retained full body may be retry-safe for continuity + # while still naming an account-scoped uploaded file. In + # that case retry on the same owner-bound anchor instead of + # letting visible-output replay strip the anchor and migrate. + fresh_retry_safe = request_state.fresh_upstream_request_is_retry_safe + request_state.fresh_upstream_request_is_retry_safe = False + try: + request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state) + finally: + request_state.fresh_upstream_request_is_retry_safe = fresh_retry_safe if request_text is None: return False - require_preferred_reconnect = True + require_preferred_reconnect = request_state.preferred_account_id is not None else: - if account_neutral_recovery: - request_state.preferred_account_id = session.account.id - switch_text = None - else: - switch_text = _prepare_websocket_request_state_for_account_switch(request_state) - if switch_text is None: - # The retained full body may be retry-safe for continuity - # while still naming an account-scoped uploaded file. In - # that case retry on the same owner-bound anchor instead of - # letting visible-output replay strip the anchor and migrate. - fresh_retry_safe = request_state.fresh_upstream_request_is_retry_safe - request_state.fresh_upstream_request_is_retry_safe = False - try: - request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state) - finally: - request_state.fresh_upstream_request_is_retry_safe = fresh_retry_safe - if request_text is None: - return False - require_preferred_reconnect = request_state.preferred_account_id is not None - else: - request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state) - if request_text is None: - return False - if not hard_owner_bound: - request_state.excluded_account_ids.add(session.account.id) + request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state) + if request_text is None: + return False + if not hard_owner_bound: + request_state.excluded_account_ids.add(session.account.id) else: # Account-scoped uploaded files cannot be replayed on a # different owner. Keep the preferred account mandatory for # both silent recovery and clean-close recovery. - require_preferred_reconnect = account_neutral_recovery or request_state.file_required_preferred_account + candidate_text = ( + request_state.fresh_upstream_request_text + if request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text + else request_state.request_text + ) + # The send boundary decorates durable operations with + # codex_lb_operation_id after selection. Keep that operation + # identity on its owner unless a dedicated rebind path has + # already replaced the operation ID. + candidate_portable = request_state.operation_id is None and ( + _websocket_request_text_is_account_neutral_fresh_replay(candidate_text) + ) request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state) - if request_text is None: + if request_text is None or request_text != candidate_text: return False - if account_neutral_recovery: + account_bound_replay = not candidate_portable + require_preferred_reconnect = ( + account_neutral_recovery or account_bound_replay or request_state.file_required_preferred_account + ) + if account_neutral_recovery or account_bound_replay: request_state.preferred_account_id = session.account.id elif not request_state.file_required_preferred_account: if hard_owner_bound and not model_fallback_replay and not fresh_hard_request_account_switch_allowed: @@ -2802,7 +3263,7 @@ def request_is_retryable(request_state: _WebSocketRequestState) -> bool: await self._reconnect_http_bridge_session( session, request_state=request_state, - require_same_account=account_neutral_recovery, + require_same_account=account_neutral_recovery or account_bound_replay, require_preferred_account=True, **reconnect_reader_kwargs, ) @@ -2872,86 +3333,18 @@ def request_is_retryable(request_state: _WebSocketRequestState) -> bool: await self._release_request_state_account_response_create_lease(request_state) return False request_text = self._http_bridge_text_with_account_installation_id(session, request_state, request_text) - if request_state.rowless_recovery_authority_id is not None: - if ( - request_state.rowless_recovery_generation is None - or request_state.rowless_recovery_task_authority_digest is None - or request_state.rowless_recovery_wire_fingerprint is None - or session.durable_session_id is None - or session.durable_owner_epoch is None - or rowless_actual_wire_fingerprint(request_text) != request_state.rowless_recovery_wire_fingerprint - ): - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The automatic semantic-rebase wire or durable owner changed before dispatch.", - ), - ) - async with SessionLocal() as rowless_session: - repository = RowlessRecoveryRepository(rowless_session) - await repository.claim_dispatch( - authority_id=request_state.rowless_recovery_authority_id, - generation=request_state.rowless_recovery_generation, - replacement_session_id=session.durable_session_id, - request_id=request_state.request_id, - wire_request_fingerprint=request_state.rowless_recovery_wire_fingerprint, - model=request_state.model, - task_authority_digest=request_state.rowless_recovery_task_authority_digest, - ) - send_started = await repository.mark_dispatch_send_started( - authority_id=request_state.rowless_recovery_authority_id, - generation=request_state.rowless_recovery_generation, - request_id=request_state.request_id, - wire_request_fingerprint=request_state.rowless_recovery_wire_fingerprint, - ) - if not send_started: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The automatic semantic-rebase send fence could not be persisted.", - ), - ) - request_state.rowless_recovery_send_primitive_reached = True await _send_http_bridge_request_text_with_archive_id(session, request_state, request_text) session.last_used_at = _service_time().monotonic() request_state.clean_close_retry_result = True return True except asyncio.CancelledError: request_state.clean_close_retry_result = False - if ( - request_state.rowless_recovery_authority_id is not None - and not request_state.rowless_recovery_send_primitive_reached - ): - rollback_task = asyncio.create_task( - self._rollback_rowless_preflight_setup_failure_if_unbound(request_state) - ) - await _await_task_deferring_cancellation(rollback_task) raise - except UpstreamWebSocketTransportError as exc: + except UpstreamWebSocketTransportError: request_state.clean_close_retry_result = False - if request_state.rowless_recovery_authority_id is not None: - if exc.error_code == UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE: - restored = await self._rollback_rowless_cancelled_before_fresh_send(request_state) - if not restored: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The physically-unsent automatic semantic rebase could not be restored.", - ), - ) from exc - elif not request_state.rowless_recovery_send_primitive_reached: - await self._rollback_rowless_preflight_setup_failure_if_unbound(request_state) raise except Exception as exc: request_state.clean_close_retry_result = False - if ( - request_state.rowless_recovery_authority_id is not None - and not request_state.rowless_recovery_send_primitive_reached - ): - await self._rollback_rowless_preflight_setup_failure_if_unbound(request_state) ( request_state.error_http_status_override, request_state.error_code_override, @@ -2979,7 +3372,22 @@ async def _retry_http_bridge_precreated_auth_request( error_message: str | None, ) -> Literal["not_replayable", "retried", "failed"]: permanent_failure_code = _websocket_auth_failure_permanent_code(error_message) - request_text = _prepare_websocket_request_state_for_auth_replay(request_state) + bound_to_current_account = request_state.replay_required_account_id == session.account.id + if bound_to_current_account and ( + _websocket_auth_failure_requires_reauth(error_message) + or request_state.auth_replay_counts_by_account.get(session.account.id, 0) > 0 + ): + failure_code = permanent_failure_code or _WEBSOCKET_AUTH_INVALIDATED_FAILURE_CODE + await self._load_balancer.mark_permanent_failure(session.account, failure_code) + setattr(request_state, "account_health_error_handled", True) + request_state.force_refresh_account_id = None + request_state.preferred_account_id = None + request_state.excluded_account_ids.add(session.account.id) + return "not_replayable" + request_text = _prepare_websocket_request_state_for_auth_replay( + request_state, + current_account_id=session.account.id, + ) if request_text is None: await self._load_balancer.mark_permanent_failure(session.account, permanent_failure_code) setattr(request_state, "account_health_error_handled", True) @@ -3028,10 +3436,14 @@ async def _retry_http_bridge_precreated_auth_request( await self._reconnect_http_bridge_session( session, request_state=request_state, - require_same_account=is_http_bridge_account_neutral_replay( - kind=session.key.affinity_kind, - key=session.key.affinity_key, + require_same_account=( + bound_to_current_account + or is_http_bridge_account_neutral_replay( + kind=session.key.affinity_kind, + key=session.key.affinity_key, + ) ), + require_preferred_account=bound_to_current_account, ) request_text = self._http_bridge_text_with_account_installation_id(session, request_state, request_text) await _send_http_bridge_request_text_with_archive_id(session, request_state, request_text) @@ -3073,13 +3485,13 @@ async def _retry_http_bridge_security_work_request( key=session.key.affinity_key, ): return False - retry_text = request_state.request_text - if not retry_text: - return False if request_state.file_required_preferred_account: return False if not _websocket_request_can_replay_before_visible_output(request_state): return False + retry_text = _prepare_websocket_request_state_for_account_switch(request_state) + if retry_text is None: + return False owner_account_id = session.account.id previous_replay_count = request_state.replay_count @@ -3096,11 +3508,6 @@ async def _retry_http_bridge_security_work_request( session.turn_state_alias_registration_generations ) previous_session_headers = session.headers - if request_state.previous_response_id is not None: - retry_text = _prepare_websocket_request_state_for_account_switch(request_state) - if retry_text is None: - return False - request_state.preferred_account_id = None request_state.excluded_account_ids.add(owner_account_id) request_state.affinity_policy = replace( @@ -3143,6 +3550,13 @@ async def _retry_http_bridge_security_work_request( model_class=_extract_model_class(session.request_model) if session.request_model else None, ) reconnected = False + operation_rebound_for_retry = False + security_retry_send_started = False + + def mark_security_retry_send_started() -> None: + nonlocal security_retry_send_started + security_retry_send_started = True + try: request_state.precreated_replay_account_id = session.account.id await self._release_request_state_account_response_create_lease(request_state) @@ -3167,7 +3581,7 @@ async def _retry_http_bridge_security_work_request( request_state.account_response_create_release = self._load_balancer.release_account_lease if session.account.id != owner_account_id: if ( - previous_session_affinity.codex_session_source == "session_header" + previous_session_affinity.codex_session_source in {"session_header", "thread_header"} and previous_session_affinity.selection_key is not None and previous_session_affinity.kind is not None ): @@ -3177,14 +3591,83 @@ async def _retry_http_bridge_security_work_request( session.account.id, kind=previous_session_affinity.kind, ) + if ( + request_state.operation_registered + and request_state.operation_id is not None + and request_state.operation_fingerprint is not None + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + record_operation = getattr(self._durable_bridge, "record_operation", None) + if not callable(record_operation): + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "Security-work recovery operation could not be re-fenced; retry the request.", + ), + ) + rebound_operation = await record_operation( + operation_id=request_state.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + request_fingerprint=request_state.operation_fingerprint, + api_key_scope=durable_bridge_api_key_scope(session.key.api_key_id), + account_id=session.account.id, + model=request_state.model, + parent_response_id=request_state.operation_parent_response_id or request_state.previous_response_id, + ) + if rebound_operation is None or getattr(rebound_operation, "state", None) != "submitted": + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "Security-work recovery operation could not be re-fenced; retry the request.", + ), + ) + operation_rebound_for_retry = True retry_text = self._http_bridge_text_with_account_installation_id(session, request_state, retry_text) - await _send_http_bridge_request_text_with_archive_id(session, request_state, retry_text) + await _send_http_bridge_request_text_with_archive_id( + session, + request_state, + retry_text, + on_send_started=mark_security_retry_send_started, + ) session.last_used_at = _service_time().monotonic() return True except UpstreamWebSocketTransportError: raise except Exception as exc: logger.warning("HTTP bridge security-work retry failed", exc_info=True) + if ( + operation_rebound_for_retry + and not security_retry_send_started + and request_state.operation_id is not None + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + update_operation = getattr(self._durable_bridge, "update_operation", None) + if callable(update_operation): + try: + restored = await update_operation( + operation_id=request_state.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + state="failed", + ) + if not restored: + logger.info( + "HTTP bridge security retry failed to restore operation fence operation_id=%s", + request_state.operation_id, + ) + except Exception: + logger.warning( + "Failed to restore HTTP bridge security retry operation operation_id=%s", + request_state.operation_id, + exc_info=True, + ) if isinstance(exc, ProxyResponseError): error = _parse_openai_error(exc.payload) code = _normalize_error_code(error.code if error else None, error.type if error else None) @@ -3243,12 +3726,20 @@ async def _claim_http_bridge_replacement_before_swap( if account_id == session.account.id: return try: - if owner_rebind_affinity.legacy_selection_key is not None and owner_rebind_affinity.kind is not None: + if owner_rebind_affinity.legacy_selection_key is not None: async with self._repo_factory() as repos: + # A goal restart abandons only session-header interpretation + # of the legacy raw row. Preserve that typed capability here: + # omitting it would resurrect the retained turn-state owner + # during a later security-authorized replacement. legacy_owner_id = await repos.sticky_sessions.get_account_id( owner_rebind_affinity.legacy_selection_key, - kind=owner_rebind_affinity.kind, - max_age_seconds=owner_rebind_affinity.max_age_seconds, + # The new thread row may be PROMPT_CACHE, but the raw + # compatibility row has always been CODEX_SESSION and + # remains durable hard ownership. + kind=StickySessionKind.CODEX_SESSION, + max_age_seconds=None, + continuity_source=(owner_rebind_affinity.legacy_continuity_source or "session_header"), ) if legacy_owner_id is not None and legacy_owner_id != account_id: raise ProxyResponseError( diff --git a/app/modules/proxy/_service/http_bridge/retry_circuit.py b/app/modules/proxy/_service/http_bridge/retry_circuit.py index cdf459dc29..e2642de2da 100644 --- a/app/modules/proxy/_service/http_bridge/retry_circuit.py +++ b/app/modules/proxy/_service/http_bridge/retry_circuit.py @@ -9,7 +9,11 @@ from app.core.metrics.prometheus import PROMETHEUS_AVAILABLE, http_bridge_retry_circuit_total from app.modules.proxy._service.observability import _hash_identifier -from app.modules.proxy._service.support import _HTTPBridgeSession +from app.modules.proxy._service.support import ( + _HTTPBridgeResponseCreateAttempt, + _HTTPBridgeRetryCircuitAttemptSelection, + _HTTPBridgeSession, +) from app.modules.proxy.durable_bridge_repository import DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS logger = logging.getLogger(__name__) @@ -41,8 +45,13 @@ def _http_bridge_anchor_poison_detail(detail: str | None) -> str | None: - """Map an eventless transport failure to the upstream anchor-poison class.""" + """Map an eventless retry-circuit failure class to its anchor-poison detail. + Consecutive eventless failures on one bridge key are same-anchor failures: + the durable anchor only advances on a completed response, which resets the + circuit. Both ambiguous transport classes therefore count toward anchor + poison (issue #1830); ``clean_close`` never does. + """ if detail is None: return None aliased = _HTTP_BRIDGE_RETRY_CIRCUIT_DETAIL_ALIASES.get(detail, detail) @@ -61,38 +70,104 @@ class _HTTPBridgeRetryCircuitState: half_open_until: float = 0.0 -@dataclass(frozen=True, slots=True) -class _HTTPBridgeRetryCircuitDecision: - allowed: bool - retry_after_seconds: float = 0.0 - last_detail: str | None = None - consecutive_failures: int = 0 +def _initialize_http_bridge_retry_circuit(service: Any, reset_transient_cache: Any = None) -> None: + if reset_transient_cache is not None: + reset_transient_cache() + service._http_bridge_retry_circuits = {} + service._http_bridge_retry_circuit_loaded_keys = set() + service._http_bridge_retry_circuit_persisted_keys = set() + service._http_bridge_retry_circuit_lock = anyio.Lock() -_HTTP_BRIDGE_RETRY_CIRCUIT_DETAIL_MESSAGES = { - "stream_incomplete": "repeated incomplete upstream WebSocket streams", - "clean_close": "repeated clean upstream WebSocket closes", - "stream_idle_timeout": "repeated upstream response timeouts", -} +def _record_http_bridge_retry_circuit_duplicate_suppressed( + session: _HTTPBridgeSession, + *, + attempt: _HTTPBridgeResponseCreateAttempt, + consecutive_failures: int, + detail: str, +) -> None: + if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None: + http_bridge_retry_circuit_total.labels(outcome="duplicate_suppressed").inc() + logger.info( + "http_bridge_retry_circuit event=duplicate_suppressed bridge_kind=%s bridge_key=%s " + "failures=%s detail=%s attempt=%s", + session.key.affinity_kind, + _hash_identifier(session.key.affinity_key), + consecutive_failures, + detail, + attempt.ordinal, + ) -def _http_bridge_retry_circuit_error_message( - detail: str | None, - *, - retry_after_seconds: int, -) -> str: - cause = _HTTP_BRIDGE_RETRY_CIRCUIT_DETAIL_MESSAGES.get(detail, "repeated upstream transport failures") - return f"HTTP responses session bridge is cooling down after {cause}; retry after {retry_after_seconds} seconds." +class _HTTPBridgeRetryCircuitMixin: + async def _http_bridge_retry_circuit_current_count(self: Any, session: _HTTPBridgeSession) -> int: + async with self._http_bridge_retry_circuit_lock: + current_state = self._http_bridge_retry_circuits.get(session.key) + return current_state.consecutive_failures if current_state is not None else 0 + async def _await_http_bridge_retry_circuit_attempt_settlement( + self: Any, + session: _HTTPBridgeSession, + *, + attempt: _HTTPBridgeResponseCreateAttempt, + detail: str, + ) -> int: + settled = attempt.retry_circuit_failure_settled + if settled is not None: + await settled.wait() + consecutive_failures = await self._http_bridge_retry_circuit_current_count(session) + _record_http_bridge_retry_circuit_duplicate_suppressed( + session, + attempt=attempt, + consecutive_failures=consecutive_failures, + detail=detail, + ) + return consecutive_failures -def _initialize_http_bridge_retry_circuit(service: Any) -> None: - service._http_bridge_retry_circuits = {} - service._http_bridge_retry_circuit_loaded_keys = set() - service._http_bridge_retry_circuit_persisted_keys = set() - service._http_bridge_retry_circuit_lock = anyio.Lock() + async def _record_http_bridge_retry_circuit_failure_for_attempt_selection( + self: Any, + session: _HTTPBridgeSession, + *, + detail: str, + selection: _HTTPBridgeRetryCircuitAttemptSelection, + ) -> int | None: + attempt = selection.attempt + if attempt is not None: + return await self._record_http_bridge_retry_circuit_failure( + session, + detail=detail, + attempt=attempt, + ) + if selection.kind == "absent": + return await self._record_http_bridge_retry_circuit_failure(session, detail=detail) + if selection.kind == "recorded": + for recorded_attempt in selection.attempts: + settled = recorded_attempt.retry_circuit_failure_settled + if settled is not None: + await settled.wait() + consecutive_failures = await self._http_bridge_retry_circuit_current_count(session) + for recorded_attempt in selection.attempts: + _record_http_bridge_retry_circuit_duplicate_suppressed( + session, + attempt=recorded_attempt, + consecutive_failures=consecutive_failures, + detail=detail, + ) + return consecutive_failures + outcome = "ambiguous_suppressed" if selection.ambiguous else "ineligible_suppressed" + if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None: + http_bridge_retry_circuit_total.labels(outcome=outcome).inc() + logger.info( + "http_bridge_retry_circuit event=%s bridge_kind=%s bridge_key=%s detail=%s candidate_attempts=%s", + outcome, + session.key.affinity_kind, + _hash_identifier(session.key.affinity_key), + detail, + len(selection.attempts), + ) + return None -class _HTTPBridgeRetryCircuitMixin: def _prune_http_bridge_retry_circuit_state(self: Any, now: float) -> None: expiry = now - DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS for key, state in list(self._http_bridge_retry_circuits.items()): @@ -290,47 +365,34 @@ async def _persist_http_bridge_retry_circuit( exc_info=True, ) - async def _http_bridge_precreated_retry_decision( + async def _http_bridge_precreated_retry_allowed( self: Any, session: _HTTPBridgeSession, *, allow_fresh_hard_account_switch: bool = False, allow_proof_gated_continuity_replay: bool = False, - ) -> _HTTPBridgeRetryCircuitDecision: - """Return one refreshed admission decision for a hard-affinity retry.""" + allow_operation_fenced_continuity_replay: bool = False, + ) -> bool: + """Avoid replaying a repeatedly failing hard-affinity request in a tight loop.""" if session.key.strength != "hard": - return _HTTPBridgeRetryCircuitDecision(allowed=True) + return True await self._load_http_bridge_retry_circuit(session) now = time.monotonic() async with self._http_bridge_retry_circuit_lock: state = self._http_bridge_retry_circuits.get(session.key) - if state is None or state.consecutive_failures < _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD: - # A successful reset can race with a stale durable read, and - # the first new failure intentionally remains below the open - # threshold. Neither state represents an open circuit, even - # if it carries an old cooldown timestamp. - return _HTTPBridgeRetryCircuitDecision( - allowed=True, - last_detail=state.last_detail if state is not None else None, - consecutive_failures=state.consecutive_failures if state is not None else 0, - ) - if state.cooldown_until <= now: + if state is None or state.cooldown_until <= now: if ( - state.half_open_until > now + state is not None + and state.consecutive_failures >= _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD + and state.half_open_until > now and not allow_fresh_hard_account_switch and not allow_proof_gated_continuity_replay ): if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None: http_bridge_retry_circuit_total.labels(outcome="suppressed").inc() - retry_after = max(0.0, state.half_open_until - now) - return _HTTPBridgeRetryCircuitDecision( - allowed=False, - retry_after_seconds=retry_after, - last_detail=state.last_detail, - consecutive_failures=state.consecutive_failures, - ) - if state.cooldown_until > 0: + return False + if state is not None and state.cooldown_until > 0: state.cooldown_until = 0.0 state.half_open_until = now + _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS logger.info( @@ -339,11 +401,7 @@ async def _http_bridge_precreated_retry_decision( _hash_identifier(session.key.affinity_key), state.consecutive_failures, ) - return _HTTPBridgeRetryCircuitDecision( - allowed=True, - last_detail=state.last_detail, - consecutive_failures=state.consecutive_failures, - ) + return True retry_after = max(0.0, state.cooldown_until - now) if allow_fresh_hard_account_switch: @@ -355,12 +413,7 @@ async def _http_bridge_precreated_retry_decision( state.consecutive_failures, retry_after, ) - return _HTTPBridgeRetryCircuitDecision( - allowed=True, - retry_after_seconds=retry_after, - last_detail=state.last_detail, - consecutive_failures=state.consecutive_failures, - ) + return True if allow_proof_gated_continuity_replay: logger.info( "http_bridge_retry_circuit event=bypass_proof_gated_continuity_replay bridge_kind=%s " @@ -370,12 +423,17 @@ async def _http_bridge_precreated_retry_decision( state.consecutive_failures, retry_after, ) - return _HTTPBridgeRetryCircuitDecision( - allowed=True, - retry_after_seconds=retry_after, - last_detail=state.last_detail, - consecutive_failures=state.consecutive_failures, + return True + if allow_operation_fenced_continuity_replay: + logger.info( + "http_bridge_retry_circuit event=bypass_operation_fenced_continuity_replay bridge_kind=%s " + "bridge_key=%s failures=%s retry_after_seconds=%.1f", + session.key.affinity_kind, + _hash_identifier(session.key.affinity_key), + state.consecutive_failures, + retry_after, ) + return True if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None: http_bridge_retry_circuit_total.labels(outcome="suppressed").inc() logger.info( @@ -387,116 +445,104 @@ async def _http_bridge_precreated_retry_decision( retry_after, state.last_detail, ) - return _HTTPBridgeRetryCircuitDecision( - allowed=False, - retry_after_seconds=retry_after, - last_detail=state.last_detail, - consecutive_failures=state.consecutive_failures, - ) - - async def _http_bridge_precreated_retry_allowed( - self: Any, - session: _HTTPBridgeSession, - *, - allow_fresh_hard_account_switch: bool = False, - allow_proof_gated_continuity_replay: bool = False, - ) -> bool: - """Avoid replaying a repeatedly failing hard-affinity request in a tight loop.""" - - decision = await self._http_bridge_precreated_retry_decision( - session, - allow_fresh_hard_account_switch=allow_fresh_hard_account_switch, - allow_proof_gated_continuity_replay=allow_proof_gated_continuity_replay, - ) - return decision.allowed - - async def _http_bridge_retry_circuit_snapshot( - self: Any, - session: _HTTPBridgeSession, - ) -> _HTTPBridgeRetryCircuitDecision: - """Read the active cooldown without consuming half-open admission. - - ``half_open_until`` fences additional submissions after one probe has - already been admitted. It must not be reported as a cooldown to the - admitted request's own streaming path, otherwise that request can - suppress itself before its upstream response starts. - """ + return False + async def _http_bridge_precreated_retry_cooldown_seconds(self: Any, session: _HTTPBridgeSession) -> float: if session.key.strength != "hard": - return _HTTPBridgeRetryCircuitDecision(allowed=True) + return 0.0 await self._load_http_bridge_retry_circuit(session) now = time.monotonic() async with self._http_bridge_retry_circuit_lock: state = self._http_bridge_retry_circuits.get(session.key) - if state is None or state.consecutive_failures < _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD: - return _HTTPBridgeRetryCircuitDecision( - allowed=True, - last_detail=state.last_detail if state is not None else None, - consecutive_failures=state.consecutive_failures if state is not None else 0, - ) - retry_after = max(0.0, state.cooldown_until - now) - return _HTTPBridgeRetryCircuitDecision( - allowed=retry_after <= 0, - retry_after_seconds=retry_after, - last_detail=state.last_detail, - consecutive_failures=state.consecutive_failures, - ) - - async def _http_bridge_precreated_retry_cooldown_seconds(self: Any, session: _HTTPBridgeSession) -> float: - snapshot = await self._http_bridge_retry_circuit_snapshot(session) - return snapshot.retry_after_seconds + if state is None: + return 0.0 + return max(0.0, state.cooldown_until - now) async def _record_http_bridge_retry_circuit_failure( self: Any, session: _HTTPBridgeSession, *, detail: str, + attempt: _HTTPBridgeResponseCreateAttempt | None = None, ) -> int | None: detail = _HTTP_BRIDGE_RETRY_CIRCUIT_DETAIL_ALIASES.get(detail, detail) if session.key.strength != "hard" or detail not in _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_DETAILS: return None + scoped_attempt = attempt + if scoped_attempt is not None: + if scoped_attempt.retry_circuit_failure_recorded: + return await self._await_http_bridge_retry_circuit_attempt_settlement( + session, + attempt=scoped_attempt, + detail=detail, + ) + if scoped_attempt.disarmed or scoped_attempt.response_observed: + return None + await self._load_http_bridge_retry_circuit(session) threshold = max(1, _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD) base_backoff = max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS) max_backoff = max(base_backoff, _HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS) clean_close_max_backoff = max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS) now = time.monotonic() + duplicate_attempt: _HTTPBridgeResponseCreateAttempt | None = None + state: _HTTPBridgeRetryCircuitState | None = None async with self._http_bridge_retry_circuit_lock: - state = self._http_bridge_retry_circuits.setdefault( - session.key, - _HTTPBridgeRetryCircuitState(last_touched_monotonic=now), - ) - state.last_touched_monotonic = now - state.last_failure_monotonic = now - state.half_open_until = 0.0 - state.consecutive_failures += 1 - state.last_detail = detail - if state.consecutive_failures >= threshold: - backoff = min( - max_backoff, - base_backoff * (2 ** min(state.consecutive_failures - threshold, 30)), - ) - if detail == "clean_close": - backoff = min(backoff, clean_close_max_backoff) - state.cooldown_until = max(state.cooldown_until, now + backoff) - if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None: - http_bridge_retry_circuit_total.labels(outcome="opened").inc() - logger.warning( - "http_bridge_retry_circuit event=opened bridge_kind=%s bridge_key=%s " - "failures=%s cooldown_seconds=%.1f detail=%s", - session.key.affinity_kind, - _hash_identifier(session.key.affinity_key), - state.consecutive_failures, - backoff, - detail, + if scoped_attempt is not None and scoped_attempt.retry_circuit_failure_recorded: + duplicate_attempt = scoped_attempt + elif scoped_attempt is not None and (scoped_attempt.disarmed or scoped_attempt.response_observed): + return None + else: + state = self._http_bridge_retry_circuits.setdefault( + session.key, + _HTTPBridgeRetryCircuitState(last_touched_monotonic=now), ) - await self._persist_http_bridge_retry_circuit(session, state) - async with self._http_bridge_retry_circuit_lock: - if self._http_bridge_retry_circuits.get(session.key) is state: - self._http_bridge_retry_circuit_loaded_keys.add(session.key) - return state.consecutive_failures + state.last_touched_monotonic = now + state.last_failure_monotonic = now + state.half_open_until = 0.0 + if scoped_attempt is not None: + scoped_attempt.retry_circuit_failure_recorded = True + scoped_attempt.retry_circuit_failure_settled = anyio.Event() + state.consecutive_failures += 1 + state.last_detail = detail + if state.consecutive_failures >= threshold: + backoff = min( + max_backoff, + base_backoff * (2 ** min(state.consecutive_failures - threshold, 30)), + ) + if detail == "clean_close": + backoff = min(backoff, clean_close_max_backoff) + state.cooldown_until = max(state.cooldown_until, now + backoff) + if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None: + http_bridge_retry_circuit_total.labels(outcome="opened").inc() + logger.warning( + "http_bridge_retry_circuit event=opened bridge_kind=%s bridge_key=%s " + "failures=%s cooldown_seconds=%.1f detail=%s", + session.key.affinity_kind, + _hash_identifier(session.key.affinity_key), + state.consecutive_failures, + backoff, + detail, + ) + if duplicate_attempt is not None: + return await self._await_http_bridge_retry_circuit_attempt_settlement( + session, + attempt=duplicate_attempt, + detail=detail, + ) + assert state is not None + try: + await self._persist_http_bridge_retry_circuit(session, state) + async with self._http_bridge_retry_circuit_lock: + if self._http_bridge_retry_circuits.get(session.key) is state: + self._http_bridge_retry_circuit_loaded_keys.add(session.key) + consecutive_failures = state.consecutive_failures + return consecutive_failures + finally: + if scoped_attempt is not None and scoped_attempt.retry_circuit_failure_settled is not None: + scoped_attempt.retry_circuit_failure_settled.set() async def _clear_http_bridge_retry_circuit(self: Any, session: _HTTPBridgeSession) -> None: if session.key.strength != "hard": diff --git a/app/modules/proxy/_service/http_bridge/service_stubs.py b/app/modules/proxy/_service/http_bridge/service_stubs.py index b4f7ac4a27..86ed81c9a3 100644 --- a/app/modules/proxy/_service/http_bridge/service_stubs.py +++ b/app/modules/proxy/_service/http_bridge/service_stubs.py @@ -376,6 +376,10 @@ def _classify_upstream_close(*args: Any, **kwargs: Any) -> Any: return _service_global("_classify_upstream_close")(*args, **kwargs) +def _is_account_neutral_transport_drop(*args: Any, **kwargs: Any) -> Any: + return _service_global("_is_account_neutral_transport_drop")(*args, **kwargs) + + def _websocket_auth_failure_permanent_code(*args: Any, **kwargs: Any) -> Any: return _service_global("_websocket_auth_failure_permanent_code")(*args, **kwargs) @@ -448,6 +452,10 @@ def _prepare_websocket_request_state_for_account_switch(*args: Any, **kwargs: An return _service_global("_prepare_websocket_request_state_for_account_switch")(*args, **kwargs) +def _websocket_request_text_is_account_neutral_fresh_replay(*args: Any, **kwargs: Any) -> Any: + return _service_global("_websocket_request_text_is_account_neutral_fresh_replay")(*args, **kwargs) + + def _matching_websocket_request_states_for_previous_response_error(*args: Any, **kwargs: Any) -> Any: return _service_global("_matching_websocket_request_states_for_previous_response_error")(*args, **kwargs) diff --git a/app/modules/proxy/_service/http_bridge/session_registry.py b/app/modules/proxy/_service/http_bridge/session_registry.py index c3368a1c74..3129217870 100644 --- a/app/modules/proxy/_service/http_bridge/session_registry.py +++ b/app/modules/proxy/_service/http_bridge/session_registry.py @@ -3,6 +3,7 @@ import asyncio import logging from collections.abc import Mapping +from typing import Any from app.core.clients.proxy import ProxyResponseError from app.core.config.settings import Settings @@ -14,6 +15,8 @@ ) from app.db.models import StickySessionKind from app.modules.proxy._service.http_bridge.helpers import ( + _await_task_deferring_cancellation, + _http_bridge_allow_durable_takeover, _http_bridge_durable_lease_ttl_seconds, _http_bridge_live_previous_response_alias_owner, _http_bridge_live_turn_state_alias_owner, @@ -48,7 +51,6 @@ DurableBridgeAliasRegistrationReceipt, ) from app.modules.proxy.durable_bridge_runtime import http_bridge_owner_process_epoch -from app.modules.proxy.response_transition_manifest import ResponseTransitionManifest logger = logging.getLogger("app.modules.proxy.service") @@ -74,6 +76,71 @@ def _requires_durable_recovery_alias_serialization(session: _HTTPBridgeSession) class _HTTPBridgeSessionRegistryMixin: + async def prune_idle_http_bridge_sessions(self: Any) -> int: + """Run the idle sweep off the request path (issue #1354). + + The sweep is otherwise reached only from + ``_get_or_create_http_bridge_session``, so a replica that stops taking + bridge requests keeps idle sessions' upstream WebSockets open until + restart. Heartbeat-driven, so it runs without traffic or leadership. + """ + async with self._http_bridge_lock: + pruned_sessions = self._prune_http_bridge_sessions_locked() + if not pruned_sessions: + return 0 + self._schedule_http_bridge_session_closes(pruned_sessions, reason="idle_sweep") + return len(pruned_sessions) + + def _initialize_http_bridge_session_registry(self: _HTTPBridgeServiceProtocol) -> None: + # Canonical and detached registries both own live generations until + # common resource finalization removes the latter entry. + self._http_bridge_sessions = {} + self._http_bridge_detached_sessions = {} + + async def close_all_http_bridge_sessions(self: _HTTPBridgeServiceProtocol) -> None: + async with self._http_bridge_lock: + sessions_to_close, inflight_futures = self._take_all_http_bridge_sessions_locked() + shutdown_error = ProxyResponseError( + 503, + openai_error( + "upstream_unavailable", + "HTTP responses session bridge is shutting down", + error_type="server_error", + ), + ) + + async def finish_shutdown() -> None: + for inflight_future in inflight_futures: + if inflight_future.done(): + continue + inflight_future.set_exception(shutdown_error) + inflight_future.exception() + # The registry snapshot is no longer discoverable after the lock is + # released. Start every close concurrently, then await every result, + # so cancellation cannot strand the tail of a sequential close loop. + close_results = await asyncio.gather( + *(self._close_http_bridge_session(session) for session in sessions_to_close), + return_exceptions=True, + ) + await self._drain_http_bridge_background_cleanup_tasks(reason="shutdown") + # Session/background cleanup may still enqueue durable operation + # events, so the spooler must outlive both. Close it before + # propagating an individual session-close failure: the batcher's + # flusher is a service-owned task and must not leak merely because + # one detached generation remains registered for a later retry. + event_batcher = getattr(self, "_http_bridge_operation_event_batcher", None) + close_batcher = getattr(event_batcher, "close", None) + if callable(close_batcher): + await close_batcher() + for result in close_results: + if isinstance(result, BaseException): + raise result + + shutdown_task = asyncio.create_task(finish_shutdown(), name="http-bridge-shutdown-close-all") + _, cancellation = await _await_task_deferring_cancellation(shutdown_task) + if cancellation is not None: + raise cancellation + async def _register_http_bridge_turn_state( self: _HTTPBridgeServiceProtocol, session: _HTTPBridgeSession, @@ -119,7 +186,13 @@ async def _register_http_bridge_turn_state_core( defer_durable_publication = False deferred_live_alias_owner: _HTTPBridgeSession | None = None async with self._http_bridge_lock: - if session.closed: + if session.closed or ( + session.upstream_control.retire_after_drain + and self._http_bridge_sessions.get(session.key) is not session + ): + # A detached predecessor may finish its admitted response, but + # publishing continuity aliases under its reused key would make + # them resolve to the replacement generation (and account). return False, None account_neutral_recovery = is_http_bridge_account_neutral_replay( kind=session.key.affinity_kind, @@ -216,7 +289,6 @@ async def _register_http_bridge_previous_response_id( input_item_count: int | None = None, input_full_fingerprint: str | None = None, pending_tool_calls: Mapping[str, str] | None = None, - response_transition_manifest: ResponseTransitionManifest | None = None, ) -> bool: if _requires_durable_recovery_alias_serialization(session): async with session.recovery_alias_lock: @@ -226,7 +298,6 @@ async def _register_http_bridge_previous_response_id( input_item_count=input_item_count, input_full_fingerprint=input_full_fingerprint, pending_tool_calls=pending_tool_calls, - response_transition_manifest=response_transition_manifest, ) return await self._register_http_bridge_previous_response_id_impl( session, @@ -234,7 +305,6 @@ async def _register_http_bridge_previous_response_id( input_item_count=input_item_count, input_full_fingerprint=input_full_fingerprint, pending_tool_calls=pending_tool_calls, - response_transition_manifest=response_transition_manifest, ) async def _register_http_bridge_previous_response_id_impl( @@ -245,7 +315,6 @@ async def _register_http_bridge_previous_response_id_impl( input_item_count: int | None = None, input_full_fingerprint: str | None = None, pending_tool_calls: Mapping[str, str] | None = None, - response_transition_manifest: ResponseTransitionManifest | None = None, ) -> bool: stripped_response_id = response_id.strip() if not stripped_response_id: @@ -302,7 +371,6 @@ async def _register_http_bridge_previous_response_id_impl( input_item_count=input_item_count, input_full_fingerprint=input_full_fingerprint, pending_tool_calls=pending_tool_calls, - response_transition_manifest=response_transition_manifest, instance_id=_service_get_settings().http_responses_session_bridge_instance_id, lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(), local_alias_was_published=not defer_durable_publication, @@ -373,10 +441,31 @@ def _detach_http_bridge_session_locked( self._http_bridge_sessions.pop(key, None) if mark_closed: session.closed = True + # Detachment removes only canonical routing. Even an idle generation + # marked closed may retain a slow-closing socket, reader, durable lease, + # or account lease, so lifecycle/capacity ownership lasts through the + # common resource-close finalizer for every detached generation. + self._http_bridge_detached_sessions[id(session)] = session self._unregister_http_bridge_turn_states_locked(session) self._unregister_http_bridge_previous_response_ids_locked(session) return session + def _take_all_http_bridge_sessions_locked( + self: _HTTPBridgeServiceProtocol, + ) -> tuple[list[_HTTPBridgeSession], list[asyncio.Future[_HTTPBridgeSession]]]: + sessions = [*self._http_bridge_sessions.values(), *self._http_bridge_detached_sessions.values()] + inflight_futures = list(self._http_bridge_inflight_sessions.values()) + # Shutdown removes canonical routing immediately, but resource ownership + # remains discoverable until each close succeeds. A failed close can + # then be retried by a later shutdown pass instead of orphaning its + # socket, durable lease, account lease, or unsettled requests. + for session in self._http_bridge_sessions.values(): + self._http_bridge_detached_sessions[id(session)] = session + self._http_bridge_sessions.clear() + self._http_bridge_inflight_sessions.clear() + self._http_bridge_previous_response_index.clear() + return sessions, inflight_futures + def _unregister_http_bridge_turn_states_locked( self: _HTTPBridgeServiceProtocol, session: _HTTPBridgeSession, @@ -436,10 +525,9 @@ async def _claim_durable_http_bridge_session( *, allow_takeover: bool, force_owner_epoch_advance: bool = False, - expected_takeover_owner_instance_id: str | None = None, - expected_takeover_owner_process_epoch: str | None = None, claim_account_id: str | None = None, clear_latest_turn_state: bool = False, + record_restart_takeover: bool = False, ) -> None: current_instance = _service_get_settings().http_responses_session_bridge_instance_id current_process_epoch = http_bridge_owner_process_epoch() @@ -460,13 +548,18 @@ async def _claim_durable_http_bridge_session( latest_response_id=None, allow_takeover=allow_takeover, force_owner_epoch_advance=force_owner_epoch_advance or claim_attempt > 0, - expected_takeover_owner_instance_id=expected_takeover_owner_instance_id, - expected_takeover_owner_process_epoch=expected_takeover_owner_process_epoch, ) if lookup.owner_instance_id == current_instance: break if not allow_takeover or claim_attempt > 0: break + if not _http_bridge_allow_durable_takeover(lookup): + # The claim reported a live foreign owner: we lost the race + # rather than hitting transient contention. The repository + # already dropped its takeover permission for that reason, + # and retrying here with a fresh call would restore it and + # steal the winner's live lease (issue #1695). + break await asyncio.sleep(0) assert lookup is not None if lookup.owner_instance_id != current_instance: @@ -499,7 +592,7 @@ async def _claim_durable_http_bridge_session( if ( PROMETHEUS_AVAILABLE and bridge_durable_recover_total is not None - and allow_takeover + and record_restart_takeover and lookup.owner_epoch > 1 ): bridge_durable_recover_total.labels(path="restart_takeover").inc() diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index 597b779541..1285996fad 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -6,11 +6,10 @@ import logging import math from collections.abc import AsyncGenerator, Callable -from typing import Any, AsyncIterator, Literal, Mapping, TypeVar, cast +from typing import Any, AsyncIterator, Mapping, TypeVar, cast from uuid import uuid4 import anyio -from sqlalchemy import select from app.core.clients.files import create_file as core_create_file # noqa: F401 from app.core.clients.files import finalize_file as core_finalize_file # noqa: F401 @@ -36,7 +35,11 @@ from app.core.clients.proxy import compact_responses as core_compact_responses # noqa: F401 from app.core.clients.proxy import transcribe_audio as core_transcribe_audio # noqa: F401 from app.core.clients.proxy_websocket import UpstreamWebSocketTransportError -from app.core.errors import openai_error, response_failed_event +from app.core.errors import ( + OpenAIErrorEnvelope, + openai_error, + response_failed_event, +) from app.core.metrics.prometheus import ( PROMETHEUS_AVAILABLE, bridge_durable_recover_total, @@ -52,12 +55,9 @@ from app.core.utils.sse import format_sse_event, parse_sse_data_json from app.core.utils.time import utcnow from app.db.models import ( - Account, - HttpBridgeRowlessRecoveryState, HttpBridgeSessionState, StickySessionKind, ) -from app.db.session import SessionLocal from app.modules.api_keys.service import ( ApiKeyData, ApiKeyUsageReservationData, @@ -72,7 +72,6 @@ _sticky_key_from_compact_payload as _sticky_key_from_compact_payload, ) from app.modules.proxy._service.http_bridge.helpers import ( - _await_task_deferring_cancellation, _effective_http_bridge_idle_ttl_seconds, _http_bridge_durable_lease_ttl_seconds, _http_bridge_durable_lookup_allows_turn_state_takeover, @@ -86,6 +85,8 @@ _http_bridge_request_budget_seconds, _http_bridge_request_needs_unanchored_handoff, _http_bridge_request_stage, + _http_bridge_requires_cluster_registration, + _http_bridge_retry_circuit_attempt_selection_for_pending_requests, _http_bridge_runtime_config, _http_bridge_should_attempt_local_bootstrap_rebind, _http_bridge_should_attempt_local_previous_response_recovery, @@ -104,14 +105,11 @@ _reserve_http_bridge_unanchored_handoff, _trim_http_bridge_previous_response_input_items, ) +from app.modules.proxy._service.http_bridge.owner_forwarding import ( + _owner_forward_failure_allows_local_recovery, +) from app.modules.proxy._service.http_bridge.quarantine import ( - _HTTP_BRIDGE_QUARANTINE_REJECTED_STALE_ANCHOR_REASON, _http_bridge_session_key_quarantined, - _quarantine_http_bridge_session, -) -from app.modules.proxy._service.http_bridge.request_submit import _text_with_account_installation_id -from app.modules.proxy._service.http_bridge.retry_circuit import ( - _http_bridge_retry_circuit_error_message, ) from app.modules.proxy._service.http_bridge.service_stubs import ( _build_rewritten_stream_response_failed_event, @@ -172,6 +170,7 @@ _is_local_account_cap_code, _signal_propagated_capacity_startup_ready, _signal_propagated_capacity_startup_wait, + _signal_propagated_responses_service_cleanup_ready, _ttft_event_visible_at, _WebSocketRequestState, ) @@ -210,6 +209,7 @@ ) from app.modules.proxy.affinity import ( _AffinityPolicy, + _codex_backend_identity, _extract_model_class, _prompt_cache_key_from_request_model, _request_allows_bare_session_cap_spillover, @@ -225,143 +225,22 @@ without_http_bridge_session_affinity_headers, ) from app.modules.proxy.durable_bridge_coordinator import DurableBridgeLookup -from app.modules.proxy.durable_bridge_repository import ( - durable_bridge_api_key_scope, - durable_bridge_hash, -) +from app.modules.proxy.durable_bridge_repository import durable_bridge_hash from app.modules.proxy.durable_bridge_runtime import http_bridge_owner_process_epoch from app.modules.proxy.helpers import ( _normalize_error_code, ) from app.modules.proxy.replay_safety import ( - AccountNeutralCodexTurnMetadataEvidence, - abandoned_pending_agent_boundary_rejection_reason, - account_neutral_codex_turn_metadata_identity, - project_responses_input_for_abandoned_pending_fresh_replay, project_responses_input_for_account_neutral_fresh_replay, responses_input_suffix_matches_pending_tool_calls, - responses_input_suffix_matches_transition_manifest, responses_input_suffix_retains_prior_output, responses_payload_is_account_neutral_fresh_replay, ) -from app.modules.proxy.rowless_recovery import ( - RowlessRecoveryCaptureIntent, - approved_rowless_recovery_projection, - build_rowless_recovery_capture_facts, - rowless_actual_wire_fingerprint, - rowless_strong_session_hash, - rowless_task_authority_digest, -) -from app.modules.proxy.rowless_recovery_repository import ( - RowlessRecoveryRepository, - RowlessRecoveryStateError, -) logger = logging.getLogger("app.modules.proxy.service") T = TypeVar("T") _REQUEST_TRANSPORT_HTTP = "http" _RESPONSE_CREATE_GATE_RETRY_SLEEP_SECONDS = 10.0 -# Keep the old authority/marker schemas readable for rollback safety, but use -# the upstream durable-owner/full-resend recovery path for all live requests. -_DURABLE_RECOVERY_MARKER_REQUEST_PATH_ENABLED = False -_ROWLESS_SEMANTIC_REBASE_REQUEST_PATH_ENABLED = False -_OBSERVABLE_REPLAY_SUFFIX_ITEM_TYPES = frozenset( - { - "agent_message", - "apply_patch_call", - "apply_patch_call_output", - "custom_tool_call", - "custom_tool_call_output", - "function_call", - "function_call_output", - "reasoning", - } -) - - -def _durable_recovery_marker_request_path_active(lookup: DurableBridgeLookup | None) -> bool: - return bool( - _DURABLE_RECOVERY_MARKER_REQUEST_PATH_ENABLED - and lookup is not None - and lookup.recovery_is_required_for_latest_anchor() - ) - - -def _full_resend_suffix_shape_for_observability( - input_items: list[JsonValue], - *, - stored_count: int, -) -> str: - """Return a bounded, content-free shape for rejected replay proofs.""" - - labels: list[str] = [] - for item in input_items[stored_count : stored_count + 8]: - if not isinstance(item, dict): - labels.append("scalar") - continue - item_type = item.get("type") - role = item.get("role") - if ( - item_type in (None, "message") - and isinstance(role, str) - and role in {"assistant", "developer", "system", "user"} - ): - labels.append(str(role)) - elif isinstance(item_type, str) and item_type in _OBSERVABLE_REPLAY_SUFFIX_ITEM_TYPES: - labels.append(item_type) - elif isinstance(item_type, str) and item_type in {"input_file", "input_image", "input_text"}: - labels.append("input_part") - else: - labels.append("other") - if len(input_items) - stored_count > 8: - labels.append("more") - return ">".join(labels) or "empty" - - -def _log_abandoned_pending_full_resend_rejection( - *, - bridge_session_key: _HTTPBridgeSessionKey, - payload: ResponsesRequest, - durable_lookup: DurableBridgeLookup | None, - reason_code: str | None, - stage: str, -) -> None: - """Log one bounded proof branch without request content or raw ids.""" - - if ( - reason_code is None - or durable_lookup is None - or not durable_lookup.latest_pending_tool_calls - or not isinstance(payload.input, list) - or not _http_bridge_payload_looks_like_full_resend(payload) - ): - return - stored_count = durable_lookup.latest_input_item_count - if stored_count is None or stored_count < 0 or stored_count > len(payload.input): - stored_items = "invalid" - suffix_items = "invalid" - suffix_shape = "unavailable" - else: - stored_items = str(stored_count) - suffix_items = str(len(payload.input) - stored_count) - suffix_shape = _full_resend_suffix_shape_for_observability( - payload.input, - stored_count=stored_count, - ) - _log_http_bridge_event( - "abandoned_pending_full_resend_proof_rejected", - bridge_session_key, - account_id=None, - model=payload.model, - detail=( - f"reason_code={reason_code},stage={stage}," - f"stored_items={stored_items},suffix_items={suffix_items}," - f"suffix_shape={suffix_shape}" - ), - cache_key_family=bridge_session_key.affinity_kind, - model_class=_extract_model_class(payload.model) if payload.model else None, - owner_check_applied=True, - ) def _http_bridge_continuity_bound_without_safe_replay(request_state: _WebSocketRequestState) -> bool: @@ -373,6 +252,11 @@ def _http_bridge_continuity_bound_without_safe_replay(request_state: _WebSocketR ) +def _http_bridge_durable_recovery_predecessor_proven(request_state: _WebSocketRequestState) -> bool: + """Return whether the operation has a durable predecessor anchor.""" + return request_state.previous_response_id is not None or request_state.operation_parent_response_id is not None + + class _VerifiedDurableFullResend: """Immutable proof that one payload contains a durable turn's complete context.""" @@ -383,7 +267,6 @@ class _VerifiedDurableFullResend: _pending_tool_calls: tuple[tuple[str, str], ...] | None _stored_input_fingerprint: str _stored_input_item_count: int - _transition_manifest_digest: str | None __slots__ = ( "_durable_session_id", "_full_input_fingerprint", @@ -392,7 +275,6 @@ class _VerifiedDurableFullResend: "_pending_tool_calls", "_stored_input_fingerprint", "_stored_input_item_count", - "_transition_manifest_digest", ) __construction_token = object() @@ -407,7 +289,6 @@ def __init__( stored_input_fingerprint: str, full_input_fingerprint: str, pending_tool_calls: tuple[tuple[str, str], ...] | None, - transition_manifest_digest: str | None = None, ) -> None: if _token is not self.__construction_token: raise TypeError("verified durable full resend proofs are created only by the verifier") @@ -418,7 +299,6 @@ def __init__( object.__setattr__(self, "_stored_input_fingerprint", stored_input_fingerprint) object.__setattr__(self, "_full_input_fingerprint", full_input_fingerprint) object.__setattr__(self, "_pending_tool_calls", pending_tool_calls) - object.__setattr__(self, "_transition_manifest_digest", transition_manifest_digest) def __setattr__(self, _name: str, _value: object) -> None: raise AttributeError("verified durable full resend proofs are immutable") @@ -451,12 +331,6 @@ def matches( and durable_lookup.latest_input_item_count == self._stored_input_item_count and durable_lookup.latest_input_full_fingerprint == self._stored_input_fingerprint and _pending_tool_calls_identity(durable_lookup.latest_pending_tool_calls) == self._pending_tool_calls - and ( - durable_lookup.latest_response_transition_manifest.digest - if durable_lookup.latest_response_transition_manifest is not None - else None - ) - == self._transition_manifest_digest and _fingerprint_input_items(cast(list[JsonValue], input_items)) == self._full_input_fingerprint ) @@ -492,46 +366,27 @@ def _verify( # inline Responses-Lite developer IDs must remain visible until # the exact-manifest check rejects response-owned messages. preserve_developer_message_ids=True, - preserve_response_owned_agent_message_ids=True, ) pending_tool_calls = durable_lookup.latest_pending_tool_calls if replay_projection is None: return None - transition_manifest = durable_lookup.latest_response_transition_manifest - if transition_manifest is not None: - safe_fresh_context = pending_tool_calls is not None and responses_input_suffix_matches_transition_manifest( - input_items, - stored_count=stored_count, - response_id=latest_response_id, + safe_fresh_context = responses_input_suffix_retains_prior_output( + replay_projection.input_items, + stored_count=replay_projection.stored_prefix_count, + canonical_lite_developer_index=replay_projection.canonical_lite_developer_index, + ) or ( + pending_tool_calls is not None + and responses_input_suffix_matches_pending_tool_calls( + replay_projection.input_items, + stored_count=replay_projection.stored_prefix_count, pending_tool_calls=pending_tool_calls, - transition_manifest=transition_manifest, - ) - else: - safe_fresh_context = ( - False - if pending_tool_calls is None - else responses_input_suffix_matches_pending_tool_calls( - replay_projection.input_items, - stored_count=replay_projection.stored_prefix_count, - pending_tool_calls=pending_tool_calls, - canonical_lite_developer_index=replay_projection.canonical_lite_developer_index, - ) - if pending_tool_calls - else responses_input_suffix_retains_prior_output( - replay_projection.input_items, - stored_count=replay_projection.stored_prefix_count, - canonical_lite_developer_index=replay_projection.canonical_lite_developer_index, - # The prefix fingerprint matched the exact context previously - # completed by this same-account session. Only the new - # inter-agent boundary additionally requires an explicitly - # empty manifest. - exact_stored_prefix_without_pending_manifest=True, - allow_response_owned_agent_message=pending_tool_calls == {}, - ) + canonical_lite_developer_index=replay_projection.canonical_lite_developer_index, ) + ) if not safe_fresh_context: return None - return cls._seal( + return cls( + _token=cls.__construction_token, durable_session_id=durable_lookup.session_id, owner_account_id=owner_account_id, latest_response_id=latest_response_id, @@ -539,32 +394,6 @@ def _verify( stored_input_fingerprint=stored_fingerprint, full_input_fingerprint=_fingerprint_input_items(input_items), pending_tool_calls=_pending_tool_calls_identity(pending_tool_calls), - transition_manifest_digest=(transition_manifest.digest if transition_manifest is not None else None), - ) - - @classmethod - def _seal( - cls, - *, - durable_session_id: str, - owner_account_id: str, - latest_response_id: str, - stored_input_item_count: int, - stored_input_fingerprint: str, - full_input_fingerprint: str, - pending_tool_calls: tuple[tuple[str, str], ...] | None, - transition_manifest_digest: str | None = None, - ) -> "_VerifiedDurableFullResend": - return cls( - _token=cls.__construction_token, - durable_session_id=durable_session_id, - owner_account_id=owner_account_id, - latest_response_id=latest_response_id, - stored_input_item_count=stored_input_item_count, - stored_input_fingerprint=stored_input_fingerprint, - full_input_fingerprint=full_input_fingerprint, - pending_tool_calls=pending_tool_calls, - transition_manifest_digest=transition_manifest_digest, ) @@ -583,269 +412,50 @@ def _verify_durable_full_resend( return _VerifiedDurableFullResend._verify(payload, durable_lookup) -def _verify_durable_abandoned_pending_full_resend( - payload: ResponsesRequest, - durable_lookup: DurableBridgeLookup | None, -) -> _VerifiedDurableFullResend | None: - """Seal a full resend that may recover only after exact anchor rejection. +def _http_bridge_client_full_history_recovery_enabled(request_state: _WebSocketRequestState) -> bool: + """Return whether an ambiguous anchored turn may fall back to client replay. - Unlike ``_verify_durable_full_resend``, this proof does not make a request - generally replay-safe. It binds an exact stored prefix, a non-empty - durable pending-call manifest, and a later response-owned inter-agent - boundary whose client history contains none of those pending call ids. - Callers may use it only after upstream rejects the exact durable anchor - before producing any response event. + The client can recover an unknown upstream handoff by dropping + ``previous_response_id`` and resending its full local history. This is + intentionally opt-in: upstream acceptance is still ambiguous and the + fallback therefore has at-least-once (possible duplicate) semantics. """ - - proof, _reason_code = _verify_durable_abandoned_pending_full_resend_with_reason( - payload, - durable_lookup, + settings = _service_get_settings() + return ( + getattr(settings, "http_responses_session_bridge_ambiguous_continuation_recovery_mode", "fail_closed") + == "client_full_history_once" + and request_state.previous_response_id is not None + and request_state.response_id is None + and request_state.response_event_count == 0 + and not request_state.fresh_upstream_request_is_retry_safe ) - return proof - -def _verify_durable_abandoned_pending_full_resend_with_reason( - payload: ResponsesRequest, - durable_lookup: DurableBridgeLookup | None, -) -> tuple[_VerifiedDurableFullResend | None, str | None]: - """Return a sealed proof or its first content-free rejection branch.""" - if durable_lookup is None: - return None, "durable_lookup_missing" - owner_account_id = durable_lookup.account_id - latest_response_id = durable_lookup.latest_response_id - stored_count = durable_lookup.latest_input_item_count - stored_fingerprint = durable_lookup.latest_input_full_fingerprint - pending_tool_calls = durable_lookup.latest_pending_tool_calls - if owner_account_id is None: - return None, "durable_owner_missing" - if latest_response_id is None: - return None, "durable_anchor_missing" - if stored_count is None or stored_count <= 0 or stored_fingerprint is None: - return None, "stored_prefix_invalid" - if not pending_tool_calls: - return None, "pending_call_manifest_missing" - if not isinstance(payload.input, list): - return None, "payload_input_not_list" - if not _http_bridge_payload_looks_like_full_resend(payload): - return None, "payload_not_full_resend" - if not _input_prefix_matches_stored_context( - payload.input, - stored_count=stored_count, - stored_fingerprint=stored_fingerprint, - ): - return None, "stored_prefix_mismatch" - input_items = cast(list[JsonValue], payload.input) - boundary_rejection = abandoned_pending_agent_boundary_rejection_reason( - input_items, - stored_count=stored_count, - pending_tool_calls=pending_tool_calls, - ) - if boundary_rejection is not None: - return None, boundary_rejection +def _http_bridge_server_anchored_replay_enabled(request_state: _WebSocketRequestState) -> bool: + """Return whether the one permitted server-side anchored replay is unused.""" + settings = _service_get_settings() return ( - _VerifiedDurableFullResend._seal( - durable_session_id=durable_lookup.session_id, - owner_account_id=owner_account_id, - latest_response_id=latest_response_id, - stored_input_item_count=stored_count, - stored_input_fingerprint=stored_fingerprint, - full_input_fingerprint=_fingerprint_input_items(input_items), - pending_tool_calls=_pending_tool_calls_identity(pending_tool_calls), - transition_manifest_digest=( - durable_lookup.latest_response_transition_manifest.digest - if durable_lookup.latest_response_transition_manifest is not None - else None - ), - ), - None, + getattr(settings, "http_responses_session_bridge_ambiguous_continuation_recovery_mode", "fail_closed") + in {"server_anchored_replay_once", "server_indefinite_recovery"} + and request_state.previous_response_id is not None + and request_state.response_id is None + and request_state.response_event_count == 0 + and ( + request_state.replay_count == 0 + or getattr(settings, "http_responses_session_bridge_ambiguous_continuation_recovery_mode", "") + == "server_indefinite_recovery" + ) ) -class _VerifiedStoreContextFullResend: - """Request-local proof for an exact store-context prefix trim. - - This proof is intentionally separate from durable full-resend evidence. - It is minted only while the live bridge still owns the response anchor and - only after the incoming full input exactly matches that bridge's stored - prefix. The complete untrimmed request therefore remains available for a - single same-account recovery if upstream rejects the proxy-injected anchor - before producing any response event. - """ - - _affinity_key: str - _affinity_kind: str - _api_key_id: str | None - _full_input_fingerprint: str - _latest_response_id: str - _owner_account_id: str - _pending_tool_calls: tuple[tuple[str, str], ...] | None - _stored_input_fingerprint: str - _stored_input_item_count: int - _transition_manifest_digest: str | None - __slots__ = ( - "_affinity_key", - "_affinity_kind", - "_api_key_id", - "_full_input_fingerprint", - "_latest_response_id", - "_owner_account_id", - "_pending_tool_calls", - "_stored_input_fingerprint", - "_stored_input_item_count", - "_transition_manifest_digest", +def _http_bridge_client_full_history_recovery_error() -> OpenAIErrorEnvelope: + payload = openai_error( + "previous_response_not_found", + "Previous response was not found; retry without previous_response_id.", + error_type="invalid_request_error", ) - __construction_token = object() - - def __init__( - self, - *, - _token: object, - affinity_kind: str, - affinity_key: str, - api_key_id: str | None, - owner_account_id: str, - latest_response_id: str, - stored_input_item_count: int, - stored_input_fingerprint: str, - full_input_fingerprint: str, - pending_tool_calls: tuple[tuple[str, str], ...] | None, - transition_manifest_digest: str | None = None, - ) -> None: - if _token is not self.__construction_token: - raise TypeError("verified store-context full resend proofs are created only by the verifier") - object.__setattr__(self, "_affinity_kind", affinity_kind) - object.__setattr__(self, "_affinity_key", affinity_key) - object.__setattr__(self, "_api_key_id", api_key_id) - object.__setattr__(self, "_owner_account_id", owner_account_id) - object.__setattr__(self, "_latest_response_id", latest_response_id) - object.__setattr__(self, "_stored_input_item_count", stored_input_item_count) - object.__setattr__(self, "_stored_input_fingerprint", stored_input_fingerprint) - object.__setattr__(self, "_full_input_fingerprint", full_input_fingerprint) - object.__setattr__(self, "_pending_tool_calls", pending_tool_calls) - object.__setattr__(self, "_transition_manifest_digest", transition_manifest_digest) - - def __setattr__(self, _name: str, _value: object) -> None: - raise AttributeError("verified store-context full resend proofs are immutable") - - def __copy__(self) -> "_VerifiedStoreContextFullResend": - return self - - def __deepcopy__(self, _memo: dict[int, object]) -> "_VerifiedStoreContextFullResend": - return self - - def __reduce_ex__(self, _protocol: object) -> str | tuple[Any, ...]: - raise TypeError("verified store-context full resend proofs cannot be serialized") - - def matches(self, payload: ResponsesRequest, session: _HTTPBridgeSession) -> bool: - input_items = payload.input - return ( - payload.previous_response_id is None - and isinstance(input_items, list) - and session.key.affinity_kind == self._affinity_kind - and session.key.affinity_key == self._affinity_key - and session.key.api_key_id == self._api_key_id - and session.account.id == self._owner_account_id - and session.last_completed_response_account_id == self._owner_account_id - and session.last_completed_response_id == self._latest_response_id - and session.last_completed_input_count == self._stored_input_item_count - and session.last_completed_input_prefix_fingerprint == self._stored_input_fingerprint - and not session.last_pending_tool_call_manifest_invalid - and _pending_tool_calls_identity(session.last_pending_tool_calls) == self._pending_tool_calls - and ( - session.last_response_transition_manifest.digest - if session.last_response_transition_manifest is not None - else None - ) - == self._transition_manifest_digest - and _fingerprint_input_items(cast(list[JsonValue], input_items)) == self._full_input_fingerprint - ) - - @classmethod - def _verify( - cls, - payload: ResponsesRequest, - session: _HTTPBridgeSession, - ) -> "_VerifiedStoreContextFullResend | None": - owner_account_id = session.last_completed_response_account_id - latest_response_id = session.last_completed_response_id - stored_count = session.last_completed_input_count - stored_fingerprint = session.last_completed_input_prefix_fingerprint - if ( - payload.previous_response_id is not None - or owner_account_id is None - or owner_account_id != session.account.id - or latest_response_id is None - or stored_count <= 0 - or stored_fingerprint is None - or session.last_pending_tool_call_manifest_invalid - or not _http_bridge_payload_looks_like_full_resend(payload) - or not isinstance(payload.input, list) - or not _input_prefix_matches_stored_context( - payload.input, - stored_count=stored_count, - stored_fingerprint=stored_fingerprint, - ) - ): - return None - input_items = cast(list[JsonValue], payload.input) - replay_projection = project_responses_input_for_account_neutral_fresh_replay( - input_items, - stored_count=stored_count, - preserve_developer_message_ids=True, - preserve_response_owned_agent_message_ids=True, - ) - pending_tool_calls = session.last_pending_tool_calls - if replay_projection is None: - return None - transition_manifest = session.last_response_transition_manifest - if transition_manifest is not None: - safe_fresh_context = responses_input_suffix_matches_transition_manifest( - input_items, - stored_count=stored_count, - response_id=latest_response_id, - pending_tool_calls=pending_tool_calls, - transition_manifest=transition_manifest, - ) - else: - safe_fresh_context = ( - responses_input_suffix_matches_pending_tool_calls( - replay_projection.input_items, - stored_count=replay_projection.stored_prefix_count, - pending_tool_calls=pending_tool_calls, - canonical_lite_developer_index=replay_projection.canonical_lite_developer_index, - ) - if pending_tool_calls - else responses_input_suffix_retains_prior_output( - replay_projection.input_items, - stored_count=replay_projection.stored_prefix_count, - canonical_lite_developer_index=replay_projection.canonical_lite_developer_index, - exact_stored_prefix_without_pending_manifest=True, - allow_response_owned_agent_message=pending_tool_calls == {}, - ) - ) - if not safe_fresh_context: - return None - return cls( - _token=cls.__construction_token, - affinity_kind=session.key.affinity_kind, - affinity_key=session.key.affinity_key, - api_key_id=session.key.api_key_id, - owner_account_id=owner_account_id, - latest_response_id=latest_response_id, - stored_input_item_count=stored_count, - stored_input_fingerprint=stored_fingerprint, - full_input_fingerprint=_fingerprint_input_items(input_items), - pending_tool_calls=_pending_tool_calls_identity(pending_tool_calls), - transition_manifest_digest=(transition_manifest.digest if transition_manifest is not None else None), - ) - - -def _verify_store_context_full_resend( - payload: ResponsesRequest, - session: _HTTPBridgeSession, -) -> _VerifiedStoreContextFullResend | None: - return _VerifiedStoreContextFullResend._verify(payload, session) + payload["error"]["param"] = "previous_response_id" + return payload _HTTP_BRIDGE_DEAD_OWNER_NOT_FOUND_DETAIL = "The previous bridge owner is no longer available." @@ -1219,80 +829,6 @@ async def _registered_turn_state_anchor_lookup( return lookup -def _rowless_client_metadata_evidence( - *, - raw_client_metadata: JsonValue | None, - normalized_headers: Mapping[str, str], - session_id: str, - task_identity: str, -) -> tuple[bool, bool, bool, bool]: - """Validate body and direct-header Codex identity carriers.""" - - if raw_client_metadata is None: - client_metadata: Mapping[str, JsonValue] = {} - elif isinstance(raw_client_metadata, Mapping): - client_metadata = raw_client_metadata - else: - return False, False, False, False - - child_signal = any( - key in normalized_headers or key in client_metadata for key in ("x-codex-parent-thread-id", "x-openai-subagent") - ) - metadata_thread_present = False - - for key, expected in (("session_id", session_id), ("thread_id", task_identity)): - if key not in client_metadata: - continue - value = client_metadata[key] - if not isinstance(value, str) or not value.strip() or value.strip() != expected: - return False, child_signal, metadata_thread_present, False - if key == "thread_id": - metadata_thread_present = True - - flat_turn_id = client_metadata.get("turn_id") - if flat_turn_id is not None and not (isinstance(flat_turn_id, str) and bool(flat_turn_id.strip())): - return False, child_signal, metadata_thread_present, False - turn_metadata_carriers: list[tuple[JsonValue, Literal["body", "direct"]]] = [] - if "x-codex-turn-metadata" in client_metadata: - turn_metadata_carriers.append((client_metadata["x-codex-turn-metadata"], "body")) - if "x-codex-turn-metadata" in normalized_headers: - turn_metadata_carriers.append((normalized_headers["x-codex-turn-metadata"], "direct")) - carrier_identities: list[AccountNeutralCodexTurnMetadataEvidence] = [] - for raw_turn_metadata, carrier in turn_metadata_carriers: - identity = account_neutral_codex_turn_metadata_identity( - raw_turn_metadata, - carrier=carrier, - expected_session_identity=session_id, - expected_task_identity=task_identity, - expected_turn_identity=flat_turn_id, - ) - if identity is None: - return False, child_signal, metadata_thread_present, False - carrier_identities.append(identity) - metadata_thread_present = True - if len(set(carrier_identities)) > 1: - return False, child_signal, metadata_thread_present, False - if carrier_identities: - canonical = carrier_identities[0] - flat_projection_pairs = ( - (client_metadata.get("root_turn_id"), canonical.root_turn_identity), - (client_metadata.get("x-codex-installation-id"), canonical.installation_identity), - (client_metadata.get("x-codex-window-id"), canonical.window_identity), - (normalized_headers.get("x-codex-installation-id"), canonical.installation_identity), - (normalized_headers.get("x-codex-window-id"), canonical.window_identity), - ) - if any(flat is not None and flat != nested for flat, nested in flat_projection_pairs): - return False, child_signal, metadata_thread_present, False - - automatic_live_recovery = bool( - carrier_identities - and carrier_identities[0].workspace_kind is not None - and carrier_identities[0].workspace_kind.strip() - and carrier_identities[0].forked_from_thread_identity is None - ) - return True, child_signal, metadata_thread_present, automatic_live_recovery - - class _HTTPBridgeStreamingMixin: async def validate_http_bridge_legacy_forward_anchor( self: Any, @@ -1399,17 +935,11 @@ async def _stream_http_bridge_or_retry( payload_size_estimate_bytes = len( json.dumps(payload.to_payload(), ensure_ascii=True, separators=(",", ":")).encode("utf-8") ) - # File pins are process-local. A remote owner must trust only the - # origin-resolved value carried by the authenticated forward context; - # re-looking it up here would turn a valid cross-replica pin into a miss. - local_file_owner_account_id = ( - None - if forwarded_file_owner_account_id is not None - else await self._resolve_file_account_for_responses(payload, headers) - ) - rewritten_file_account_id = resolve_required_account_id( - ("signed forwarding context", forwarded_file_owner_account_id), - ("local file pin", local_file_owner_account_id), + rewritten_file_account_id = await self._resolve_forwarded_file_account_for_responses( + payload, + headers, + forwarded_file_owner_account_id=forwarded_file_owner_account_id, + require_forwarded_file_owner=forwarded_request, ) ws_payload_budget_bytes = _ws_transport_payload_budget_bytes(_service_get_settings()) if runtime_config.enabled and payload_size_estimate_bytes > ws_payload_budget_bytes: @@ -1445,6 +975,7 @@ async def _stream_http_bridge_or_retry( suppress_text_done_events=suppress_text_done_events, request_transport=_REQUEST_TRANSPORT_HTTP, rewritten_file_account_id=rewritten_file_account_id, + file_account_resolution_complete=True, upstream_stream_transport_override=force_upstream_stream_transport, client_ip=client_ip, enforce_openai_sdk_contract=enforce_openai_sdk_contract, @@ -1653,7 +1184,9 @@ async def release_unowned_bridge_lifecycle( api_key=api_key, ) sticky_key_source = "none" - if affinity.kind == StickySessionKind.CODEX_SESSION: + if affinity.codex_session_source == "thread_header": + sticky_key_source = "thread_header" + elif affinity.kind == StickySessionKind.CODEX_SESSION: sticky_key_source = ( "turn_state_header" if _sticky_key_from_turn_state_header(headers) is not None else "session_header" ) @@ -1697,6 +1230,13 @@ async def release_unowned_bridge_lifecycle( if not forwarded_request else None ) + durable_session_header_alias = ( + None + if _codex_backend_identity(headers).thread_id is not None + else session_header_fallback_key.affinity_key + if explicit_prompt_cache_key is not None and session_header_fallback_key is not None + else incoming_session_header + ) legacy_anchor_lookup = await _legacy_forward_anchor_lookup( durable_bridge=self._durable_bridge, bridge_session_key=bridge_session_key, @@ -1724,11 +1264,9 @@ async def release_unowned_bridge_lifecycle( session_key_value=bridge_session_key.affinity_key, api_key_id=bridge_session_key.api_key_id, turn_state=durable_lookup_turn_state, - session_header=( - session_header_fallback_key.affinity_key - if explicit_prompt_cache_key is not None and session_header_fallback_key is not None - else incoming_session_header - ), + # A raw process alias is ambiguous when thread-id exists. + # Exact turn/response aliases remain independent inputs. + session_header=durable_session_header_alias, previous_response_id=payload.previous_response_id, ) except ProxyResponseError: @@ -1767,6 +1305,12 @@ async def release_unowned_bridge_lifecycle( exc_info=True, ) durable_lookup = None + if affinity.abandon_unavailable_legacy_owner: + # A verified goal restart deliberately resends all portable state. + # The old bridge row is therefore not additional ownership proof: + # promoting it to preferred_account_id below would bypass the only + # selection path allowed to atomically retire the raw sticky owner. + durable_lookup = None if durable_lookup is not None and durable_lookup.latest_response_id is not None: current_instance = _service_get_settings().http_responses_session_bridge_instance_id current_process_epoch = http_bridge_owner_process_epoch() @@ -1797,180 +1341,10 @@ async def release_unowned_bridge_lifecycle( durable_recovery_attempt_claimed = False durable_recovery_attempt_session_id: str | None = None durable_recovery_attempt_owner_epoch: int | None = None + durable_recovery_fresh_replay = False durable_full_resend_proof = _verify_durable_full_resend(payload, durable_lookup) - ( - durable_abandoned_pending_full_resend_proof, - durable_abandoned_pending_full_resend_rejection_reason, - ) = _verify_durable_abandoned_pending_full_resend_with_reason( - payload, - durable_lookup, - ) - durable_marker_abandoned_pending_candidate = bool( - _durable_recovery_marker_request_path_active(durable_lookup) - and durable_abandoned_pending_full_resend_proof is not None - and durable_abandoned_pending_full_resend_proof.matches(payload, durable_lookup) - ) - durable_marker_verified_recovery_candidate = bool( - _durable_recovery_marker_request_path_active(durable_lookup) - and ( - (durable_full_resend_proof is not None and durable_full_resend_proof.matches(payload, durable_lookup)) - or durable_marker_abandoned_pending_candidate - ) - ) - if durable_full_resend_proof is None: - _log_abandoned_pending_full_resend_rejection( - bridge_session_key=bridge_session_key, - payload=payload, - durable_lookup=durable_lookup, - reason_code=durable_abandoned_pending_full_resend_rejection_reason, - stage="initial_lookup", - ) durable_full_resend_fresh_bridge_proof: _VerifiedDurableFullResend | None = None - durable_marker_abandoned_pending_replay = False - durable_marker_verified_recovery = False force_local_recovery_creation = False - normalized_ingress_headers = {key.lower(): value for key, value in headers.items()} - official_session_id_value = normalized_ingress_headers.get("session-id") - official_session_id = ( - official_session_id_value.strip() - if isinstance(official_session_id_value, str) and official_session_id_value.strip() - else None - ) - conflicting_session_alias = any( - isinstance(normalized_ingress_headers.get(alias), str) - and bool(normalized_ingress_headers[alias].strip()) - and normalized_ingress_headers[alias].strip() != official_session_id - for alias in ("session_id", "x-codex-session-id", "x-codex-conversation-id") - ) - rowless_thread_identity = normalized_ingress_headers.get("thread-id") - rowless_client_request_identity = normalized_ingress_headers.get("x-client-request-id") - rowless_thread_identity = ( - rowless_thread_identity.strip() - if isinstance(rowless_thread_identity, str) and rowless_thread_identity.strip() - else None - ) - rowless_client_request_identity = ( - rowless_client_request_identity.strip() - if isinstance(rowless_client_request_identity, str) and rowless_client_request_identity.strip() - else None - ) - rowless_task_identity = rowless_thread_identity - marker_rowless_task_identity = rowless_thread_identity or rowless_client_request_identity - raw_client_metadata = bridge_payload.get("client_metadata") - rowless_fallback_metadata_valid = True - rowless_metadata_thread_present = False - rowless_official_workspace_metadata = False - rowless_explicit_child_signal = any( - header_name in normalized_ingress_headers - or (isinstance(raw_client_metadata, Mapping) and header_name in raw_client_metadata) - for header_name in ("x-codex-parent-thread-id", "x-openai-subagent") - ) - rowless_turn_metadata_carrier_present = "x-codex-turn-metadata" in normalized_ingress_headers or ( - isinstance(raw_client_metadata, Mapping) and "x-codex-turn-metadata" in raw_client_metadata - ) - if official_session_id is not None and marker_rowless_task_identity is not None: - ( - rowless_fallback_metadata_valid, - rowless_explicit_child_signal, - rowless_metadata_thread_present, - rowless_official_workspace_metadata, - ) = _rowless_client_metadata_evidence( - raw_client_metadata=raw_client_metadata, - normalized_headers=normalized_ingress_headers, - session_id=official_session_id, - task_identity=marker_rowless_task_identity, - ) - rowless_task_identity_sources_consistent = ( - not ( - rowless_thread_identity is not None - and rowless_client_request_identity is not None - and rowless_thread_identity != rowless_client_request_identity - ) - and rowless_fallback_metadata_valid - ) - rowless_lookup_identity_eligible = ( - _ROWLESS_SEMANTIC_REBASE_REQUEST_PATH_ENABLED - and rowless_task_identity is not None - and official_session_id is not None - and explicit_prompt_cache_key is not None - and official_session_id == explicit_prompt_cache_key == rowless_task_identity - ) - if ( - payload.previous_response_id is not None - and durable_lookup is None - and rowless_lookup_identity_eligible - and bridge_session_key.strength == "hard" - and bridge_session_key.affinity_kind == "session_header" - and rowless_turn_metadata_carrier_present - and not rowless_fallback_metadata_valid - ): - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_identity_metadata_invalid", - "The complete-context recovery metadata is invalid or conflicting.", - error_type="invalid_request_error", - ), - ) - marker_rowless_lookup_identity_eligible = ( - _ROWLESS_SEMANTIC_REBASE_REQUEST_PATH_ENABLED - and marker_rowless_task_identity is not None - and rowless_task_identity_sources_consistent - and not rowless_explicit_child_signal - and official_session_id is not None - and explicit_prompt_cache_key is not None - and official_session_id == explicit_prompt_cache_key == marker_rowless_task_identity - ) - rowless_dispatch_identity_eligible = ( - rowless_lookup_identity_eligible - and rowless_task_identity_sources_consistent - and not rowless_explicit_child_signal - and bridge_session_key.strength == "hard" - and bridge_session_key.affinity_kind == "session_header" - and incoming_turn_state_header is None - and not conflicting_session_alias - and (rowless_client_request_identity is None or rowless_client_request_identity == rowless_task_identity) - ) - rowless_automatic_live_recovery_eligible = bool( - rowless_dispatch_identity_eligible and rowless_official_workspace_metadata - ) - rowless_capture_facts = ( - build_rowless_recovery_capture_facts( - untrimmed_effective_payload, - expected_session_identity=official_session_id, - expected_task_identity=rowless_task_identity, - ) - if rowless_lookup_identity_eligible - else None - ) - task_authority_digest = ( - rowless_task_authority_digest( - session_id=official_session_id, - prompt_cache_key=explicit_prompt_cache_key, - thread_id=rowless_task_identity, - ) - if rowless_lookup_identity_eligible - and official_session_id is not None - and explicit_prompt_cache_key is not None - and rowless_task_identity is not None - else None - ) - rowless_automatic_live_recovery_eligible = bool( - rowless_automatic_live_recovery_eligible - and rowless_capture_facts is not None - and rowless_capture_facts.retains_prior_output - ) - rowless_api_key_scope = ( - durable_bridge_api_key_scope(bridge_session_key.api_key_id) if task_authority_digest is not None else None - ) - rowless_strong_hash = ( - rowless_strong_session_hash("task_authority", task_authority_digest) - if task_authority_digest is not None - else None - ) - rowless_authority = None - marker_rowless_dispatch_identity_eligible = False - automatic_marker_supersedes_rowless_authority = False payload_looks_like_full_resend = _http_bridge_payload_looks_like_full_resend(payload) # Set when the quarantine check below suppresses the durable-anchor # injection for a full-resend payload; the session hydration and the @@ -2002,148 +1376,24 @@ def classify_durable_full_resend( # response-owned messages. Cross-account replay uses the # default ID-stripping projection below. preserve_developer_message_ids=True, - preserve_response_owned_agent_message_ids=True, ) safe_fresh_context = False if replay_projection is not None: - pending_tool_calls = lookup.latest_pending_tool_calls - transition_manifest = lookup.latest_response_transition_manifest - if ( - transition_manifest is not None - and pending_tool_calls is not None - and lookup.latest_response_id is not None - ): - safe_fresh_context = responses_input_suffix_matches_transition_manifest( - cast(list[JsonValue], payload.input), - stored_count=stored_count, - response_id=lookup.latest_response_id, - pending_tool_calls=pending_tool_calls, - transition_manifest=transition_manifest, - ) - else: - safe_fresh_context = ( - False - if pending_tool_calls is None - else responses_input_suffix_matches_pending_tool_calls( - replay_projection.input_items, - stored_count=replay_projection.stored_prefix_count, - pending_tool_calls=pending_tool_calls, - canonical_lite_developer_index=replay_projection.canonical_lite_developer_index, - ) - if pending_tool_calls - else responses_input_suffix_retains_prior_output( - replay_projection.input_items, - stored_count=replay_projection.stored_prefix_count, - canonical_lite_developer_index=replay_projection.canonical_lite_developer_index, - exact_stored_prefix_without_pending_manifest=True, - allow_response_owned_agent_message=pending_tool_calls == {}, - ) + safe_fresh_context = responses_input_suffix_retains_prior_output( + replay_projection.input_items, + stored_count=replay_projection.stored_prefix_count, + canonical_lite_developer_index=replay_projection.canonical_lite_developer_index, + ) or ( + lookup.latest_pending_tool_calls is not None + and responses_input_suffix_matches_pending_tool_calls( + replay_projection.input_items, + stored_count=replay_projection.stored_prefix_count, + pending_tool_calls=lookup.latest_pending_tool_calls, + canonical_lite_developer_index=replay_projection.canonical_lite_developer_index, ) - if not safe_fresh_context: - _log_http_bridge_event( - "full_resend_proof_rejected", - bridge_session_key, - account_id=None, - model=payload.model, - detail=( - "reason=invalid_or_incomplete_suffix, " - f"stored_items={stored_count}, " - f"suffix_items={len(payload.input) - stored_count}, " - "suffix_shape=" - f"{_full_resend_suffix_shape_for_observability(payload.input, stored_count=stored_count)}" - ), - cache_key_family=bridge_session_key.affinity_kind, - model_class=_extract_model_class(payload.model) if payload.model else None, - owner_check_applied=True, ) return stored_count, lookup.latest_input_full_fingerprint, safe_fresh_context - async def prepare_durable_recovery_attempt_journal(request_fingerprint: str) -> None: - nonlocal durable_recovery_attempt_available - nonlocal durable_recovery_attempt_claimed - nonlocal durable_recovery_attempt_owner_epoch - nonlocal durable_recovery_attempt_session_id - - if durable_lookup is None: - return - try: - existing_attempt = await self._durable_bridge.lookup_recovery_attempt( - session_id=durable_lookup.session_id, - request_fingerprint=request_fingerprint, - ) - if existing_attempt is not None and ( - durable_lookup.state != HttpBridgeSessionState.ACTIVE - or not durable_lookup.lease_is_active(now=utcnow()) - ): - claim_instance_id = _service_get_settings().http_responses_session_bridge_instance_id - claim_owner_epoch = durable_lookup.owner_epoch - owner_is_current = ( - durable_lookup.owner_instance_id == claim_instance_id - and durable_lookup.lease_is_active(now=utcnow()) - ) - if not owner_is_current: - claimed_session = await self._durable_bridge.claim_live_session( - session_key_kind=durable_lookup.canonical_kind, - session_key_value=durable_lookup.canonical_key, - api_key_id=bridge_session_key.api_key_id, - instance_id=claim_instance_id, - lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(), - account_id=durable_lookup.account_id, - model=payload.model, - service_tier=None, - latest_turn_state=durable_lookup.latest_turn_state, - latest_response_id=None, - owner_process_epoch=http_bridge_owner_process_epoch(), - # Revalidate the stale lookup under the row lock; - # an active owner that appeared after lookup must - # not be displaced. - allow_takeover=False, - ) - if claimed_session.owner_instance_id != claim_instance_id: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "HTTP responses recovery ownership changed; retry the request.", - ), - ) - claim_owner_epoch = claimed_session.owner_epoch - claimed = await self._durable_bridge.mark_recovery_attempt_replayed( - session_id=durable_lookup.session_id, - api_key_id=bridge_session_key.api_key_id, - instance_id=claim_instance_id, - owner_epoch=claim_owner_epoch, - request_fingerprint=request_fingerprint, - ) - if not claimed: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "HTTP responses recovery ownership changed; retry the request.", - ), - ) - durable_recovery_attempt_claimed = True - durable_recovery_attempt_available = False - durable_recovery_attempt_session_id = durable_lookup.session_id - durable_recovery_attempt_owner_epoch = claim_owner_epoch - elif existing_attempt is None: - # The request-submit path journals this exact fingerprint - # immediately before dispatch. An ambiguous outcome may - # then consume only one same-owner replay generation. - durable_recovery_attempt_available = True - except ProxyResponseError: - raise - except Exception: - logger.warning("Failed to claim HTTP bridge recovery attempt", exc_info=True) - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "HTTP responses recovery state could not be claimed; retry the request.", - ), - ) - if durable_lookup is not None: ( durable_full_resend_anchor_count, @@ -2160,15 +1410,10 @@ async def prepare_durable_recovery_attempt_journal(request_fingerprint: str) -> stored_count=durable_full_resend_anchor_count, ) if replay_projection is not None: - durable_full_resend_retains_prior_output = ( - durable_lookup is not None - and responses_input_suffix_retains_prior_output( - replay_projection.input_items, - stored_count=replay_projection.stored_prefix_count, - canonical_lite_developer_index=replay_projection.canonical_lite_developer_index, - exact_stored_prefix_without_pending_manifest=not durable_lookup.latest_pending_tool_calls, - allow_response_owned_agent_message=durable_lookup.latest_pending_tool_calls == {}, - ) + durable_full_resend_retains_prior_output = responses_input_suffix_retains_prior_output( + replay_projection.input_items, + stored_count=replay_projection.stored_prefix_count, + canonical_lite_developer_index=replay_projection.canonical_lite_developer_index, ) durable_full_resend_fresh_payload = _http_bridge_payload_without_previous_response_id( payload @@ -2181,12 +1426,85 @@ async def prepare_durable_recovery_attempt_journal(request_fingerprint: str) -> ) del _fresh_state durable_recovery_attempt_fingerprint = durable_bridge_hash(fresh_replay_text) - if ( - durable_lookup is not None - and durable_full_resend_is_account_neutral - and not durable_marker_verified_recovery_candidate - ): - await prepare_durable_recovery_attempt_journal(durable_recovery_attempt_fingerprint) + if durable_lookup is not None and durable_full_resend_is_account_neutral: + try: + existing_attempt = await self._durable_bridge.lookup_recovery_attempt( + session_id=durable_lookup.session_id, + request_fingerprint=durable_recovery_attempt_fingerprint, + ) + if existing_attempt is not None and ( + durable_lookup.state != HttpBridgeSessionState.ACTIVE + or not durable_lookup.lease_is_active(now=utcnow()) + ): + claim_instance_id = _service_get_settings().http_responses_session_bridge_instance_id + claim_owner_epoch = durable_lookup.owner_epoch + owner_is_current = ( + durable_lookup.owner_instance_id == claim_instance_id + and durable_lookup.lease_is_active(now=utcnow()) + ) + if not owner_is_current: + claimed_session = await self._durable_bridge.claim_live_session( + session_key_kind=durable_lookup.canonical_kind, + session_key_value=durable_lookup.canonical_key, + api_key_id=bridge_session_key.api_key_id, + instance_id=claim_instance_id, + lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(), + account_id=durable_lookup.account_id, + model=payload.model, + service_tier=None, + latest_turn_state=durable_lookup.latest_turn_state, + latest_response_id=None, + owner_process_epoch=http_bridge_owner_process_epoch(), + # Revalidate the stale lookup under the + # row lock; an active owner that appeared + # after the lookup must not be displaced. + allow_takeover=False, + ) + if claimed_session.owner_instance_id != claim_instance_id: + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP responses recovery ownership changed; retry the request.", + ), + ) + claim_owner_epoch = claimed_session.owner_epoch + claimed = await self._durable_bridge.mark_recovery_attempt_replayed( + session_id=durable_lookup.session_id, + api_key_id=bridge_session_key.api_key_id, + instance_id=claim_instance_id, + owner_epoch=claim_owner_epoch, + request_fingerprint=durable_recovery_attempt_fingerprint, + ) + if not claimed: + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP responses recovery ownership changed; retry the request.", + ), + ) + durable_recovery_attempt_claimed = True + durable_recovery_attempt_available = False + durable_recovery_attempt_session_id = durable_lookup.session_id + durable_recovery_attempt_owner_epoch = claim_owner_epoch + elif existing_attempt is None: + # No prior attempt owns this fingerprint. The + # request-submit path will journal it immediately + # before dispatch, and an ambiguous transport + # outcome may then consume the one replay fence. + durable_recovery_attempt_available = True + except ProxyResponseError: + raise + except Exception: + logger.warning("Failed to claim HTTP bridge recovery attempt", exc_info=True) + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP responses recovery state could not be claimed; retry the request.", + ), + ) durable_anchor_trimmable = durable_full_resend_anchor_count is not None durable_model_transition_lookup = ( durable_lookup @@ -2253,455 +1571,13 @@ async def prepare_durable_recovery_attempt_journal(request_fingerprint: str) -> and durable_lookup.latest_response_id is not None and (not payload_looks_like_full_resend or durable_anchor_trimmable) ) - verified_quarantine_full_resend = bool( - durable_full_resend_proof is not None and durable_full_resend_proof.matches(payload, durable_lookup) - ) - durable_recovery_marker_active = _durable_recovery_marker_request_path_active(durable_lookup) - verified_marker_full_resend = bool( - verified_quarantine_full_resend - or ( - durable_abandoned_pending_full_resend_proof is not None - and durable_abandoned_pending_full_resend_proof.matches(payload, durable_lookup) - ) - ) - if durable_recovery_marker_active and not verified_marker_full_resend: - # Codex sends a fresh turn-state affinity value on ordinary root-task - # turns. The durable lookup above has already resolved that alias - # back to this exact hard session-header marker, so marker-backed - # recovery can accept it without weakening rowless/no-row identity. - marker_rowless_dispatch_identity_eligible = ( - marker_rowless_lookup_identity_eligible - and bridge_session_key.strength == "hard" - and bridge_session_key.affinity_kind == "session_header" - and not conflicting_session_alias - and ( - rowless_client_request_identity is None - or rowless_client_request_identity == marker_rowless_task_identity - ) - ) - if ( - marker_rowless_dispatch_identity_eligible - and not rowless_lookup_identity_eligible - and official_session_id is not None - and explicit_prompt_cache_key is not None - and marker_rowless_task_identity is not None - ): - rowless_task_identity = marker_rowless_task_identity - rowless_capture_facts = build_rowless_recovery_capture_facts( - untrimmed_effective_payload, - expected_session_identity=official_session_id, - expected_task_identity=marker_rowless_task_identity, - ) - task_authority_digest = rowless_task_authority_digest( - session_id=official_session_id, - prompt_cache_key=explicit_prompt_cache_key, - thread_id=marker_rowless_task_identity, - ) - rowless_api_key_scope = durable_bridge_api_key_scope(bridge_session_key.api_key_id) - rowless_strong_hash = rowless_strong_session_hash("task_authority", task_authority_digest) - rowless_automatic_live_recovery_eligible = bool( - rowless_official_workspace_metadata - and (rowless_dispatch_identity_eligible or marker_rowless_dispatch_identity_eligible) - and rowless_capture_facts is not None - and rowless_capture_facts.retains_prior_output - ) - if ( - task_authority_digest is not None - and rowless_api_key_scope is not None - and rowless_strong_hash is not None - and durable_lookup.latest_response_id is not None - and durable_lookup.account_id is not None - ): - stale_anchor_hash = durable_bridge_hash(durable_lookup.latest_response_id) - try: - async with SessionLocal() as rowless_session: - rowless_repository = RowlessRecoveryRepository(rowless_session) - rowless_authority = await rowless_repository.lookup( - api_key_scope=rowless_api_key_scope, - strong_session_hash=rowless_strong_hash, - stale_anchor_hash=stale_anchor_hash, - ) - if rowless_authority is None and rowless_capture_facts is not None: - rowless_authority = await rowless_repository.lookup_exact_request_contract( - api_key_scope=rowless_api_key_scope, - strong_session_hash=rowless_strong_hash, - facts=rowless_capture_facts, - ) - if ( - rowless_authority is None - and marker_rowless_dispatch_identity_eligible - and rowless_capture_facts is not None - and rowless_capture_facts.unresolved_count == 0 - and rowless_capture_facts.self_contained - and rowless_capture_facts.account_neutral - and rowless_task_identity is not None - and official_session_id is not None - ): - installation_id = await rowless_session.scalar( - select(Account.codex_installation_id).where(Account.id == durable_lookup.account_id) - ) - if installation_id: - marker_projected_payload = untrimmed_effective_payload.model_copy( - update={ - "input": rowless_capture_facts.projected_input, - "previous_response_id": None, - } - ) - _marker_state, marker_text = prepare_bridge_request(marker_projected_payload) - del _marker_state - marker_text = _text_with_account_installation_id(marker_text, installation_id) - rowless_authority = await rowless_repository.capture( - api_key_scope=rowless_api_key_scope, - session_key_kind=bridge_session_key.affinity_kind, - strong_session_hash=rowless_strong_hash, - stale_anchor_hash=stale_anchor_hash, - selected_account_intent=durable_lookup.account_id, - task_identity=rowless_task_identity, - session_identity=official_session_id, - task_authority_digest=task_authority_digest, - facts=dataclasses.replace( - rowless_capture_facts, - actual_wire_fingerprint=rowless_actual_wire_fingerprint(marker_text), - ), - origin_marker_session_id=durable_lookup.session_id, - ) - except RowlessRecoveryStateError as exc: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The durable-marker semantic-rebase authority could not be fenced safely.", - ), - ) from exc - except Exception as exc: - logger.warning( - "Failed to persist durable-marker rowless recovery authority", - exc_info=True, - ) - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The durable-marker semantic-rebase authority could not be persisted.", - ), - ) from exc - if rowless_authority is not None: - # Legacy authorities can predate marker-origin binding. - # Their terminal state is already sufficient to reject any - # replay, so report that stronger fence before provenance. - if rowless_authority.state == HttpBridgeRowlessRecoveryState.UNKNOWN: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_dispatch_outcome_unknown", - "The semantic-rebase dispatch outcome is unknown and cannot be replayed.", - error_type="invalid_request_error", - ), - ) - if rowless_authority.state == HttpBridgeRowlessRecoveryState.CONSUMED: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_already_consumed", - "This stale semantic-rebase turn was already consumed; " - "continue from its new checkpoint.", - error_type="invalid_request_error", - ), - ) - if rowless_authority.origin_marker_session_id != durable_lookup.session_id: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_marker_origin_mismatch", - "The semantic-rebase authority belongs to a different durable marker.", - error_type="invalid_request_error", - ), - ) - if rowless_authority.selected_account_intent != durable_lookup.account_id: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_pinned_account_changed", - "The durable marker account changed; the semantic rebase remains unsent.", - error_type="invalid_request_error", - ), - ) - else: - diagnostic_capture_facts = rowless_capture_facts - if ( - diagnostic_capture_facts is None - and isinstance(untrimmed_effective_payload.input, list) - and len(untrimmed_effective_payload.input) <= 512 - ): - diagnostic_capture_facts = build_rowless_recovery_capture_facts( - untrimmed_effective_payload, - expected_session_identity=official_session_id, - expected_task_identity=marker_rowless_task_identity, - ) - capture_facts_present = diagnostic_capture_facts is not None - _log_http_bridge_event( - "recovery_required_request_rejected", - bridge_session_key, - account_id=durable_lookup.account_id, - model=payload.model, - detail=( - ( - "reason=pending_call_resolution_required" - if durable_lookup.latest_pending_tool_calls - else "reason=complete_context_rehydration_required" - ) - + ",identity_session_present=" - + str(official_session_id is not None).lower() - + ",identity_prompt_cache_present=" - + str(explicit_prompt_cache_key is not None).lower() - + ",identity_thread_present=" - + str(rowless_thread_identity is not None).lower() - + ",identity_client_request_present=" - + str(rowless_client_request_identity is not None).lower() - + ",identity_sources_consistent=" - + str(rowless_task_identity_sources_consistent).lower() - + ",identity_explicit_child_signal=" - + str(rowless_explicit_child_signal).lower() - + ",identity_metadata_thread_present=" - + str(rowless_metadata_thread_present).lower() - + ",identity_fallback_metadata_valid=" - + str(rowless_fallback_metadata_valid).lower() - + ",identity_turn_state_present=" - + str(incoming_turn_state_header is not None).lower() - + ",identity_conflicting_session_alias=" - + str(conflicting_session_alias).lower() - + ",identity_lookup_eligible=" - + str(rowless_lookup_identity_eligible).lower() - + ",identity_marker_dispatch_eligible=" - + str(marker_rowless_dispatch_identity_eligible).lower() - + ",capture_facts_present=" - + str(capture_facts_present).lower() - + ",capture_diagnostic_bounded=" - + str( - isinstance(untrimmed_effective_payload.input, list) - and len(untrimmed_effective_payload.input) <= 512 - ).lower() - + ",capture_unresolved_zero=" - + str( - diagnostic_capture_facts is not None and diagnostic_capture_facts.unresolved_count == 0 - ).lower() - + ",capture_self_contained=" - + str( - diagnostic_capture_facts is not None and diagnostic_capture_facts.self_contained - ).lower() - + ",capture_account_neutral=" - + str( - diagnostic_capture_facts is not None and diagnostic_capture_facts.account_neutral - ).lower() - ), - cache_key_family=bridge_session_key.affinity_kind, - model_class=_extract_model_class(payload.model) if payload.model else None, - owner_check_applied=True, - ) - raise ProxyResponseError( - 400, - openai_error( - ( - "previous_response_pending_call_resolution_required" - if durable_lookup.latest_pending_tool_calls - else "previous_response_complete_context_required" - ), - ( - "The saved response anchor has an unresolved pending call. " - "Retry once with the complete conversation context after resolving that call." - if durable_lookup.latest_pending_tool_calls - else ( - "The saved response anchor requires complete conversation context. " - "Retry once with a verified complete resend." - ) - ), - ), - retryable_same_contract=False, - failure_detail="durable_recovery_required", - upstream_error_code="previous_response_not_found", - ) - if ( - durable_recovery_marker_active - and verified_marker_full_resend - and not (rowless_automatic_live_recovery_eligible and rowless_authority is not None) - ): - # A prior request already supplied the physical stale-anchor - # rejection. The durable marker is therefore equivalent to - # the process-local quarantine for this exact owner/anchor, - # but it remains proof-neutral: only the already sealed full - # resend objects above may suppress the anchor. - fresh_reattach_can_use_durable_anchor = False - fresh_reattach_anchor_suppressed_quarantined = True - effective_payload = _http_bridge_payload_without_previous_response_id(payload) - durable_full_resend_fresh_bridge_proof = ( - durable_full_resend_proof - if verified_quarantine_full_resend - else durable_abandoned_pending_full_resend_proof - ) - if not verified_quarantine_full_resend: - abandoned_pending_proof = durable_abandoned_pending_full_resend_proof - if abandoned_pending_proof is None: - raise RuntimeError("verified marker replay lost its sealed abandoned-pending proof") - if not isinstance(payload.input, list): - raise RuntimeError("sealed abandoned-pending proof requires list input") - abandoned_projection = project_responses_input_for_abandoned_pending_fresh_replay( - cast(list[JsonValue], payload.input), - stored_count=abandoned_pending_proof.stored_input_item_count, - pending_tool_calls=dict(durable_lookup.latest_pending_tool_calls or {}), - ) - if abandoned_projection is None: - raise RuntimeError("sealed abandoned-pending proof lost its replay projection") - effective_payload = _http_bridge_payload_without_previous_response_id(payload).model_copy( - update={"input": abandoned_projection.input_items} - ) - durable_marker_abandoned_pending_replay = True - if _ROWLESS_SEMANTIC_REBASE_REQUEST_PATH_ENABLED and durable_lookup.latest_response_id is not None: - try: - async with SessionLocal() as rowless_session: - rowless_repository = RowlessRecoveryRepository(rowless_session) - rowless_authority = await rowless_repository.lookup_stale_anchor_in_scope( - api_key_scope=durable_lookup.api_key_scope, - stale_anchor_hash=durable_bridge_hash(durable_lookup.latest_response_id), - ) - if ( - rowless_authority is None - and rowless_api_key_scope is not None - and rowless_strong_hash is not None - and rowless_capture_facts is not None - ): - rowless_authority = await rowless_repository.lookup_exact_request_contract( - api_key_scope=rowless_api_key_scope, - strong_session_hash=rowless_strong_hash, - facts=rowless_capture_facts, - ) - except RowlessRecoveryStateError as exc: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_authority_ambiguous", - "Multiple semantic-rebase authorities match this exact turn; dispatch is unsafe.", - error_type="invalid_request_error", - ), - ) from exc - if rowless_authority is not None: - if ( - task_authority_digest is not None - and rowless_authority.captured_task_authority_digest != task_authority_digest - ): - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_task_identity_mismatch", - "The approved semantic rebase belongs to a different Codex task.", - error_type="invalid_request_error", - ), - ) - if ( - rowless_authority.origin_marker_session_id == durable_lookup.session_id - and rowless_authority.selected_account_intent != durable_lookup.account_id - ): - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_pinned_account_changed", - "The durable marker account changed; the semantic rebase remains unsent.", - error_type="invalid_request_error", - ), - ) - automatic_marker_supersedes_rowless_authority = bool( - rowless_authority.origin_marker_session_id == durable_lookup.session_id - and rowless_authority.selected_account_intent == durable_lookup.account_id - and rowless_authority.state - in { - HttpBridgeRowlessRecoveryState.CAPTURED, - HttpBridgeRowlessRecoveryState.APPROVED, - } - ) - if not automatic_marker_supersedes_rowless_authority: - if rowless_authority.state == HttpBridgeRowlessRecoveryState.CAPTURED: - envelope = openai_error( - "previous_response_recovery_authorization_required", - "A dashboard administrator must approve this same-turn semantic rebase.", - error_type="invalid_request_error", - ) - envelope["error"]["action"] = "retry_same_turn_after_admin_approval" - raise ProxyResponseError(400, envelope) - if rowless_authority.state == HttpBridgeRowlessRecoveryState.UNKNOWN: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_dispatch_outcome_unknown", - "The semantic-rebase dispatch outcome is unknown and cannot be replayed.", - error_type="invalid_request_error", - ), - ) - if rowless_authority.state == HttpBridgeRowlessRecoveryState.CONSUMED: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_already_consumed", - "This stale semantic-rebase turn was already consumed; " - "continue from its new checkpoint.", - error_type="invalid_request_error", - ), - ) - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_marker_origin_mismatch", - "The approved semantic rebase belongs to a different durable marker.", - error_type="invalid_request_error", - ), - ) - durable_marker_verified_recovery = True - _marker_request_state, marker_request_text = prepare_bridge_request(effective_payload) - del _marker_request_state - durable_recovery_attempt_fingerprint = durable_bridge_hash(marker_request_text) - if durable_lookup.latest_response_id is None or durable_lookup.account_id is None: - raise RuntimeError("sealed marker recovery lost its owner-bound rejected anchor") - existing_marker_attempt = await self._durable_bridge.lookup_recovery_attempt( - session_id=durable_lookup.session_id, - request_fingerprint=durable_recovery_attempt_fingerprint, - ) - if existing_marker_attempt is not None: - # A marker-authorized recovery may contain irreversible - # tool results. UNKNOWN can mean the prior process died - # after the upstream accepted the request, so it is never - # eligible for the generic lease-expiry replay claim. - # Reject before opening another physical upstream bridge; - # only a terminal replacement checkpoint may clear the - # marker and settle this generation. - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The recovery checkpoint has an ambiguous prior delivery; retrying is unsafe.", - ), - ) - _log_http_bridge_event( - "recovery_required_full_resend_admitted", - bridge_session_key, - account_id=durable_lookup.account_id, - model=payload.model, - detail=( - "proof_source=abandoned_pending_agent_boundary" - if durable_marker_abandoned_pending_replay - else "proof_source=durable_full_resend" - ), - cache_key_family=bridge_session_key.affinity_kind, - model_class=_extract_model_class(payload.model) if payload.model else None, - owner_check_applied=True, - ) - if verified_quarantine_full_resend and _http_bridge_session_key_quarantined( - self, - bridge_session_key, - ): + if payload_looks_like_full_resend and _http_bridge_session_key_quarantined(self, bridge_session_key): # The previous attach on this key proved silent/wedged - # (#1534). Only a durable-owner-bound proof that the client's - # payload carries the complete conversation may suppress the - # anchor. A merely multi-item/full-resend-shaped payload can - # still omit completed assistant or tool output and must keep - # the anchor so it fails closed instead of losing history. + # (#1534). The client's own payload already carries the full + # conversation, so send it unanchored on the fresh path + # instead of rebuilding the same reattach. Delta-only + # payloads keep the anchor: it is their only way to convey + # prior context (same boundary as the fenced anchor clear). # Evaluated independently of the fresh-reattach eligibility # above: even when that gate is already false (for example a # conversation-scoped payload, a live alias session, or an @@ -2779,6 +1655,7 @@ async def prepare_durable_recovery_attempt_journal(request_fingerprint: str) -> affinity = _AffinityPolicy() incoming_turn_state_header = None session_header_fallback_key = None + durable_session_header_alias = None owner_bound_full_resend_ignores_broad_session = ( not forwarded_request and durable_full_resend_fresh_bridge_proof is not None @@ -2793,6 +1670,7 @@ async def prepare_durable_recovery_attempt_journal(request_fingerprint: str) -> affinity = _AffinityPolicy(kind=StickySessionKind.CODEX_SESSION) incoming_session_header = None session_header_fallback_key = None + durable_session_header_alias = None _log_http_bridge_event( "fresh_reattach_broad_session_owner_ignored", bridge_session_key, @@ -2811,343 +1689,8 @@ async def prepare_durable_recovery_attempt_journal(request_fingerprint: str) -> previous_response_trimmed_input_fingerprint = _fingerprint_input_items(previous_response_input_items) effective_payload = effective_payload.model_copy(update={"input": trimmed_input_items}) request_state, text_data = prepare_bridge_request(effective_payload) - # A hard-session reattach can inject the marker's stale anchor after - # rowless capture facts are built. Keep the original client payload for - # contract proof, but let exactly-once preflight bind that verified - # owner-backed anchor. - rowless_recovery_stale_anchor_id = untrimmed_effective_payload.previous_response_id - if rowless_recovery_stale_anchor_id is None and proxy_injected_previous_response_id: - rowless_recovery_stale_anchor_id = effective_payload.previous_response_id - if ( - not durable_marker_verified_recovery - and task_authority_digest is not None - and rowless_api_key_scope is not None - and rowless_strong_hash is not None - and rowless_dispatch_identity_eligible - and rowless_capture_facts is not None - and rowless_capture_facts.unresolved_count == 0 - and rowless_capture_facts.self_contained - and rowless_capture_facts.account_neutral - and rowless_task_identity is not None - and official_session_id is not None - and rowless_recovery_stale_anchor_id is not None - ): - request_state.rowless_recovery_capture_intent = RowlessRecoveryCaptureIntent( - api_key_scope=rowless_api_key_scope, - session_key_kind=bridge_session_key.affinity_kind, - strong_session_hash=rowless_strong_hash, - task_authority_digest=task_authority_digest, - task_identity=rowless_task_identity, - session_identity=official_session_id, - facts=rowless_capture_facts, - automatic_live_recovery=rowless_automatic_live_recovery_eligible, - ) - if task_authority_digest is not None and rowless_api_key_scope is not None and rowless_strong_hash is not None: - if rowless_authority is None: - async with SessionLocal() as rowless_session: - rowless_repository = RowlessRecoveryRepository(rowless_session) - if rowless_recovery_stale_anchor_id is not None: - rowless_authority = await rowless_repository.lookup( - api_key_scope=rowless_api_key_scope, - strong_session_hash=rowless_strong_hash, - stale_anchor_hash=durable_bridge_hash(rowless_recovery_stale_anchor_id), - ) - if rowless_authority is None and rowless_capture_facts is not None: - try: - rowless_authority = await rowless_repository.lookup_exact_request_contract( - api_key_scope=rowless_api_key_scope, - strong_session_hash=rowless_strong_hash, - facts=rowless_capture_facts, - ) - except RowlessRecoveryStateError as exc: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_authority_ambiguous", - "Multiple semantic-rebase authorities match this exact turn; dispatch is unsafe.", - error_type="invalid_request_error", - ), - ) from exc - if rowless_authority is not None and not automatic_marker_supersedes_rowless_authority: - if rowless_authority.captured_task_authority_digest != task_authority_digest: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_task_identity_mismatch", - "The approved semantic rebase belongs to a different Codex task.", - error_type="invalid_request_error", - ), - ) - if rowless_authority.state == HttpBridgeRowlessRecoveryState.UNKNOWN: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_dispatch_outcome_unknown", - "The semantic-rebase dispatch outcome is unknown and cannot be replayed.", - error_type="invalid_request_error", - ), - ) - if rowless_authority.state == HttpBridgeRowlessRecoveryState.CONSUMED: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_already_consumed", - "This stale semantic-rebase turn was already consumed; continue from its new checkpoint.", - error_type="invalid_request_error", - ), - ) - automatic_preflight_claimed = False - automatic_request_state = None - automatic_text = None - if ( - rowless_automatic_live_recovery_eligible - and rowless_authority.state - in { - HttpBridgeRowlessRecoveryState.CAPTURED, - HttpBridgeRowlessRecoveryState.APPROVED, - } - and rowless_capture_facts is not None - and rowless_capture_facts.unresolved_count == 0 - and rowless_capture_facts.self_contained - and rowless_capture_facts.account_neutral - and rowless_task_identity is not None - and official_session_id is not None - and rowless_recovery_stale_anchor_id is not None - ): - automatic_payload = untrimmed_effective_payload.model_copy( - update={ - "input": rowless_capture_facts.projected_input, - "previous_response_id": None, - } - ) - automatic_request_state, automatic_text = prepare_bridge_request(automatic_payload) - async with SessionLocal() as rowless_session: - installation_id = await rowless_session.scalar( - select(Account.codex_installation_id).where( - Account.id == rowless_authority.selected_account_intent - ) - ) - if not installation_id: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_pinned_account_changed", - "The pinned account metadata is unavailable; the semantic rebase remains unsent.", - error_type="invalid_request_error", - ), - ) - automatic_text = _text_with_account_installation_id(automatic_text, installation_id) - automatic_facts = dataclasses.replace( - rowless_capture_facts, - actual_wire_fingerprint=rowless_actual_wire_fingerprint(automatic_text), - ) - claimed_automatic_authority = None - try: - async with SessionLocal() as rowless_session: - claimed_automatic_authority = await RowlessRecoveryRepository( - rowless_session - ).capture_and_claim_automatic_preflight( - api_key_scope=rowless_api_key_scope, - session_key_kind=bridge_session_key.affinity_kind, - strong_session_hash=rowless_strong_hash, - stale_anchor_hash=durable_bridge_hash(rowless_recovery_stale_anchor_id), - selected_account_intent=rowless_authority.selected_account_intent, - task_identity=rowless_task_identity, - session_identity=official_session_id, - task_authority_digest=task_authority_digest, - facts=automatic_facts, - request_id=automatic_request_state.request_id, - wire_request_fingerprint=automatic_facts.actual_wire_fingerprint, - origin_marker_session_id=rowless_authority.origin_marker_session_id, - expected_authority_id=rowless_authority.id, - expected_generation=rowless_authority.generation, - ) - # Bind the committed claim before AsyncSession.__aexit__ - # can observe cancellation, so cleanup can prove the - # request remained physically unsent. - automatic_request_state.rowless_recovery_authority_id = claimed_automatic_authority.id - automatic_request_state.rowless_recovery_generation = claimed_automatic_authority.generation - automatic_request_state.rowless_recovery_wire_fingerprint = ( - claimed_automatic_authority.actual_wire_fingerprint - ) - except asyncio.CancelledError: - if claimed_automatic_authority is not None: - rollback_task = asyncio.create_task( - self._rollback_rowless_preflight_setup_failure_if_unbound(automatic_request_state) - ) - await _await_task_deferring_cancellation(rollback_task) - raise - except RowlessRecoveryStateError as exc: - raise ProxyResponseError( - 400, - openai_error( - "rowless_automatic_recovery_proof_rejected", - "The live Codex turn could not prove a physically-unsent semantic rebase.", - error_type="invalid_request_error", - ), - ) from exc - except Exception as exc: - if claimed_automatic_authority is not None: - rollback_task = asyncio.create_task( - self._rollback_rowless_preflight_setup_failure_if_unbound(automatic_request_state) - ) - try: - _, rollback_cancellation = await _await_task_deferring_cancellation(rollback_task) - except Exception: - logger.warning( - "Failed to restore request-start unsent rowless authority", - exc_info=True, - ) - else: - if rollback_cancellation is not None: - raise rollback_cancellation - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The semantic-rebase authority could not be persisted; retrying is unsafe.", - ), - ) from exc - rowless_authority = claimed_automatic_authority - automatic_preflight_claimed = True - if rowless_authority.state == HttpBridgeRowlessRecoveryState.CAPTURED: - envelope = openai_error( - "previous_response_recovery_authorization_required", - "A dashboard administrator must approve this same-turn semantic rebase.", - error_type="invalid_request_error", - ) - envelope["error"]["action"] = "retry_same_turn_after_admin_approval" - raise ProxyResponseError(400, envelope) - if automatic_preflight_claimed: - if rowless_capture_facts is None: # pragma: no cover - eligibility requires it - raise AssertionError("automatic rowless recovery lost its captured projection") - projected_input = rowless_capture_facts.projected_input - else: - projected_input = approved_rowless_recovery_projection( - untrimmed_effective_payload, - captured_input_item_count=rowless_authority.captured_input_item_count, - captured_input_fingerprint=rowless_authority.captured_input_fingerprint, - non_input_contract_fingerprint=rowless_authority.non_input_contract_fingerprint, - direct_call_ledger_digest=rowless_authority.settled_direct_call_ledger_digest, - projected_payload_fingerprint=rowless_authority.projected_payload_fingerprint, - ) - if not ( - rowless_dispatch_identity_eligible - or ( - rowless_authority.origin_marker_session_id is not None - and marker_rowless_dispatch_identity_eligible - ) - ): - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_exact_identity_required", - "Retry with the exact captured root-task routing identity.", - error_type="invalid_request_error", - ), - ) - if projected_input is None: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_exact_same_turn_required", - "Retry the exact captured turn after administrator approval; " - "new suffix items are not allowed.", - error_type="invalid_request_error", - ), - ) - effective_payload = untrimmed_effective_payload.model_copy( - update={"input": projected_input, "previous_response_id": None} - ) - # The semantic rebase has replaced the temporary reattach - # anchor with a claimed, self-contained wire. Do not let the - # later store-context path trim or rebuild that exact wire. - proxy_injected_previous_response_id = False - fresh_upstream_request_text = None - if automatic_preflight_claimed: - if automatic_request_state is None or automatic_text is None: # pragma: no cover - raise AssertionError("automatic rowless recovery lost its in-memory wire") - request_state = automatic_request_state - text_data = automatic_text - else: - request_state, text_data = prepare_bridge_request(effective_payload) - async with SessionLocal() as rowless_session: - installation_id = await rowless_session.scalar( - select(Account.codex_installation_id).where( - Account.id == rowless_authority.selected_account_intent - ) - ) - if not installation_id: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_pinned_account_changed", - "The pinned account metadata is unavailable; the semantic rebase remains unsent.", - error_type="invalid_request_error", - ), - ) - text_data = _text_with_account_installation_id(text_data, installation_id) - if rowless_actual_wire_fingerprint(text_data) != rowless_authority.actual_wire_fingerprint: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_actual_wire_changed", - "The exact upstream wire changed after approval; the semantic rebase remains unsent.", - error_type="invalid_request_error", - ), - ) - request_state.request_text = text_data - request_state.rowless_recovery_authority_id = rowless_authority.id - request_state.rowless_recovery_generation = rowless_authority.generation - request_state.rowless_recovery_task_authority_digest = rowless_authority.captured_task_authority_digest - rowless_wire_fingerprint = rowless_authority.actual_wire_fingerprint - request_state.rowless_recovery_wire_fingerprint = rowless_wire_fingerprint - request_state.preferred_account_id = rowless_authority.selected_account_intent - request_state.input_item_count = rowless_authority.captured_input_item_count - request_state.input_full_fingerprint = rowless_authority.captured_input_fingerprint - request_state.rowless_recovery_capture_intent = None - if not automatic_preflight_claimed: - try: - async with SessionLocal() as rowless_session: - await RowlessRecoveryRepository(rowless_session).claim_dispatch_preflight( - authority_id=rowless_authority.id, - generation=rowless_authority.generation, - request_id=request_state.request_id, - wire_request_fingerprint=rowless_wire_fingerprint, - task_authority_digest=rowless_authority.captured_task_authority_digest, - ) - except RowlessRecoveryStateError as exc: - raise ProxyResponseError( - 400, - openai_error( - "rowless_recovery_dispatch_already_claimed", - "This semantic-rebase generation has already been claimed and cannot be replayed.", - error_type="invalid_request_error", - ), - ) from exc - except Exception as exc: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The semantic-rebase dispatch fence could not be persisted.", - ), - ) from exc - force_local_recovery_creation = True - elif durable_lookup is not None: - # A normal durable checkpoint is not eligible for a new - # rowless authority. The broader candidate exists only so a - # previously captured non-cascading tombstone can be found - # after the failed turn created a transient replacement row. - request_state.rowless_recovery_capture_intent = None request_state.enforce_openai_sdk_contract = enforce_openai_sdk_contract request_state.affinity_policy = affinity - if durable_marker_abandoned_pending_replay and isinstance(payload.input, list): - # Upstream receives the fail-closed projection while the new - # durable anchor remains bound to the exact complete client - # context that sealed the abandoned-pending proof. - request_state.input_item_count = len(payload.input) - request_state.input_full_fingerprint = _fingerprint_input_items(cast(list[JsonValue], payload.input)) _apply_http_bridge_downstream_turn_state( request_state, downstream_turn_state=downstream_turn_state, @@ -3210,11 +1753,32 @@ async def prepare_durable_recovery_attempt_journal(request_fingerprint: str) -> session_id=request_state.session_id, surface="http_bridge", ) - request_state.preferred_account_id = resolve_required_account_id( - ("durable bridge", request_state.preferred_account_id), - ("live bridge", local_previous_response_owner), - ("previous-response index", indexed_previous_response_owner), - ) + try: + request_state.preferred_account_id = resolve_required_account_id( + ("durable bridge", request_state.preferred_account_id), + ("live bridge", local_previous_response_owner), + ("previous-response index", indexed_previous_response_owner), + ) + except ProxyResponseError: + # The request-log owner cache is intentionally only a fast + # path. If it conflicts with a durable anchor, re-read the + # authoritative request-log row once before failing closed; + # this repairs stale in-process pins without adding a DB read + # to the normal continuation path. + if durable_lookup is None or indexed_previous_response_owner is None: + raise + indexed_previous_response_owner = await self._resolve_websocket_previous_response_owner( + previous_response_id=request_state.previous_response_id, + api_key=api_key, + session_id=request_state.session_id, + surface="http_bridge", + force_request_log_lookup=True, + ) + request_state.preferred_account_id = resolve_required_account_id( + ("durable bridge", request_state.preferred_account_id), + ("live bridge", local_previous_response_owner), + ("previous-response index", indexed_previous_response_owner), + ) durable_lookup_requires_owner = durable_lookup is not None and ( request_state.previous_response_id is not None or bridge_session_key.strength == "hard" @@ -3262,73 +1826,6 @@ async def prepare_durable_recovery_attempt_journal(request_fingerprint: str) -> # Only the trim branch below (which verifies the stored prefix # fingerprint) is allowed to flip this flag to ``True``. request_state.fresh_upstream_request_is_retry_safe = False - elif durable_marker_verified_recovery: - # The durable marker and the sealed abandoned-pending proof have - # already authorized this exact same-account projection. Bind the - # attempt journal to that exact projected wire body; the terminal - # checkpoint separately retains the sealed complete client input - # count/fingerprint above. The projection remains pinned to the - # durable owner account rather than becoming account-neutral. - request_state.fresh_upstream_request_text = text_data - request_state.fresh_upstream_request_is_retry_safe = True - request_state.fresh_upstream_request_is_account_neutral = False - if durable_recovery_attempt_fingerprint is None or durable_lookup is None: - raise RuntimeError("sealed marker recovery lost its durable journal identity") - request_state.recovery_attempt_fingerprint = durable_recovery_attempt_fingerprint - request_state.recovery_attempt_session_id = durable_lookup.session_id - request_state.recovery_attempt_owner_epoch = durable_lookup.owner_epoch - request_state.marker_recovery_terminal_settlement_required = True - request_state.marker_recovery_claim_request_id = request_id - request_state.marker_recovery_rejected_response_id = durable_lookup.latest_response_id - request_state.marker_recovery_claimed = True - marker_session_id = durable_lookup.session_id - marker_owner_epoch = durable_lookup.owner_epoch - - async def rollback_failed_marker_claim() -> None: - await self._rollback_marker_recovery_claim_before_dispatch( - request_state, - api_key_id=bridge_session_key.api_key_id, - durable_session_id=marker_session_id, - durable_owner_epoch=marker_owner_epoch, - ) - - try: - marker_attempt = await self._durable_bridge.claim_and_record_live_session_recovery_attempt( - session_id=durable_lookup.session_id, - instance_id=_service_get_settings().http_responses_session_bridge_instance_id, - owner_epoch=durable_lookup.owner_epoch, - account_id=durable_lookup.account_id, - rejected_response_id=durable_lookup.latest_response_id, - attempt_fingerprint=durable_recovery_attempt_fingerprint, - claim_request_id=request_id, - journal_request_id=request_state.request_id, - model=request_state.model, - ) - except asyncio.CancelledError: - rollback_task = asyncio.create_task(rollback_failed_marker_claim()) - try: - await asyncio.shield(rollback_task) - except asyncio.CancelledError: - await rollback_task - raise - except Exception as exc: - await rollback_failed_marker_claim() - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The recovery generation could not be journaled; retrying is unsafe.", - ), - ) from exc - if marker_attempt is None: - request_state.marker_recovery_claimed = False - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "Another complete-context request already owns this recovery generation.", - ), - ) elif ( effective_payload.previous_response_id is not None and payload_looks_like_full_resend @@ -3347,20 +1844,6 @@ async def rollback_failed_marker_claim() -> None: del _fresh_state request_state.fresh_upstream_request_text = client_full_resend_fresh_upstream_request_text request_state.fresh_upstream_request_is_retry_safe = True - request_state.fresh_upstream_request_is_account_neutral = ( - _http_bridge_payload_is_account_neutral_fresh_replay(client_full_resend_payload) - ) - - async def rollback_marker_recovery_setup_failure() -> None: - if not request_state.marker_recovery_claimed or durable_lookup is None: - return - await self._rollback_marker_recovery_claim_before_dispatch( - request_state, - api_key_id=bridge_session_key.api_key_id, - durable_session_id=durable_lookup.session_id, - durable_owner_epoch=durable_lookup.owner_epoch, - ) - settings = _service_get_settings() request_deadline = request_state.started_at + _http_bridge_request_budget_seconds(settings) session_creation_headers = ( @@ -3390,19 +1873,13 @@ def durable_full_resend_allows_account_neutral_replay() -> bool: cast(list[JsonValue], payload.input), stored_count=durable_full_resend_anchor_count, preserve_developer_message_ids=True, - preserve_response_owned_agent_message_ids=True, ) if eligibility_projection is None: return False - durable_full_resend_retains_prior_output = ( - durable_lookup is not None - and responses_input_suffix_retains_prior_output( - eligibility_projection.input_items, - stored_count=eligibility_projection.stored_prefix_count, - canonical_lite_developer_index=eligibility_projection.canonical_lite_developer_index, - exact_stored_prefix_without_pending_manifest=not durable_lookup.latest_pending_tool_calls, - allow_response_owned_agent_message=durable_lookup.latest_pending_tool_calls == {}, - ) + durable_full_resend_retains_prior_output = responses_input_suffix_retains_prior_output( + eligibility_projection.input_items, + stored_count=eligibility_projection.stored_prefix_count, + canonical_lite_developer_index=eligibility_projection.canonical_lite_developer_index, ) if not durable_full_resend_retains_prior_output: return False @@ -3455,6 +1932,21 @@ def switch_to_account_neutral_replay() -> None: nonlocal text_data nonlocal untrimmed_effective_payload + preserve_operation_identity = durable_recovery_attempt_claimed or durable_recovery_fresh_replay + prior_operation_id = request_state.operation_id if preserve_operation_identity else None + prior_operation_fingerprint = request_state.operation_fingerprint if preserve_operation_identity else None + prior_operation_parent_response_id = ( + request_state.operation_parent_response_id or request_state.previous_response_id + if preserve_operation_identity + else None + ) + prior_operation_registered = request_state.operation_registered if preserve_operation_identity else False + prior_operation_attempt_generation = ( + request_state.operation_attempt_generation if preserve_operation_identity else 0 + ) + prior_operation_persisted_response_id = ( + request_state.operation_persisted_response_id if preserve_operation_identity else None + ) failed_owner_id = request_state.preferred_account_id _log_http_bridge_event( "owner_unavailable_fresh_resend", @@ -3479,6 +1971,14 @@ def switch_to_account_neutral_replay() -> None: if fresh_payload is None: raise RuntimeError("account-neutral replay projection missing after eligibility check") request_state, text_data = prepare_bridge_request(fresh_payload) + if preserve_operation_identity: + request_state.operation_id = prior_operation_id + request_state.operation_fingerprint = prior_operation_fingerprint + request_state.operation_parent_response_id = prior_operation_parent_response_id + request_state.operation_registered = prior_operation_registered + request_state.operation_attempt_generation = prior_operation_attempt_generation + request_state.operation_persisted_response_id = prior_operation_persisted_response_id + request_state.operation_rebind_required = True request_state.enforce_openai_sdk_contract = enforce_openai_sdk_contract request_state.affinity_policy = affinity request_state.excluded_account_ids.update(fresh_replay_excluded_account_ids) @@ -3508,8 +2008,7 @@ def switch_to_account_neutral_replay() -> None: file_required_preferred_account = False if durable_recovery_attempt_claimed: - if not durable_marker_verified_recovery_candidate: - switch_to_account_neutral_replay() + switch_to_account_neutral_replay() request_state.recovery_attempt_fingerprint = durable_recovery_attempt_fingerprint request_state.recovery_attempt_session_id = durable_recovery_attempt_session_id request_state.recovery_attempt_owner_epoch = durable_recovery_attempt_owner_epoch @@ -3533,7 +2032,7 @@ def switch_to_account_neutral_replay() -> None: ) switch_to_account_neutral_replay() - if required_continuity_owner_missing and request_state.rowless_recovery_capture_intent is None: + if required_continuity_owner_missing: owner_unavailable = ProxyResponseError( 502, openai_error( @@ -3569,7 +2068,19 @@ def switch_to_account_neutral_replay() -> None: previous_response_id=request_state.previous_response_id, gateway_safe_mode=runtime_config.gateway_safe_mode, allow_forward_to_owner=( - not fresh_replay_excluded_account_ids and not force_local_recovery_creation + not fresh_replay_excluded_account_ids + and not force_local_recovery_creation + and not affinity.abandon_unavailable_legacy_owner + ), + # A single-instance restart can leave an anchored durable + # row owned by the previous process epoch. Once the old + # owner is proven dead, let the initial continuation + # rebind locally; clustered deployments still route or + # fail closed through the normal owner path. + allow_previous_response_recovery_rebind=( + request_state.previous_response_id is not None + and dead_owner_anchor + and not _http_bridge_requires_cluster_registration(settings) ), forwarded_request=forwarded_request, forwarded_original_request_unanchored=original_request_unanchored, @@ -3580,7 +2091,6 @@ def switch_to_account_neutral_replay() -> None: preferred_account_id=request_state.preferred_account_id, preferred_account_has_continuity_provenance=preferred_account_has_continuity_provenance, fallback_on_preferred_account_unavailable=not file_required_preferred_account, - allow_previous_response_recovery_rebind=(request_state.rowless_recovery_capture_intent is not None), request_usage_budget=request_state.request_usage_budget, request_deadline=request_deadline, session_header_fallback_key=session_header_fallback_key, @@ -3588,40 +2098,7 @@ def switch_to_account_neutral_replay() -> None: deferred_account_backoff_lifecycle=request_state.deferred_account_backoff_lifecycle, defer_account_health_writes=request_state.api_key_reservation is not None, ) - except asyncio.CancelledError: - - async def rollback_cancelled_setup() -> None: - await self._rollback_rowless_preflight_setup_failure_if_unbound(request_state) - await rollback_marker_recovery_setup_failure() - - rollback_task = asyncio.create_task(rollback_cancelled_setup()) - try: - await asyncio.shield(rollback_task) - except asyncio.CancelledError: - await rollback_task - raise except ProxyResponseError as exc: - if ( - request_state.rowless_recovery_authority_id is not None - and request_state.rowless_recovery_generation is not None - and request_state.rowless_recovery_wire_fingerprint is not None - ): - async with SessionLocal() as rowless_session: - restored = await RowlessRecoveryRepository(rowless_session).rollback_preflight_setup_failure( - authority_id=request_state.rowless_recovery_authority_id, - generation=request_state.rowless_recovery_generation, - request_id=request_state.request_id, - wire_request_fingerprint=request_state.rowless_recovery_wire_fingerprint, - ) - if not restored: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The failed semantic-rebase setup could not be restored safely.", - ), - ) from exc - raise if not owner_unavailable_allows_account_neutral_replay(exc): exc_code, _exc_message = _proxy_error_code_message(exc) if not unanchored_fork_spill_attempted and _http_bridge_unanchored_fork_can_spill_on_cap( @@ -3667,10 +2144,8 @@ async def rollback_cancelled_setup() -> None: ): yield line if _service_time().monotonic() >= request_deadline: - await rollback_marker_recovery_setup_failure() raise continue - await rollback_marker_recovery_setup_failure() raise _log_http_bridge_event( "owner_unavailable_fresh_resend", @@ -3683,13 +2158,8 @@ async def rollback_cancelled_setup() -> None: ) switch_to_account_neutral_replay() continue - except Exception: - await self._rollback_rowless_preflight_setup_failure_if_unbound(request_state) - await rollback_marker_recovery_setup_failure() - raise break if isinstance(session_or_forward, _HTTPBridgeOwnerForward): - await rollback_marker_recovery_setup_failure() await _current_origin_legacy_owner_anchor_lookup( durable_bridge=self._durable_bridge, bridge_session_key=session_or_forward.key, @@ -3716,6 +2186,8 @@ async def rollback_cancelled_setup() -> None: yield line return except ProxyResponseError as exc: + if not _owner_forward_failure_allows_local_recovery(exc): + raise if forwarded_any: yield _partial_output_proxy_error_event_block( exc, @@ -3765,11 +2237,7 @@ async def rollback_cancelled_setup() -> None: session_key_value=bridge_session_key.affinity_key, api_key_id=bridge_session_key.api_key_id, turn_state=takeover_turn_state, - session_header=( - session_header_fallback_key.affinity_key - if explicit_prompt_cache_key is not None and session_header_fallback_key is not None - else incoming_session_header - ), + session_header=durable_session_header_alias, previous_response_id=effective_payload.previous_response_id, ) except Exception: @@ -3780,21 +2248,6 @@ async def rollback_cancelled_setup() -> None: else: if _http_bridge_durable_lookup_allows_turn_state_takeover(fresh_turn_state_lookup): durable_lookup = fresh_turn_state_lookup - ( - durable_abandoned_pending_full_resend_proof, - durable_abandoned_pending_full_resend_rejection_reason, - ) = _verify_durable_abandoned_pending_full_resend_with_reason( - payload, - fresh_turn_state_lookup, - ) - if _verify_durable_full_resend(payload, fresh_turn_state_lookup) is None: - _log_abandoned_pending_full_resend_rejection( - bridge_session_key=bridge_session_key, - payload=payload, - durable_lookup=fresh_turn_state_lookup, - reason_code=durable_abandoned_pending_full_resend_rejection_reason, - stage="takeover_lookup", - ) if fresh_turn_state_lookup is None: durable_full_resend_anchor_count = None durable_full_resend_anchor_fingerprint = None @@ -4028,13 +2481,7 @@ async def rollback_cancelled_setup() -> None: } ) if durable_lookup.latest_response_id != session.last_completed_response_id: - session.last_pending_tool_call_manifest_invalid = ( - durable_lookup.latest_pending_tool_calls is None - ) - session.last_pending_tool_calls = dict(durable_lookup.latest_pending_tool_calls or {}) - session.last_response_transition_manifest = ( - durable_lookup.latest_response_transition_manifest - ) + session.last_pending_tool_calls = {} session.last_completed_response_id = durable_lookup.latest_response_id session.last_completed_response_account_id = durable_lookup.account_id session.last_completed_input_count = durable_full_resend_anchor_count @@ -4096,7 +2543,6 @@ async def rollback_cancelled_setup() -> None: request_state.request_stage if owner_forward_fresh_replay else "reattach" ) retry_request_state.preferred_account_id = request_state.preferred_account_id - retry_request_state.file_required_preferred_account = file_required_preferred_account retry_request_state.excluded_account_ids.update(request_state.excluded_account_ids) if recovery_anchor_input_count is not None: retry_request_state.input_item_count = recovery_anchor_input_count @@ -4161,12 +2607,9 @@ async def rollback_cancelled_setup() -> None: ): if durable_lookup.latest_response_id != session.last_completed_response_id: # The pending tool calls were recorded for the session's own - # last completed response. Rebind them to the durable anchor, - # preserving an unavailable manifest as invalid instead of - # silently treating it as a verified empty manifest. - session.last_pending_tool_call_manifest_invalid = durable_lookup.latest_pending_tool_calls is None - session.last_pending_tool_calls = dict(durable_lookup.latest_pending_tool_calls or {}) - session.last_response_transition_manifest = durable_lookup.latest_response_transition_manifest + # last completed response; a durable anchor pointing elsewhere + # must not trigger interrupted-output injection. + session.last_pending_tool_calls = {} session.last_completed_response_id = durable_lookup.latest_response_id # The durable anchor is owned by the durable session's account, which # may differ from this session's account after a failover. Record the @@ -4219,7 +2662,6 @@ async def rollback_cancelled_setup() -> None: ) and (not _http_bridge_payload_looks_like_full_resend(effective_payload) or session_anchor_trimmable) session_anchor_candidate = ( session.codex_session - and request_state.rowless_recovery_authority_id is None # Honor the quarantine decision end to end: the durable anchor # skipped above must not come back as a session-level injection. and not fresh_reattach_anchor_suppressed_quarantined @@ -4291,7 +2733,6 @@ async def rollback_cancelled_setup() -> None: store_context_trim_applied = False store_context_original_count = 0 store_context_original_fingerprint: str | None = None - store_context_full_resend_proof: _VerifiedStoreContextFullResend | None = None if ( has_previous_response_id and stored_count > 0 @@ -4302,10 +2743,6 @@ async def rollback_cancelled_setup() -> None: incoming_input_list = cast(list[JsonValue], incoming_input) incoming_prefix_fingerprint = _fingerprint_input_items(incoming_input_list[:stored_count]) if incoming_prefix_fingerprint == stored_fingerprint: - store_context_full_resend_proof = _verify_store_context_full_resend( - untrimmed_effective_payload, - session, - ) store_context_trim_applied = True store_context_original_count = len(incoming_input_list) store_context_original_fingerprint = _fingerprint_input_items(incoming_input_list) @@ -4326,14 +2763,10 @@ async def rollback_cancelled_setup() -> None: stored_count, effective_payload.previous_response_id, ) - injected_input_items = ( - None - if request_state.rowless_recovery_authority_id is not None - else _http_bridge_interrupted_tool_outputs_input( - session, - payload=submit_payload, - request_id=request_id, - ) + injected_input_items = _http_bridge_interrupted_tool_outputs_input( + session, + payload=submit_payload, + request_id=request_id, ) if injected_input_items is not None: submit_payload = submit_payload.model_copy(update={"input": injected_input_items}) @@ -4389,46 +2822,13 @@ async def rollback_cancelled_setup() -> None: # keep the replay-safety decision made when the anchor was # injected. request_state.fresh_upstream_request_is_retry_safe = ( - store_context_full_resend_proof is not None + (durable_full_resend_anchor_count is None or durable_full_resend_has_safe_fresh_context) if store_context_trim_applied else previous_request_state.fresh_upstream_request_is_retry_safe ) - if request_state.fresh_upstream_request_is_retry_safe: - fresh_replay_payload = _http_bridge_payload_without_previous_response_id( - untrimmed_effective_payload - ) - request_state.fresh_upstream_request_is_account_neutral = ( - _http_bridge_payload_is_account_neutral_fresh_replay(fresh_replay_payload) - ) - elif durable_marker_verified_recovery: - # Interrupted-tool normalization re-prepares the projected - # request. Preserve both authorities across that mechanical - # rewrite: the final projected wire text/journal identity and - # the original complete client input for the new durable - # terminal checkpoint. - request_state.fresh_upstream_request_text = text_data - request_state.fresh_upstream_request_is_retry_safe = True - request_state.fresh_upstream_request_is_account_neutral = False - request_state.recovery_attempt_fingerprint = previous_request_state.recovery_attempt_fingerprint - request_state.recovery_attempt_session_id = previous_request_state.recovery_attempt_session_id - request_state.recovery_attempt_owner_epoch = previous_request_state.recovery_attempt_owner_epoch - request_state.recovery_attempt_claimed = previous_request_state.recovery_attempt_claimed - request_state.marker_recovery_terminal_settlement_required = ( - previous_request_state.marker_recovery_terminal_settlement_required - ) - request_state.marker_recovery_claimed = previous_request_state.marker_recovery_claimed - request_state.marker_recovery_claim_request_id = previous_request_state.marker_recovery_claim_request_id - request_state.marker_recovery_rejected_response_id = ( - previous_request_state.marker_recovery_rejected_response_id - ) - request_state.input_item_count = previous_request_state.input_item_count - request_state.input_full_fingerprint = previous_request_state.input_full_fingerprint elif client_full_resend_fresh_upstream_request_text is not None: request_state.fresh_upstream_request_text = client_full_resend_fresh_upstream_request_text request_state.fresh_upstream_request_is_retry_safe = True - request_state.fresh_upstream_request_is_account_neutral = ( - previous_request_state.fresh_upstream_request_is_account_neutral - ) initial_handoff_session = session initial_handoff_scope_id = ensure_request_scope_id() if original_request_unanchored else None if initial_handoff_scope_id is not None: @@ -4456,9 +2856,22 @@ async def rollback_cancelled_setup() -> None: durable_recovery_fresh_replay = False retry_request_state: _WebSocketRequestState | None = None + async def release_recovery_origin_lease() -> None: + if durable_recovery_attempt_session_id is None or durable_recovery_attempt_owner_epoch is None: + return + try: + await self._durable_bridge.release_live_session( + session_id=durable_recovery_attempt_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=durable_recovery_attempt_owner_epoch, + draining=False, + ) + except Exception: + logger.warning("Failed to release HTTP bridge recovery origin lease", exc_info=True) + async def rollback_pre_dispatch_recovery_claim() -> None: if not ( - durable_recovery_fresh_replay + (durable_recovery_fresh_replay or durable_recovery_attempt_claimed) and (retry_request_state is None or not retry_request_state.recovery_attempt_dispatched) and durable_recovery_attempt_fingerprint is not None and durable_recovery_attempt_session_id is not None @@ -4466,13 +2879,15 @@ async def rollback_pre_dispatch_recovery_claim() -> None: ): return try: - await self._durable_bridge.rollback_recovery_attempt_replayed( + rolled_back = await self._durable_bridge.rollback_recovery_attempt_replayed( session_id=durable_recovery_attempt_session_id, api_key_id=bridge_session_key.api_key_id, instance_id=_service_get_settings().http_responses_session_bridge_instance_id, owner_epoch=durable_recovery_attempt_owner_epoch, request_fingerprint=durable_recovery_attempt_fingerprint, ) + if rolled_back: + await release_recovery_origin_lease() except Exception: logger.warning("Failed to roll back pre-dispatch HTTP bridge recovery claim", exc_info=True) @@ -4480,6 +2895,19 @@ async def rollback_pre_dispatch_recovery_claim() -> None: yield event_block yielded_any = True except ProxyResponseError as exc: + if ( + request_state.operation_registered + and request_state.operation_id is not None + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + and _http_bridge_durable_recovery_predecessor_proven(request_state) + ): + # The API-level recovery loop must only run when this request + # has an actual durable operation fence. Settings alone are + # insufficient during a rolling migration where the durable + # tables may be unavailable and the bridge falls back to an + # in-memory session. + setattr(exc, "http_bridge_durable_recovery_eligible", True) if yielded_any: yield _partial_output_proxy_error_event_block( exc, @@ -4753,6 +3181,10 @@ async def rollback_pre_dispatch_recovery_claim() -> None: session, error_code="stream_incomplete", error_message="Upstream websocket closed before response.completed", + # Keep the origin lease fenced while the replacement + # session is admitted and the one-shot journal is + # either rolled back or settled. Releasing it here + # would make both transitions fail their owner fence. preserve_durable_lease=True, ) switch_to_account_neutral_replay() @@ -4773,74 +3205,8 @@ async def rollback_pre_dispatch_recovery_claim() -> None: key=bridge_session_key, ) should_attempt_previous_response_recovery = ( - effective_payload.previous_response_id is not None - and _http_bridge_should_attempt_local_previous_response_recovery(exc) - ) - recovery_error = exc.payload.get("error") if isinstance(exc.payload, dict) else None - recovery_error_code = ( - str(recovery_error["code"]) if isinstance(recovery_error, dict) and recovery_error.get("code") else None - ) - stale_anchor_rejected = bool( - isinstance(recovery_error, dict) - and ( - recovery_error_code == "bridge_previous_response_not_found" - or _is_previous_response_not_found_error( - code=recovery_error_code, - param=(str(recovery_error["param"]) if recovery_error.get("param") is not None else None), - message=(str(recovery_error["message"]) if recovery_error.get("message") is not None else None), - ) - ) - ) - proxy_injected_stale_anchor = bool( - should_attempt_previous_response_recovery - and stale_anchor_rejected - and request_state.proxy_injected_previous_response_id - and request_state.response_event_count == 0 - and untrimmed_effective_payload.previous_response_id is None - ) - durable_stale_anchor_replay = bool( - proxy_injected_stale_anchor - and request_state.proxy_injected_anchor_had_full_resend_payload - and durable_lookup is not None - and durable_full_resend_proof is not None - and durable_full_resend_proof.matches(untrimmed_effective_payload, durable_lookup) - ) - store_context_stale_anchor_replay = bool( - proxy_injected_stale_anchor - and request_state.proxy_injected_anchor_had_full_resend_payload - and store_context_full_resend_proof is not None - and store_context_full_resend_proof.matches(untrimmed_effective_payload, session) - ) - client_provided_stale_anchor_replay = bool( - should_attempt_previous_response_recovery - and stale_anchor_rejected - and not request_state.proxy_injected_previous_response_id - and request_state.response_event_count == 0 - and effective_payload.previous_response_id is not None - and durable_lookup is not None - and durable_full_resend_proof is not None - and durable_full_resend_proof.matches(untrimmed_effective_payload, durable_lookup) - ) - abandoned_pending_stale_anchor_replay = bool( - should_attempt_previous_response_recovery - and stale_anchor_rejected - and request_state.response_event_count == 0 - and durable_lookup is not None - and durable_abandoned_pending_full_resend_proof is not None - and durable_abandoned_pending_full_resend_proof.matches( - untrimmed_effective_payload, - durable_lookup, - ) - and ( - request_state.proxy_injected_previous_response_id - or effective_payload.previous_response_id is not None - ) - ) - proof_gated_stale_anchor_replay = ( - durable_stale_anchor_replay - or store_context_stale_anchor_replay - or client_provided_stale_anchor_replay - or abandoned_pending_stale_anchor_replay + effective_payload.previous_response_id is not None + and _http_bridge_should_attempt_local_previous_response_recovery(exc) ) should_attempt_context_overflow_fresh_turn_recovery = ( is_context_overflow @@ -4909,166 +3275,6 @@ async def rollback_pre_dispatch_recovery_claim() -> None: error_message="Upstream websocket closed before response.completed", ) raise - elif proof_gated_stale_anchor_replay: - # Keep the durable response anchor until the replacement - # request reaches response.completed and atomically publishes - # its new anchor. Quarantine plus the sealed full-resend proof - # below is the only authority to bypass the rejected anchor, - # including an explicit client anchor whose complete resend - # exactly matches this durable session's stored prefix. - # If replacement creation, reservation, or pre-dispatch send - # fails, the old durable row remains intact: delta requests - # stay anchored/fail closed and a later verified full resend - # can retry safely. - _quarantine_http_bridge_session( - self, - session, - reason=_HTTP_BRIDGE_QUARANTINE_REJECTED_STALE_ANCHOR_REASON, - ) - if PROMETHEUS_AVAILABLE and bridge_durable_recover_total is not None: - bridge_durable_recover_total.labels( - path=( - "stale_anchor_store_context_full_resend" - if store_context_stale_anchor_replay - else "stale_anchor_full_resend" - ) - ).inc() - proof_source = ( - "abandoned_pending_agent_boundary" - if abandoned_pending_stale_anchor_replay - else "client_explicit" - if client_provided_stale_anchor_replay - else "store_context" - if store_context_stale_anchor_replay - else "durable" - ) - _log_http_bridge_event( - "previous_response_recover_full_resend", - bridge_session_key, - account_id=session.account.id, - model=effective_payload.model, - detail=(f"outcome=single_unanchored_same_account_replay, proof_source={proof_source}"), - cache_key_family=bridge_session_key.affinity_kind, - model_class=_extract_model_class(effective_payload.model) if effective_payload.model else None, - owner_check_applied=True, - ) - await self._reset_http_bridge_session_after_local_terminal_error( - session, - error_code="previous_response_anchor_invalid", - error_message="A verified previous response anchor was rejected before execution", - preserve_durable_lease=True, - ) - recovery_path = ( - "stale_anchor_store_context_full_resend" - if store_context_stale_anchor_replay - else "stale_anchor_full_resend" - ) - retry_payload = _http_bridge_payload_without_previous_response_id(untrimmed_effective_payload) - if abandoned_pending_stale_anchor_replay: - if not isinstance(untrimmed_effective_payload.input, list): - raise - abandoned_projection = project_responses_input_for_abandoned_pending_fresh_replay( - cast(list[JsonValue], untrimmed_effective_payload.input), - stored_count=durable_abandoned_pending_full_resend_proof.stored_input_item_count, - pending_tool_calls=dict(durable_lookup.latest_pending_tool_calls or {}), - ) - if abandoned_projection is None: - raise - retry_payload = retry_payload.model_copy(update={"input": abandoned_projection.input_items}) - retry_previous_response_id = None - retry_request_stage = "durable_recovery" - retry_preferred_account_id = session.account.id - allow_previous_response_recovery_rebind = False - elif proxy_injected_stale_anchor: - # The upstream explicitly rejected an anchor that the proxy, - # rather than the client, injected on this request. Without a - # verified complete full-resend payload it would be unsafe to - # drop that anchor in-place: doing so could silently lose - # conversation history. Quarantine the logical session key so - # only the next owner-bound verified complete resend takes the - # unanchored fresh path; merely full-resend-shaped and - # delta-only requests retain the anchor and fail closed. - _quarantine_http_bridge_session( - self, - session, - reason=_HTTP_BRIDGE_QUARANTINE_REJECTED_STALE_ANCHOR_REASON, - ) - marker_persisted = False - if ( - session.durable_session_id is not None - and session.durable_owner_epoch is not None - and request_state.previous_response_id is not None - ): - try: - marker_persisted = await self._durable_bridge.mark_live_session_recovery_required( - session_id=session.durable_session_id, - instance_id=_service_get_settings().http_responses_session_bridge_instance_id, - owner_epoch=session.durable_owner_epoch, - account_id=session.account.id, - rejected_response_id=request_state.previous_response_id, - ) - except Exception: - logger.warning( - "Failed to persist durable HTTP bridge recovery-required marker", - exc_info=True, - ) - if durable_lookup is not None and not marker_persisted: - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The rejected response anchor could not be fenced; retry the request.", - ), - retryable_same_contract=False, - failure_detail="recovery_required_marker_not_persisted", - upstream_error_code="previous_response_not_found", - ) from exc - _log_http_bridge_event( - "previous_response_anchor_rejected_quarantined", - bridge_session_key, - account_id=session.account.id, - model=effective_payload.model, - detail="outcome=await_verified_full_resend", - cache_key_family=bridge_session_key.affinity_kind, - model_class=_extract_model_class(effective_payload.model) if effective_payload.model else None, - owner_check_applied=True, - ) - await self._reset_http_bridge_session_after_local_terminal_error( - session, - error_code="previous_response_anchor_invalid", - error_message="The proxy-injected previous response anchor was rejected before execution", - preserve_durable_lease=True, - ) - pending_call_resolution_required = bool( - durable_lookup is not None and durable_lookup.latest_pending_tool_calls - ) - raise ProxyResponseError( - 400, - openai_error( - ( - "previous_response_pending_call_resolution_required" - if pending_call_resolution_required - else "previous_response_complete_context_required" - ), - ( - "The saved response anchor has an unresolved pending call. " - "Retry once with the complete conversation context after resolving that call." - if pending_call_resolution_required - else ( - "The upstream rejected the saved response anchor. " - "Retry once with the complete conversation context." - ) - ), - ), - failure_phase=exc.failure_phase, - retryable_same_contract=False, - failure_detail="durable_recovery_required", - failure_exception_type=exc.failure_exception_type, - upstream_status_code=( - exc.upstream_status_code if exc.upstream_status_code is not None else exc.status_code - ), - upstream_error_code="previous_response_not_found", - ) from exc else: if PROMETHEUS_AVAILABLE and bridge_durable_recover_total is not None: bridge_durable_recover_total.labels(path="local_previous_response_error").inc() @@ -5107,69 +3313,72 @@ async def rollback_pre_dispatch_recovery_claim() -> None: retry_preferred_account_id = request_state.preferred_account_id allow_previous_response_recovery_rebind = True - while True: - try: - session = await self._get_or_create_http_bridge_session( - bridge_session_key, - headers=dict(session_creation_headers), - affinity=affinity, - api_key=api_key, - request_model=retry_payload.model, - request_service_tier=request_state.requested_service_tier, - idle_ttl_seconds=_effective_http_bridge_idle_ttl_seconds( + try: + while True: + try: + session = await self._get_or_create_http_bridge_session( + bridge_session_key, + headers=dict(session_creation_headers), affinity=affinity, - idle_ttl_seconds=idle_ttl_seconds, - codex_idle_ttl_seconds=codex_idle_ttl_seconds, - prompt_cache_idle_ttl_seconds=prompt_cache_idle_ttl_seconds, - ), - max_sessions=max_sessions, - previous_response_id=retry_previous_response_id, - gateway_safe_mode=runtime_config.gateway_safe_mode, - allow_forward_to_owner=False, - forwarded_request=False, - allow_previous_response_recovery_rebind=allow_previous_response_recovery_rebind, - session_header_fallback_key=session_header_fallback_key, - durable_lookup=durable_lookup, - request_stage=retry_request_stage, - preferred_account_id=retry_preferred_account_id, - preferred_account_has_continuity_provenance=preferred_account_has_continuity_provenance, - fallback_on_preferred_account_unavailable=not ( - (file_required_preferred_account or proof_gated_stale_anchor_replay) - and retry_preferred_account_id is not None - ), - request_usage_budget=estimate_api_key_request_usage(retry_payload), - request_deadline=request_deadline, - exclude_account_ids=request_state.excluded_account_ids or None, - deferred_account_backoff_lifecycle=request_state.deferred_account_backoff_lifecycle, - defer_account_health_writes=request_state.api_key_reservation is not None, - ) - except ProxyResponseError as capacity_exc: - wait_plan = _http_bridge_capacity_wait_plan(capacity_exc, request_deadline=request_deadline) - if wait_plan is None: - raise - bounded_wait_seconds, account_capacity_wait_seconds, message = wait_plan - logger.info( - "Waiting for an account to recover before retrying HTTP bridge local recovery session " - "request_id=%s model=%s sleep_seconds=%.1f recovery_hint_seconds=%.1f path=%s error=%s", - request_id, - retry_payload.model, - bounded_wait_seconds, - account_capacity_wait_seconds, - recovery_path, - message, - ) - async for line in _iter_account_capacity_wait_sse( - request_id=request_id, - reason=message, - sleep_seconds=bounded_wait_seconds, - emit_keepalives=not propagate_http_errors, - request_state=request_state, - ): - yield line - if _service_time().monotonic() >= request_deadline: - raise - continue - break + api_key=api_key, + request_model=retry_payload.model, + request_service_tier=request_state.requested_service_tier, + idle_ttl_seconds=_effective_http_bridge_idle_ttl_seconds( + affinity=affinity, + idle_ttl_seconds=idle_ttl_seconds, + codex_idle_ttl_seconds=codex_idle_ttl_seconds, + prompt_cache_idle_ttl_seconds=prompt_cache_idle_ttl_seconds, + ), + max_sessions=max_sessions, + previous_response_id=retry_previous_response_id, + gateway_safe_mode=runtime_config.gateway_safe_mode, + allow_forward_to_owner=False, + forwarded_request=False, + allow_previous_response_recovery_rebind=allow_previous_response_recovery_rebind, + session_header_fallback_key=session_header_fallback_key, + durable_lookup=durable_lookup, + request_stage=retry_request_stage, + preferred_account_id=retry_preferred_account_id, + preferred_account_has_continuity_provenance=preferred_account_has_continuity_provenance, + fallback_on_preferred_account_unavailable=not ( + file_required_preferred_account and retry_preferred_account_id is not None + ), + request_usage_budget=estimate_api_key_request_usage(retry_payload), + request_deadline=request_deadline, + exclude_account_ids=request_state.excluded_account_ids or None, + deferred_account_backoff_lifecycle=request_state.deferred_account_backoff_lifecycle, + defer_account_health_writes=request_state.api_key_reservation is not None, + ) + except ProxyResponseError as capacity_exc: + wait_plan = _http_bridge_capacity_wait_plan(capacity_exc, request_deadline=request_deadline) + if wait_plan is None: + raise + bounded_wait_seconds, account_capacity_wait_seconds, message = wait_plan + logger.info( + "Waiting for an account to recover before retrying HTTP bridge local recovery session " + "request_id=%s model=%s sleep_seconds=%.1f recovery_hint_seconds=%.1f path=%s error=%s", + request_id, + retry_payload.model, + bounded_wait_seconds, + account_capacity_wait_seconds, + recovery_path, + message, + ) + async for line in _iter_account_capacity_wait_sse( + request_id=request_id, + reason=message, + sleep_seconds=bounded_wait_seconds, + emit_keepalives=not propagate_http_errors, + request_state=request_state, + ): + yield line + if _service_time().monotonic() >= request_deadline: + raise + continue + break + except BaseException: + await rollback_pre_dispatch_recovery_claim() + raise _record_bridge_reattach(path=recovery_path, outcome="success") local_recovery_scope_id = ensure_request_scope_id() if original_request_unanchored else None @@ -5199,36 +3408,56 @@ async def rollback_pre_dispatch_recovery_claim() -> None: retry_payload, reservation=retry_api_key_reservation, ) + if ( + recovery_path == "local_previous_response_error" + and request_state.operation_registered + and request_state.operation_id is not None + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + reset_operation_event_spool = getattr(self._durable_bridge, "reset_operation_event_spool", None) + if callable(reset_operation_event_spool): + reset_ok = await reset_operation_event_spool( + operation_id=request_state.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + ) + if not reset_ok: + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP response recovery spool could not be reset; retry the request.", + ), + ) + # A recovery request is the one bounded server-side replay; + # prevent a second cooldown bypass if this fresh socket also + # fails before response.created. + if recovery_path == "local_previous_response_error": + retry_request_state.replay_count = max(1, request_state.replay_count + 1) + # Keep the durable operation identity attached to the + # server-owned recovery attempt. Re-registering the same + # fingerprint would be interpreted as an already-dispatched + # unknown operation and suppress the intended one-shot replay. + retry_request_state.operation_id = request_state.operation_id + retry_request_state.operation_fingerprint = request_state.operation_fingerprint + retry_request_state.operation_parent_response_id = request_state.operation_parent_response_id + retry_request_state.operation_registered = request_state.operation_registered + retry_request_state.operation_attempt_generation = request_state.operation_attempt_generation + retry_request_state.operation_persisted_response_id = request_state.operation_persisted_response_id + retry_request_state.operation_rebind_required = request_state.operation_rebind_required + if recovery_path == "local_previous_response_error": + # The prior response.failed/error made the operation + # terminal. Re-enter record_operation so its owner fence + # atomically moves it back to submitted before send. + retry_request_state.operation_rebind_required = True retry_request_state.enforce_openai_sdk_contract = enforce_openai_sdk_contract - if abandoned_pending_stale_anchor_replay: - # The upstream must receive only the fail-closed projection, - # but continuity is still owned by the client's verified - # full resend. Persist that original prefix so the next - # client full resend can trim against the recovered anchor - # instead of replaying the projected history again. - abandoned_pending_client_input = cast( - list[JsonValue], - untrimmed_effective_payload.input, - ) - retry_request_state.input_item_count = len(abandoned_pending_client_input) - retry_request_state.input_full_fingerprint = _fingerprint_input_items( - abandoned_pending_client_input - ) if durable_recovery_fresh_replay and durable_recovery_attempt_fingerprint is not None: retry_request_state.recovery_attempt_fingerprint = durable_recovery_attempt_fingerprint retry_request_state.recovery_attempt_session_id = request_state.recovery_attempt_session_id retry_request_state.recovery_attempt_owner_epoch = request_state.recovery_attempt_owner_epoch retry_request_state.recovery_attempt_claimed = True - retry_request_state.marker_recovery_terminal_settlement_required = ( - request_state.marker_recovery_terminal_settlement_required - ) - retry_request_state.marker_recovery_claimed = request_state.marker_recovery_claimed - retry_request_state.marker_recovery_claim_request_id = ( - request_state.marker_recovery_claim_request_id - ) - retry_request_state.marker_recovery_rejected_response_id = ( - request_state.marker_recovery_rejected_response_id - ) _apply_http_bridge_downstream_turn_state( retry_request_state, downstream_turn_state=downstream_turn_state, @@ -5237,7 +3466,6 @@ async def rollback_pre_dispatch_recovery_claim() -> None: retry_request_state.transport = _REQUEST_TRANSPORT_HTTP retry_request_state.request_stage = retry_request_stage retry_request_state.preferred_account_id = retry_preferred_account_id - retry_request_state.file_required_preferred_account = file_required_preferred_account retry_request_state.excluded_account_ids.update(request_state.excluded_account_ids) retry_events: AsyncGenerator[str, None] = self._stream_http_bridge_session_events( @@ -5299,8 +3527,11 @@ async def _reset_http_bridge_session_after_local_terminal_error( preserve_durable_lease: bool = False, ) -> None: async with self._http_bridge_lock: - if self._http_bridge_sessions.get(session.key) is session: - self._http_bridge_sessions.pop(session.key, None) + # Pending settlement below may block or fail before resource close + # starts. Transfer canonical routing into detached lifecycle + # ownership first so capacity, invalidation, and shutdown continue + # to see the live socket and leases throughout that interval. + self._detach_http_bridge_session_locked(session.key, expected_session=session) async with session.pending_lock: session.queued_request_count = 0 await self._fail_pending_websocket_requests( @@ -5380,6 +3611,23 @@ async def retry_precreated_for_idle_recovery( request_state.request_id, exc.error_code, ) + if getattr( + _service_get_settings(), + "http_responses_session_bridge_ambiguous_continuation_recovery_mode", + "fail_closed", + ) == "server_indefinite_recovery" and _http_bridge_server_anchored_replay_enabled(request_state): + # Let the outer server-owned recovery loop classify this + # eventless transport failure as retryable. Returning a + # synthetic response.failed event would make the loop + # believe the attempt completed successfully after one try. + raise ProxyResponseError( + 502, + openai_error( + "stream_idle_timeout", + str(exc), + error_type="server_error", + ), + ) from exc return ( False, format_sse_event( @@ -5394,9 +3642,150 @@ async def retry_precreated_for_idle_recovery( ), ) + def operation_fenced_cooldown_wait_enabled() -> bool: + """Allow a hard turn to wait until its durable fence can arbitrate recovery.""" + return ( + getattr( + _service_get_settings(), + "http_responses_session_bridge_ambiguous_continuation_recovery_mode", + "fail_closed", + ) + in {"server_anchored_replay_once", "server_indefinite_recovery"} + and getattr(_service_get_settings(), "http_responses_session_bridge_operation_ledger_enabled", True) + and request_state.hard_continuity_anchor + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + and request_state.previous_response_id is None + and request_state.response_id is None + and request_state.response_event_count == 0 + ) + def continuity_bound_without_safe_replay() -> bool: """Do not hold a client stream through a cooldown we cannot use.""" - return _http_bridge_continuity_bound_without_safe_replay(request_state) + return _http_bridge_continuity_bound_without_safe_replay(request_state) and not ( + _http_bridge_server_anchored_replay_enabled(request_state) or operation_fenced_cooldown_wait_enabled() + ) + + async def wait_through_operation_fenced_startup_cooldown() -> bool: + if session.key.strength != "hard" or not operation_fenced_cooldown_wait_enabled(): + return False + retry_cooldown_seconds = await self._http_bridge_precreated_retry_cooldown_seconds(session) + if retry_cooldown_seconds <= 0: + return False + remaining_budget_seconds = request_deadline - _service_time().monotonic() + if remaining_budget_seconds <= 0: + return False + wait_seconds = min(retry_cooldown_seconds, remaining_budget_seconds) + async with session.pending_lock: + if session.queued_request_count >= queue_limit: + raise ProxyResponseError( + 429, + openai_error( + "bridge_queue_full", + "HTTP responses session bridge queue is full", + error_type="rate_limit_error", + ), + ) + session.queued_request_count += 1 + _log_http_bridge_event( + "wait_operation_fenced_cooldown", + session.key, + account_id=session.account.id, + model=session.request_model, + detail="hard_turn_operation_fence", + cache_key_family=session.key.affinity_kind, + ) + logger.info( + "HTTP bridge waiting through retry-circuit cooldown before durable hard-turn arbitration " + "request_id=%s wait_seconds=%.1f remaining_budget_seconds=%.1f", + request_state.request_id, + wait_seconds, + remaining_budget_seconds, + ) + # No upstream request has been dispatched on this path. After the + # cooldown, normal submission still has to create or claim the + # durable operation fence before response.create can be sent. + try: + current_instance = _service_get_settings().http_responses_session_bridge_instance_id + lease_refresh_interval_seconds = max( + 1.0, + min( + _http_bridge_durable_lease_ttl_seconds() / 3.0, + wait_seconds, + ), + ) + remaining_wait_seconds = wait_seconds + while remaining_wait_seconds > 0: + sleep_seconds = min(remaining_wait_seconds, lease_refresh_interval_seconds) + await asyncio.sleep(sleep_seconds) + remaining_wait_seconds = max(0.0, remaining_wait_seconds - sleep_seconds) + if remaining_wait_seconds <= 0: + break + try: + owner_lookup = await self._durable_bridge.renew_live_session( + session_id=session.durable_session_id, + api_key_id=session.key.api_key_id, + instance_id=current_instance, + owner_epoch=session.durable_owner_epoch, + lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(), + latest_turn_state=session.downstream_turn_state, + latest_response_id=None, + ) + except Exception as exc: + session.closed = True + session.upstream_control.reconnect_requested = True + session.upstream_control.retire_after_drain = True + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP responses session ownership could not be renewed; retry the request.", + ), + ) from exc + if ( + owner_lookup is None + or owner_lookup.owner_instance_id != current_instance + or owner_lookup.owner_epoch != session.durable_owner_epoch + ): + session.closed = True + session.upstream_control.reconnect_requested = True + session.upstream_control.retire_after_drain = True + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP responses session ownership changed during cooldown; retry the request.", + ), + ) + finally: + async with session.pending_lock: + session.queued_request_count = max(0, session.queued_request_count - 1) + return True + + async def operation_fenced_request_budget_terminal_event() -> str | None: + if not operation_fenced_cooldown_wait_enabled() or _service_time().monotonic() < request_deadline: + return None + await self._release_websocket_request_state_reservation(request_state) + request_state.api_key_reservation = None + if propagate_http_errors: + raise ProxyResponseError( + 503, + openai_error( + "upstream_request_timeout", + "HTTP responses session bridge recovery exceeded the request budget.", + error_type="server_error", + ), + ) + return format_sse_event( + cast( + Mapping[str, JsonValue], + response_failed_event( + "stream_idle_timeout", + "HTTP responses session bridge recovery exceeded the request budget", + response_id=_websocket_downstream_response_id(request_state), + ), + ) + ) async def startup_continuity_cooldown_terminal_event() -> str | None: if ( @@ -5406,8 +3795,7 @@ async def startup_continuity_cooldown_terminal_event() -> str | None: or request_state.response_event_count > 0 ): return None - retry_snapshot = await self._http_bridge_retry_circuit_snapshot(session) - retry_cooldown_seconds = retry_snapshot.retry_after_seconds + retry_cooldown_seconds = await self._http_bridge_precreated_retry_cooldown_seconds(session) if retry_cooldown_seconds <= 0: return None if PROMETHEUS_AVAILABLE and stream_idle_timeout_total is not None: @@ -5424,43 +3812,31 @@ async def startup_continuity_cooldown_terminal_event() -> str | None: request_state.request_id, retry_cooldown_seconds, ) - - async def rollback_unsent_recovery_claims() -> None: - await self._rollback_rowless_preflight_setup_failure_if_unbound(request_state) - await self._rollback_marker_recovery_claim_before_dispatch( - request_state, - api_key_id=session.key.api_key_id, - durable_session_id=session.durable_session_id, - durable_owner_epoch=session.durable_owner_epoch, - ) - - rollback_task = asyncio.create_task(rollback_unsent_recovery_claims()) - _, rollback_cancellation = await _await_task_deferring_cancellation(rollback_task) - if rollback_cancellation is not None: - raise rollback_cancellation # This path returns before the request is submitted, so the normal # detach/finally cleanup cannot settle an API-key reservation. # Release it before handing the synthetic terminal event to the # non-streaming collector. await self._release_websocket_request_state_reservation(request_state) request_state.api_key_reservation = None + if propagate_http_errors and _http_bridge_client_full_history_recovery_enabled(request_state): + raise ProxyResponseError( + 400, + _http_bridge_client_full_history_recovery_error(), + ) if propagate_http_errors: if request_state.durable_owner_dead: raise _http_bridge_dead_owner_previous_response_not_found_proxy_error( previous_response_id=_http_bridge_dead_owner_previous_response_id(request_state), ) - retry_after_seconds = max(1, math.ceil(retry_cooldown_seconds)) raise ProxyResponseError( 503, openai_error( "upstream_request_timeout", - _http_bridge_retry_circuit_error_message( - retry_snapshot.last_detail, - retry_after_seconds=retry_after_seconds, - ), + "HTTP responses session bridge is cooling down after repeated upstream " + "timeouts; retry shortly.", error_type="server_error", ), - retry_after_seconds=retry_after_seconds, + retry_after_seconds=max(1, math.ceil(retry_cooldown_seconds)), ) return format_sse_event( cast( @@ -5479,6 +3855,12 @@ async def rollback_unsent_recovery_claims() -> None: ) while True: + budget_terminal_event = await operation_fenced_request_budget_terminal_event() + if budget_terminal_event is not None: + yield budget_terminal_event + return + if await wait_through_operation_fenced_startup_cooldown(): + continue startup_terminal_event = await startup_continuity_cooldown_terminal_event() if startup_terminal_event is not None: yield startup_terminal_event @@ -5502,6 +3884,7 @@ async def rollback_unsent_recovery_claims() -> None: lifecycle = request_state.deferred_account_backoff_lifecycle if lifecycle is not None: lifecycle.settlement_owned = True + _signal_propagated_responses_service_cleanup_ready() except ProxyResponseError as exc: if request_state.bridge_soft_capacity_reroute_allowed: raise @@ -5559,12 +3942,17 @@ async def rollback_unsent_recovery_claims() -> None: raise if gate_contention and session.closed: raise + # Durable hard-turn admission may rewrite the request state + # with a completed predecessor before a pre-dispatch capacity + # failure. Retry the exact rewritten body rather than the + # stale outer-loop payload, or the next attempt could lose the + # injected previous_response_id and its continuity anchor. + text_data = request_state.request_text or text_data continue break event_queue = request_state.event_queue assert event_queue is not None - initial_retry_snapshot = await self._http_bridge_retry_circuit_snapshot(session) - initial_retry_cooldown_seconds = initial_retry_snapshot.retry_after_seconds + initial_retry_cooldown_seconds = await self._http_bridge_precreated_retry_cooldown_seconds(session) if ( initial_retry_cooldown_seconds > 0 and session.key.strength == "hard" @@ -5607,23 +3995,25 @@ async def rollback_unsent_recovery_claims() -> None: # gate, reservation, and pending queue entry while marking the # upstream handoff for retirement. await self._detach_http_bridge_request(session, request_state=request_state) + if propagate_http_errors and _http_bridge_client_full_history_recovery_enabled(request_state): + raise ProxyResponseError( + 400, + _http_bridge_client_full_history_recovery_error(), + ) if propagate_http_errors: if request_state.durable_owner_dead: raise _http_bridge_dead_owner_previous_response_not_found_proxy_error( previous_response_id=_http_bridge_dead_owner_previous_response_id(request_state), ) - retry_after_seconds = max(1, math.ceil(initial_retry_cooldown_seconds)) raise ProxyResponseError( 503, openai_error( "upstream_request_timeout", - _http_bridge_retry_circuit_error_message( - initial_retry_snapshot.last_detail, - retry_after_seconds=retry_after_seconds, - ), + "HTTP responses session bridge is cooling down after repeated upstream " + "timeouts; retry shortly.", error_type="server_error", ), - retry_after_seconds=retry_after_seconds, + retry_after_seconds=max(1, math.ceil(initial_retry_cooldown_seconds)), ) yield terminal_event return @@ -5780,6 +4170,9 @@ def stream_idle_keepalive(*, downstream_response_id: str) -> str | None: if not completed_delivery_in_progress: keepalive_count += 1 if not completed_delivery_in_progress and keepalive_count >= max_keepalive_count: + timed_out_retry_circuit_attempt_selection = ( + _http_bridge_retry_circuit_attempt_selection_for_pending_requests((request_state,)) + ) if not response_started: retried = False if not circuit_keepalive_waiting: @@ -5848,6 +4241,13 @@ def stream_idle_keepalive(*, downstream_response_id: str) -> str | None: retry_cooldown_seconds, continuity_bound, ) + if propagate_http_errors and _http_bridge_client_full_history_recovery_enabled( + request_state + ): + raise ProxyResponseError( + 400, + _http_bridge_client_full_history_recovery_error(), + ) yield format_sse_event( cast( Mapping[str, JsonValue], @@ -5891,6 +4291,13 @@ def stream_idle_keepalive(*, downstream_response_id: str) -> str | None: retry_cooldown_seconds, retry_cooldown_remaining_budget, ) + if propagate_http_errors and _http_bridge_client_full_history_recovery_enabled( + request_state + ): + raise ProxyResponseError( + 400, + _http_bridge_client_full_history_recovery_error(), + ) yield format_sse_event( cast( Mapping[str, JsonValue], @@ -5956,9 +4363,10 @@ def stream_idle_keepalive(*, downstream_response_id: str) -> str | None: if keepalive_event is not None: yield keepalive_event continue - await self._record_http_bridge_retry_circuit_failure( + await self._record_http_bridge_retry_circuit_failure_for_attempt_selection( session, detail="stream_idle_timeout", + selection=timed_out_retry_circuit_attempt_selection, ) if PROMETHEUS_AVAILABLE and stream_idle_timeout_total is not None: stream_idle_timeout_total.labels(surface="http_bridge").inc() diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index 1489d6d2a8..043e1bef46 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -2,6 +2,8 @@ import asyncio import logging +import time +from collections.abc import Awaitable, Callable from dataclasses import replace from typing import Any, TypeVar, cast @@ -37,17 +39,16 @@ ) from app.core.errors import response_failed_event from app.core.openai.models import OpenAIEvent -from app.core.openai.parsing import parse_sse_event_payload -from app.core.openai.public_output import ( - collect_public_output_item_event, - merge_public_response_output_items, +from app.core.openai.parsing import ( + _LIFECYCLE_EVENT_TYPES, + classify_event_type, + parse_sse_event_payload, ) from app.core.types import JsonValue from app.core.usage.live_hub import publish_live_usage from app.core.usage.live_snapshots import EVENT_MARKER, parse_rate_limit_event_text from app.core.utils.request_id import reset_request_id, set_request_id from app.core.utils.sse import format_sse_event, parse_sse_data_json -from app.db.session import SessionLocal from app.modules.proxy._service.api_key_usage import ( _API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS, ) @@ -64,6 +65,7 @@ _http_bridge_eventless_precreated_deadline, _http_bridge_request_budget_seconds, _http_bridge_request_counts_against_queue, + _http_bridge_retry_circuit_attempt_selection_for_pending_requests, _log_http_bridge_event, _normalize_http_bridge_error_event, _record_http_bridge_stuck_retire, @@ -83,6 +85,7 @@ _classify_upstream_close, _find_websocket_request_state_by_response_id, _http_error_status_from_payload, + _is_account_neutral_transport_drop, _is_missing_tool_output_error, _is_previous_response_not_found_error, _is_security_work_authorization_required_error, @@ -142,9 +145,10 @@ _clear_websocket_deferred_reasoning_downstream_texts, _clear_websocket_precreated_replay_fallback, _clear_websocket_request_error_overrides, - _event_type_from_payload, _HTTPBridgeCompletedDeliveryScope, + _HTTPBridgeRetryCircuitAttemptSelection, _HTTPBridgeSession, + _mark_response_create_attempt_observed, _pop_websocket_deferred_reasoning_downstream_texts, _record_response_event, _signal_propagated_capacity_startup_ready, @@ -191,24 +195,10 @@ _extract_model_class, ) from app.modules.proxy.continuity import is_http_bridge_account_neutral_replay -from app.modules.proxy.durable_bridge_coordinator import DurableBridgeLookup -from app.modules.proxy.durable_bridge_repository import durable_bridge_hash from app.modules.proxy.helpers import ( _normalize_error_code, is_upstream_model_capacity_error, ) -from app.modules.proxy.response_transition_manifest import ( - ResponseTransitionManifest, - build_response_transition_manifest, -) -from app.modules.proxy.rowless_recovery import ( - rowless_actual_wire_fingerprint, - rowless_projected_actual_wire_text, -) -from app.modules.proxy.rowless_recovery_repository import ( - RowlessRecoveryRepository, - RowlessRecoveryStateError, -) from app.modules.proxy.tool_call_dedupe import ( mark_duplicate_tool_call_downstream_event, rewrite_parallel_tool_call_text, @@ -232,6 +222,295 @@ 120.0, ) _HTTP_BRIDGE_RECOVERY_SETTLEMENT_LEASE_REFRESH_INTERVAL_SECONDS = 10.0 +# A single missing response.created is not proof that an account is bad: the +# upstream may have accepted the request while the transport was silent. Only +# repeated failures on separate bridge retirements are allowed to influence +# account routing, and the signal expires quickly so a transient upstream +# incident does not permanently drain an account. +_HTTP_BRIDGE_ACCOUNT_TIMEOUT_WINDOW_SECONDS = 300.0 +_HTTP_BRIDGE_ACCOUNT_TIMEOUT_EJECTION_THRESHOLD = 3 + + +async def _record_http_bridge_account_timeout_signal( + service: Any, + session: "_HTTPBridgeSession", + *, + detail: str = _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, +) -> None: + """Drain an account after repeated eventless upstream failures. + + This is deliberately separate from the per-session retry circuit. A + timeout or abrupt eventless transport drop cannot be replayed safely for a + continuity-bound turn, but three independent eventless failures are enough + evidence to keep *new* turns away from that account until its normal + health probe succeeds. + """ + + account_id = session.account.id + now = time.monotonic() + async with service._http_bridge_account_timeout_lock: + failures = service._http_bridge_account_timeout_failures.setdefault(account_id, []) + failures[:] = [ + timestamp for timestamp in failures if now - timestamp < _HTTP_BRIDGE_ACCOUNT_TIMEOUT_WINDOW_SECONDS + ] + failures.append(now) + if len(failures) < _HTTP_BRIDGE_ACCOUNT_TIMEOUT_EJECTION_THRESHOLD: + return + # Start a fresh evidence window after applying one health penalty. A + # continuously failing account should be re-evaluated by normal + # health-tier logic, not receive an unbounded error-count increase from + # every pending request on one broken socket. + failures.clear() + + try: + # Health-tier draining starts at two transient errors. Apply exactly + # that minimum penalty so one threshold event actually removes the + # account from normal routing without over-counting the incident. + await service._load_balancer.record_errors(session.account, 2) + except Exception: + logger.warning( + "Failed to record repeated HTTP bridge account timeout account_id=%s", + account_id, + exc_info=True, + ) + else: + logger.warning( + "HTTP bridge account temporarily drained after repeated eventless upstream failures " + "account_id=%s detail=%s threshold=%s window_seconds=%.0f", + account_id, + detail, + _HTTP_BRIDGE_ACCOUNT_TIMEOUT_EJECTION_THRESHOLD, + _HTTP_BRIDGE_ACCOUNT_TIMEOUT_WINDOW_SECONDS, + ) + + +async def _update_http_bridge_operation_state( + service: Any, + session: "_HTTPBridgeSession", + request_state: Any, + *, + state: str, + response_id: str | None = None, +) -> None: + """Persist operation outcome without allowing journaling to break streaming.""" + operation_id = getattr(request_state, "operation_id", None) + session_id = getattr(session, "durable_session_id", None) + owner_epoch = getattr(session, "durable_owner_epoch", None) + update_operation = getattr(getattr(service, "_durable_bridge", None), "update_operation", None) + if not operation_id or session_id is None or owner_epoch is None or not callable(update_operation): + return + try: + marked = await update_operation( + operation_id=operation_id, + session_id=session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=owner_epoch, + state=state, + response_id=response_id, + ) + if marked and response_id is not None: + request_state.operation_persisted_response_id = response_id + if not marked: + logger.info( + "HTTP bridge operation outcome owner fence rejected operation_id=%s state=%s", + operation_id, + state, + ) + except Exception: + logger.warning( + "Failed to persist HTTP bridge operation outcome operation_id=%s state=%s", + operation_id, + state, + exc_info=True, + ) + + +def _http_bridge_operation_state_for_event(event_type: str | None) -> str | None: + return { + "response.created": "acknowledged", + "response.completed": "completed", + "response.incomplete": "incomplete", + "response.failed": "failed", + "error": "failed", + }.get(event_type) + + +async def _persist_http_bridge_operation_event( + service: Any, + session: "_HTTPBridgeSession", + request_state: Any, + event_block: str, + *, + terminal: bool = False, + terminal_state: str | None = None, + terminal_event_queue: Any | None = None, + terminal_delivery_scope: _HTTPBridgeCompletedDeliveryScope | None = None, + terminal_append_barrier: Callable[[], Awaitable[None]] | None = None, + terminal_delivery_barrier: Callable[[], Awaitable[None]] | None = None, +) -> bool: + """Spool one downstream-visible SSE block for reconnect replay. + + Return whether terminal failure handling already queued the block. + """ + operation_id = getattr(request_state, "operation_id", None) + session_id = getattr(session, "durable_session_id", None) + owner_epoch = getattr(session, "durable_owner_epoch", None) + batcher_enqueue = getattr(getattr(service, "_http_bridge_operation_event_batcher", None), "enqueue", None) + append_event = getattr(getattr(service, "_durable_bridge", None), "append_operation_event", None) + if not operation_id or session_id is None or owner_epoch is None: + return False + try: + batcher = getattr(service, "_http_bridge_operation_event_batcher", None) + append_terminal_batch = getattr(batcher, "append_terminal_event", None) + if terminal and terminal_state is not None and callable(append_terminal_batch): + instance_id = _service_get_settings().http_responses_session_bridge_instance_id + expected_response_ids = tuple( + dict.fromkeys( + response_identity + for response_identity in ( + request_state.response_id, + getattr(request_state, "operation_persisted_response_id", None), + request_state.replay_downstream_response_id, + ) + if response_identity is not None + ) + ) + expected_response_id = expected_response_ids[0] if expected_response_ids else None + alternate_expected_response_id = expected_response_ids[1] if len(expected_response_ids) > 1 else None + response_id = _websocket_downstream_response_id(request_state) + + async def enqueue_terminal_delivery() -> bool: + if terminal_event_queue is None: + return False + await terminal_event_queue.put(event_block) + await terminal_event_queue.put(None) + if terminal_delivery_scope is not None: + async with session.pending_lock: + terminal_delivery_scope.terminal_enqueued = True + return True + + async def enqueue_terminal_delivery_deferring_cancellation() -> tuple[bool, asyncio.CancelledError | None]: + delivery_task = asyncio.create_task( + enqueue_terminal_delivery(), + name=f"http-bridge-terminal-delivery-{operation_id}", + ) + return await _await_task_deferring_cancellation(delivery_task) + + append_task = asyncio.create_task( + append_terminal_batch( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + event_text=event_block, + max_bytes=int( + getattr( + _service_get_settings(), + "http_responses_session_bridge_operation_event_spool_max_bytes", + 2 * 1024 * 1024, + ) + ), + state=terminal_state, + expected_recovery_dispatch_count=request_state.operation_attempt_generation, + response_id=response_id, + ), + name=f"http-bridge-terminal-append-{operation_id}", + ) + append_result, deferred_cancellation = await _await_task_deferring_cancellation(append_task) + if terminal_append_barrier is not None: + await terminal_append_barrier() + persisted = bool(append_result) + if not persisted: + logger.info("HTTP bridge terminal event spool became incomplete operation_id=%s", operation_id) + settlement_required = bool(getattr(append_result, "settlement_required", False)) + terminal_enqueued = False + if settlement_required: + terminal_enqueued, delivery_cancellation = await enqueue_terminal_delivery_deferring_cancellation() + deferred_cancellation = deferred_cancellation or delivery_cancellation + if terminal_delivery_barrier is not None: + if not terminal_enqueued: + terminal_enqueued, delivery_cancellation = await enqueue_terminal_delivery_deferring_cancellation() + deferred_cancellation = deferred_cancellation or delivery_cancellation + await terminal_delivery_barrier() + if settlement_required: + settle_terminal_batch = getattr(batcher, "settle_terminal_event", None) + + async def settle_terminal_append_failure() -> None: + if callable(settle_terminal_batch): + await settle_terminal_batch( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + state=terminal_state, + expected_response_id=expected_response_id, + expected_recovery_dispatch_count=request_state.operation_attempt_generation, + alternate_expected_response_id=alternate_expected_response_id, + response_id=response_id, + ) + else: + await _update_http_bridge_operation_state( + service, + session, + request_state, + state=terminal_state, + response_id=response_id, + ) + + settlement_task = asyncio.create_task( + settle_terminal_append_failure(), + name=f"http-bridge-terminal-settlement-{operation_id}", + ) + _, settlement_cancellation = await _await_task_deferring_cancellation(settlement_task) + deferred_cancellation = deferred_cancellation or settlement_cancellation + if deferred_cancellation is not None: + if not terminal_enqueued: + await enqueue_terminal_delivery_deferring_cancellation() + raise deferred_cancellation + return terminal_enqueued + if callable(batcher_enqueue): + await batcher_enqueue( + operation_id=operation_id, + session_id=session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=owner_epoch, + event_text=event_block, + terminal=terminal, + ) + return False + if not callable(append_event): + return False + persisted = await append_event( + operation_id=operation_id, + session_id=session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=owner_epoch, + event_text=event_block, + max_bytes=int( + getattr( + _service_get_settings(), + "http_responses_session_bridge_operation_event_spool_max_bytes", + 2 * 1024 * 1024, + ) + ), + ) + if not persisted: + logger.info("HTTP bridge operation event spool became incomplete operation_id=%s", operation_id) + if terminal and terminal_state is not None: + await _update_http_bridge_operation_state( + service, + session, + request_state, + state=terminal_state, + response_id=_websocket_downstream_response_id(request_state), + ) + return False + except Exception: + # The upstream result is still delivered. A reconnect can only replay + # when every event was durably persisted, so never fail a live stream + # because the optional spool is unavailable. + logger.warning("Failed to persist HTTP bridge operation event operation_id=%s", operation_id, exc_info=True) + return False async def _wait_for_http_bridge_recovery_settlement_retry( @@ -366,8 +645,6 @@ def _record_http_bridge_tool_call_lifecycle( ) -> None: if event_type not in {"response.output_item.added", "response.output_item.done"}: return - if payload is None or not collect_public_output_item_event(payload, request_state.response_output_items): - request_state.response_output_items_invalid = True item = payload.get("item") if isinstance(payload, dict) else None if not isinstance(item, dict): request_state.tool_call_manifest_invalid = True @@ -401,23 +678,6 @@ def _record_http_bridge_tool_call_lifecycle( target[call_id] = item_type -def _response_transition_payload( - request_state: _WebSocketRequestState, - payload: dict[str, JsonValue] | None, -) -> dict[str, JsonValue] | None: - if request_state.response_output_items_invalid or not isinstance(payload, dict): - return None - response = payload.get("response") - if not isinstance(response, dict): - return None - merged_payload = dict(payload) - merged_payload["response"] = merge_public_response_output_items( - response, - request_state.response_output_items, - ) - return merged_payload - - def _response_completed_tool_call_types(payload: dict[str, JsonValue] | None) -> dict[str, str] | None: response = payload.get("response") if isinstance(payload, dict) else None output = response.get("output") if isinstance(response, dict) else None @@ -458,32 +718,6 @@ def _durable_pending_tool_call_manifest( return dict(request_state.pending_tool_call_types) -def _live_pending_tool_call_manifest_is_invalid( - request_state: _WebSocketRequestState, - payload: dict[str, JsonValue] | None, -) -> bool: - """Keep explicitly invalid live manifests distinct from valid empty ones. - - A live session may safely synthesize an interrupted output from a completed - ``output_item.done`` event even when the optional ``added`` event was not - observed. Durable replay needs the stricter added/done equality above, - while live replay must still reject malformed/unsupported events and any - terminal tool-call set that contradicts the observed completed calls. - """ - - terminal_calls = _response_completed_tool_call_types(payload) - added_calls_settled = all( - request_state.pending_tool_call_types.get(call_id) == call_type - for call_id, call_type in request_state.added_tool_call_types.items() - ) - return bool( - request_state.tool_call_manifest_invalid - or not added_calls_settled - or terminal_calls is None - or (terminal_calls and terminal_calls != request_state.pending_tool_call_types) - ) - - _SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE = "security_work_authorization_required" _SECURITY_WORK_RETRY_MESSAGE = ( "Upstream flagged this request as possible cybersecurity work. " @@ -627,17 +861,6 @@ def _archive_http_bridge_upstream_message( reset_request_id(token) -def _http_bridge_idle_age_bucket(session: "_HTTPBridgeSession") -> str: - idle_seconds = max(0.0, _service_time().monotonic() - session.last_used_at) - if idle_seconds < 5.0: - return "under_5s" - if idle_seconds < 60.0: - return "5s_to_60s" - if idle_seconds < 300.0: - return "60s_to_5m" - return "5m_or_more" - - async def _http_bridge_receive_timeout_with_eventless_deadline( session: "_HTTPBridgeSession", receive_timeout: _WebSocketReceiveTimeout | None, @@ -704,15 +927,15 @@ async def _cancel_http_bridge_reader_child( async def _clear_durable_http_bridge_response_anchor( service: Any, session: "_HTTPBridgeSession", -) -> DurableBridgeLookup | None: - """Invalidate and return the durable row for an anchor proved unusable. +) -> None: + """Invalidate a durable proxy-injected anchor that proved eventless. Runs while ``session`` still owns the durable row (before retirement releases the lease), so the fenced write lands under the session's own owner epoch instead of silently losing the fence to a released owner. """ if session.durable_session_id is None or session.durable_owner_epoch is None: - return None + return try: lookup = await service._durable_bridge.clear_live_session_response_anchor( session_id=session.durable_session_id, @@ -721,14 +944,14 @@ async def _clear_durable_http_bridge_response_anchor( ) except Exception: logger.warning("Failed to clear durable HTTP bridge response anchor after stuck timeout", exc_info=True) - return None + return if lookup is None or lookup.owner_epoch != session.durable_owner_epoch or lookup.latest_response_id is not None: # None means the durable row is gone entirely (e.g. purged); an # epoch or anchor mismatch means a newer owner already claimed the # session before this fenced write executed. Either way, the anchor # was never actually cleared, so do not report an invalidation that # did not happen. - return None + return _log_http_bridge_event( "durable_anchor_invalidated", session.key, @@ -738,7 +961,6 @@ async def _clear_durable_http_bridge_response_anchor( cache_key_family=session.key.affinity_kind, model_class=_extract_model_class(session.request_model) if session.request_model else None, ) - return lookup async def _abandon_durable_http_bridge_continuity( @@ -790,24 +1012,6 @@ async def _abandon_durable_http_bridge_continuity( class _HTTPBridgeUpstreamEventsMixin: - async def _restore_unsent_rowless_retry_setup( - self: Any, - session: "_HTTPBridgeSession", - request_state: _WebSocketRequestState, - *, - detach: bool, - ) -> None: - """Restore a claimed-but-unsent authority and optionally detach its retry.""" - - await self._rollback_rowless_preflight_setup_failure_if_unbound(request_state) - if not detach: - return - async with session.pending_lock: - if request_state in session.pending_requests: - session.pending_requests.remove(request_state) - if _http_bridge_request_counts_against_queue(request_state): - session.queued_request_count = max(0, session.queued_request_count - 1) - async def _fail_http_bridge_reader_and_maybe_retire( self: Any, session: "_HTTPBridgeSession", @@ -820,9 +1024,8 @@ async def _fail_http_bridge_reader_and_maybe_retire( upstream_close_code: int | None = None, response_events_seen: int | None = None, transport_classification: str | None = None, - idle_transport_retire: bool = False, - retry_action: str | None = None, - circuit_action: str | None = None, + retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None, + account_neutral_transport_drop: bool = False, ) -> bool: session.closed = True async with session.pending_lock: @@ -837,6 +1040,13 @@ async def _fail_http_bridge_reader_and_maybe_retire( default=0, ) pending_request_states = list(session.pending_requests) + if retry_circuit_attempt_selection is None: + retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + pending_request_states + ) + retry_circuit_attempt_kwargs = { + "retry_circuit_attempt_selection": retry_circuit_attempt_selection, + } # The #1534 wedge shape: a reattached stream that streamed response # events whose ``response.created`` was never assigned. The eventless # watchdog and the durable-anchor clear both key on @@ -854,72 +1064,70 @@ async def _fail_http_bridge_reader_and_maybe_retire( if observed_close_code is not None else None ) - classify_as_idle_transport_retire = bool( - idle_transport_retire and failed_pending_count == 0 and session.admission_waiter_count == 0 + _log_http_bridge_event( + "reader_failure", + session.key, + account_id=session.account.id, + model=session.request_model, + pending_count=failed_pending_count, + detail=error_code, + error_message=_truncate_identifier(error_message), + upstream_close_code=observed_close_code, + response_events_seen=observed_response_events, + transport_classification=transport_classification + or ( + f"websocket_close_{close_classification}" + if close_classification is not None + else "websocket_transport_error" + ), + cache_key_family=session.key.affinity_kind, + model_class=_extract_model_class(session.request_model) if session.request_model else None, ) - effective_retire_detail = "idle_transport_retire" if classify_as_idle_transport_retire else retire_detail - retry_circuit_detail = None - if close_classification == "clean": - retry_circuit_detail = "clean_close" - elif observed_response_events == 0: - retry_circuit_detail = next( - ( - detail - for detail in (effective_retire_detail, error_code) - if detail in {"stream_incomplete", "stream_idle_timeout", "upstream_keepalive_timeout"} - ), - None, - ) - if classify_as_idle_transport_retire: - _log_http_bridge_event( - "idle_transport_retire", - session.key, - account_id=session.account.id, - model=session.request_model, - pending_count=failed_pending_count, - admission_waiter_count=session.admission_waiter_count, - idle_age_bucket=_http_bridge_idle_age_bucket(session), - retry_action="not_attempted", - circuit_action="not_recorded", - cache_key_family=session.key.affinity_kind, - model_class=_extract_model_class(session.request_model) if session.request_model else None, - ) - else: - _log_http_bridge_event( - "reader_failure", - session.key, - account_id=session.account.id, - model=session.request_model, - pending_count=failed_pending_count, - detail=error_code, - error_message=_truncate_identifier(error_message), - upstream_close_code=observed_close_code, - response_events_seen=observed_response_events, - transport_classification=transport_classification - or ( - f"websocket_close_{close_classification}" - if close_classification is not None - else "websocket_transport_error" - ), - admission_waiter_count=session.admission_waiter_count, - retry_action=retry_action, - circuit_action=circuit_action, - cache_key_family=session.key.affinity_kind, - model_class=_extract_model_class(session.request_model) if session.request_model else None, + # Draining-only requests no longer count against the queue, but their + # event-batcher contexts still belong to the disconnected operation + # and must be discarded just like ordinary pending requests. + operation_states: list[Any] = [ + request_state for request_state in pending_request_states if getattr(request_state, "operation_id", None) + ] + # Remove the disconnected attempt's in-memory spool before publishing + # UNKNOWN/ACKNOWLEDGED state. A same-replica reconnect may reclaim the + # operation as soon as that state is visible; discarding afterward + # could then delete the replacement attempt's events. + discard_operation = getattr( + getattr(self, "_http_bridge_operation_event_batcher", None), + "discard_operation", + None, + ) + if callable(discard_operation): + for request_state in operation_states: + operation_id = getattr(request_state, "operation_id", None) + if operation_id: + await discard_operation(operation_id=operation_id) + for request_state in operation_states: + # A shared websocket can carry several logical response.create + # requests. Classify each operation from its own event count; + # using the session-wide maximum would mark an eventless + # sibling as safely retryable after another request streamed. + operation_state = "unknown" if getattr(request_state, "response_event_count", 0) == 0 else "acknowledged" + await _update_http_bridge_operation_state( + self, + session, + request_state, + state=operation_state, ) - if force_retire and effective_retire_detail: + if force_retire and retire_detail: _log_http_bridge_event( - effective_retire_detail, + retire_detail, session.key, account_id=session.account.id, model=session.request_model, pending_count=failed_pending_count, - detail=effective_retire_detail, + detail=retire_detail, cache_key_family=session.key.affinity_kind, model_class=_extract_model_class(session.request_model) if session.request_model else None, ) try: - await self._fail_pending_websocket_requests( + reservations_settled = await self._fail_pending_websocket_requests( account=session.account, account_id_value=session.account.id, pending_requests=session.pending_requests, @@ -930,43 +1138,69 @@ async def _fail_http_bridge_reader_and_maybe_retire( response_create_gate=session.response_create_gate, penalize_account=penalize_account, ) + if ( + failed_pending_count > 0 + and reservations_settled is not False + and observed_response_events == 0 + and ( + retire_detail == _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL + or account_neutral_transport_drop + ) + ): + # Only penalize the account after pending-request cleanup has + # settled its API-key reservations. A failed release must not + # be hidden behind an already-recorded timeout health signal. + # Account-neutral abrupt drops share the same windowed signal: + # one drop is infrastructure noise, but repeated eventless + # drops on the same account remain evidence of an account-side + # fault and must still drain it (issue #1754). + await _record_http_bridge_account_timeout_signal( + self, + session, + detail=( + "eventless_transport_drop" + if account_neutral_transport_drop + else _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL + ), + ) finally: - consecutive_failures = None - retry_circuit_recorded = False - if failed_pending_count > 0 and retry_circuit_detail is not None: - try: - consecutive_failures = await self._record_http_bridge_retry_circuit_failure( - session, - detail=retry_circuit_detail, - ) - except Exception: - logger.warning( - "Failed to record HTTP bridge retry circuit before retirement", - exc_info=True, - ) - retry_circuit_recorded = consecutive_failures is not None poison_detail: str | None = None if session.admission_waiter_count > 0 and not force_retire: - poison_candidate_detail = _http_bridge_anchor_poison_detail(retry_circuit_detail) - if ( - poison_candidate_detail is not None - and observed_response_events == 0 - and consecutive_failures is not None - and consecutive_failures - >= _service_get_settings().http_responses_session_bridge_anchor_poison_failure_threshold - ): - poison_detail = poison_candidate_detail - if poison_detail is not None: - durable_cleared = await _abandon_durable_http_bridge_continuity( - self, + retry_circuit_detail = None + if close_classification == "clean": + retry_circuit_detail = "clean_close" + elif observed_response_events == 0: + retry_circuit_detail = next( + ( + detail + for detail in (retire_detail, error_code) + if detail in {"stream_incomplete", "stream_idle_timeout", "upstream_keepalive_timeout"} + ), + None, + ) + if failed_pending_count > 0 and retry_circuit_detail is not None: + consecutive_failures = await self._record_http_bridge_retry_circuit_failure_for_attempt_selection( session, - detail=poison_detail, + detail=retry_circuit_detail, + selection=retry_circuit_attempt_selection, ) + poison_candidate_detail = _http_bridge_anchor_poison_detail(retry_circuit_detail) + if ( + poison_candidate_detail is not None + and observed_response_events == 0 + and consecutive_failures is not None + and consecutive_failures + >= _service_get_settings().http_responses_session_bridge_anchor_poison_failure_threshold + ): + poison_detail = poison_candidate_detail + if poison_detail is not None: + durable_cleared = await _abandon_durable_http_bridge_continuity(self, session, detail=poison_detail) if durable_cleared: await self._retire_stale_pending_http_bridge_session( session, detail=poison_detail, response_events_seen=observed_response_events, + **retry_circuit_attempt_kwargs, ) force_retire = True else: @@ -987,7 +1221,7 @@ async def _fail_http_bridge_reader_and_maybe_retire( account_id=session.account.id, model=session.request_model, pending_count=session.admission_waiter_count, - detail=effective_retire_detail or error_code, + detail=retire_detail or error_code, cache_key_family=session.key.affinity_kind, model_class=_extract_model_class(session.request_model) if session.request_model else None, ) @@ -998,14 +1232,22 @@ async def _fail_http_bridge_reader_and_maybe_retire( detail=error_code, retry_circuit_detail="clean_close", response_events_seen=observed_response_events, - **({"retry_circuit_already_recorded": True} if retry_circuit_recorded else {}), + retired_request_count=failed_pending_count, + **retry_circuit_attempt_kwargs, ) else: await self._retire_stale_pending_http_bridge_session( session, - detail=effective_retire_detail or error_code, + detail=retire_detail or error_code, response_events_seen=observed_response_events, - **({"retry_circuit_already_recorded": True} if retry_circuit_recorded else {}), + # ``_fail_pending_websocket_requests`` has already + # claimed and drained these states. Carry the count + # sampled under ``pending_lock`` across that ownership + # transfer so normal reader failures still consume one + # strike. The deferred/poison branch records its own + # strike above and intentionally does not pass it. + retired_request_count=failed_pending_count, + **retry_circuit_attempt_kwargs, ) return force_retire or session.admission_waiter_count == 0 @@ -1017,8 +1259,10 @@ async def _relay_http_bridge_upstream_messages( relay_upstream = session.upstream receive_task: asyncio.Task[UpstreamWebSocketMessage] | None = None wakeup_task: asyncio.Task[bool] | None = None + reader_failure_retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None try: while True: + reader_failure_retry_circuit_attempt_selection = None # Clear before taking the deadline snapshot. A send before the # clear is represented by its timestamp; a send after it leaves # the event set and wakes the persistent receive wait below. @@ -1106,6 +1350,12 @@ async def _relay_http_bridge_upstream_messages( ] if not expired_request_states: continue + expired_retry_circuit_attempt_selection = ( + _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + expired_request_states + ) + ) + reader_failure_retry_circuit_attempt_selection = expired_retry_circuit_attempt_selection pending_count = len(session.pending_requests) # A delta-only request has no other way to # convey prior context once its anchor is @@ -1163,6 +1413,7 @@ async def _relay_http_bridge_upstream_messages( penalize_account=False, retire_detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, force_retire=True, + retry_circuit_attempt_selection=expired_retry_circuit_attempt_selection, ) break # A successfully cancelled receive cannot deliver @@ -1208,9 +1459,17 @@ async def _relay_http_bridge_upstream_messages( penalize_account=False, retire_detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, force_retire=True, + retry_circuit_attempt_selection=expired_retry_circuit_attempt_selection, ) break + async with session.pending_lock: + retry_circuit_attempt_selection = ( + _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + tuple(session.pending_requests) + ) + ) + reader_failure_retry_circuit_attempt_selection = retry_circuit_attempt_selection if receive_task is not None: receive_cancelled = await _cancel_http_bridge_reader_child( receive_task, @@ -1228,6 +1487,7 @@ async def _relay_http_bridge_upstream_messages( session, error_code=receive_timeout.error_code, error_message=receive_timeout.error_message, + retry_circuit_attempt_selection=retry_circuit_attempt_selection, ) break @@ -1239,6 +1499,7 @@ async def _relay_http_bridge_upstream_messages( publish_live_usage( parse_rate_limit_event_text(message.text), account_id=session.account.id, + chatgpt_account_id=session.account.chatgpt_account_id, ) await self._process_http_bridge_upstream_text(session, message.text) if await self._retire_http_bridge_after_drain_if_ready(session): @@ -1247,41 +1508,58 @@ async def _relay_http_bridge_upstream_messages( async with session.pending_lock: archive_request_state = session.pending_requests[0] if len(session.pending_requests) == 1 else None - pending_count = sum( - 1 - for request_state in session.pending_requests - if _http_bridge_request_counts_against_queue(request_state) - ) response_events_seen = max( (request_state.response_event_count for request_state in session.pending_requests), default=0, ) + # Buffered reasoning preludes are suppressed from + # response_event_count on purpose, but they are still + # application-layer output: a drop after one is not an + # eventless drop for account-health purposes. + upstream_output_observed = any( + getattr(request_state, "upstream_model_output_seen", False) + for request_state in session.pending_requests + ) + reader_failure_retry_circuit_attempt_selection = ( + _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + tuple(session.pending_requests) + ) + ) _archive_http_bridge_upstream_message(session, message, archive_request_state) session.last_upstream_close_generation += 1 session.last_upstream_close_code = message.close_code retried = False - ambiguous_receive_failure = message.kind == "error" and message.error_code is None # Account-neutral transport failures do not prove that the # upstream rejected response.create. The request may still be # executing, so replay could duplicate work, billing, or tool # side effects. Clean closes remain eligible for the bounded - # pre-created retry circuit maintained by the session. An - # untyped receive error is also ambiguous: the send primitive - # already accepted response.create, so it must never enter the - # reconnect/resend path even when no response event arrived. + # pre-created retry circuit maintained by the session. account_neutral = is_account_neutral_websocket_error_code(message.error_code) - if not account_neutral and not ambiguous_receive_failure: + if not account_neutral: retried = await self._retry_http_bridge_precreated_request(session) if retried: continue - idle_transport_retire = bool( - ambiguous_receive_failure and pending_count == 0 and session.admission_waiter_count == 0 - ) close_classification = ( _classify_upstream_close(message.close_code, response_events_seen=response_events_seen) if message.close_code is not None else None ) + # An abrupt drop with no close frame and no response events is + # weaker account-health evidence than a graceful pre-created + # close, which is already exempted below. Keep the individual + # drop account-neutral; repeated eventless drops still feed + # the windowed account drain signal inside the failure path. + # Only terminal transport messages qualify: a protocol-invalid + # binary frame also carries no close code but did not end the + # socket, so it keeps the existing penalty semantics. + account_neutral_transport_drop = ( + message.kind in ("close", "error") + and not account_neutral + and not upstream_output_observed + and _is_account_neutral_transport_drop( + message.close_code, response_events_seen=response_events_seen + ) + ) async with session.lifecycle_lock: if ( session.liveness_settlement_owner == "send" @@ -1295,11 +1573,7 @@ async def _relay_http_bridge_upstream_messages( await self._fail_http_bridge_reader_and_maybe_retire( session, error_code=message.error_code or "stream_incomplete", - error_message=( - "Upstream websocket receive failed before response.completed" - if ambiguous_receive_failure - else _upstream_websocket_disconnect_message(message) - ), + error_message=_upstream_websocket_disconnect_message(message), upstream_close_code=message.close_code, response_events_seen=response_events_seen, transport_classification=( @@ -1307,18 +1581,13 @@ async def _relay_http_bridge_upstream_messages( if close_classification is not None else "websocket_transport_error" ), + retry_circuit_attempt_selection=reader_failure_retry_circuit_attempt_selection, penalize_account=( - not account_neutral and not (message.kind == "close" and close_classification == "clean") - ), - idle_transport_retire=idle_transport_retire, - retry_action=("suppressed_ambiguous_accept" if ambiguous_receive_failure else None), - circuit_action=( - "record_stream_incomplete" - if ambiguous_receive_failure and pending_count > 0 and session.key.strength == "hard" - else "not_recorded" - if ambiguous_receive_failure - else None + not account_neutral + and not account_neutral_transport_drop + and not (message.kind == "close" and close_classification == "clean") ), + account_neutral_transport_drop=account_neutral_transport_drop, **( # An admission waiter must not inherit a socket whose # heartbeat already proved it dead. Other failures @@ -1332,6 +1601,17 @@ async def _relay_http_bridge_upstream_messages( except asyncio.CancelledError: raise except Exception as exc: + if reader_failure_retry_circuit_attempt_selection is None: + # A receive/processing exception can jump here before the + # ordinary timeout or close branches publish their snapshot. + # Capture before waiting for lifecycle ownership so a + # concurrent recovery cannot replace the failed physical send. + async with session.pending_lock: + reader_failure_retry_circuit_attempt_selection = ( + _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + tuple(session.pending_requests) + ) + ) logger.warning( "HTTP bridge upstream reader crashed account_id=%s bridge_kind=%s", session.account.id, @@ -1356,6 +1636,7 @@ async def _relay_http_bridge_upstream_messages( else "HTTP bridge upstream reader crashed before response.completed" ), penalize_account=not account_neutral, + retry_circuit_attempt_selection=reader_failure_retry_circuit_attempt_selection, # Preserve ordinary crash handoff behavior, but never hand # a heartbeat-expired socket to an admission waiter. **({"force_retire": True} if error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE else {}), @@ -1381,8 +1662,8 @@ async def _process_http_bridge_upstream_text( ) -> None: event_block = f"data: {text}\n\n" payload = parse_sse_data_json(event_block) - event = parse_sse_event_payload(payload) - event_type = _event_type_from_payload(event, payload) + event_type = classify_event_type(payload) + event = parse_sse_event_payload(payload) if event_type in _LIFECYCLE_EVENT_TYPES else None completed_delivery_scope = _HTTPBridgeCompletedDeliveryScope() if event_type == "response.completed" else None claimed_terminal_request_states: list[_WebSocketRequestState] = [] try: @@ -1550,6 +1831,12 @@ async def _process_parsed_http_bridge_upstream_event( pending_request_count = len(session.pending_requests) if matched_request_state is not None: + # The deferred reasoning prelude intentionally skips ordinary + # response-event accounting below, but it still proves that the + # physical response.create received an upstream response. Publish + # that attempt transition before any later recovery await can + # classify the send as eventless. + _mark_response_create_attempt_observed(matched_request_state, event_type) now = _service_time().monotonic() if matched_request_state.latency_first_upstream_event_ms is None: matched_request_state.latency_first_upstream_event_ms = int( @@ -1734,39 +2021,177 @@ async def _process_parsed_http_bridge_upstream_event( if is_missing_tool_output_event else "stream_incomplete" ) - try: - for grouped_request_state in grouped_previous_response_request_states: - grouped_request_state.error_http_status_override = 502 + grouped_terminal_events = [] + for grouped_request_state in grouped_previous_response_request_states: + grouped_request_state.error_http_status_override = 502 + ( + _grouped_downstream_text, + grouped_event_block, + grouped_event, + grouped_payload, + grouped_event_type, + ) = _build_stream_incomplete_terminal_event_for_request( + grouped_request_state, + reason=grouped_error_reason, + ) + grouped_operation_state = _http_bridge_operation_state_for_event(grouped_event_type) + grouped_terminal_events.append( ( - _grouped_downstream_text, + grouped_request_state, grouped_event_block, grouped_event, grouped_payload, grouped_event_type, - ) = _build_stream_incomplete_terminal_event_for_request( + grouped_operation_state, + ) + ) + + append_terminal_batch = getattr( + getattr(self, "_http_bridge_operation_event_batcher", None), + "append_terminal_event", + None, + ) + append_participants = { + id(grouped_request_state) + for grouped_request_state, *_rest in grouped_terminal_events + if grouped_request_state.operation_id + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + and callable(append_terminal_batch) + } + append_ready = asyncio.Event() + append_lock = asyncio.Lock() + append_arrivals = 0 + if not append_participants: + append_ready.set() + + async def await_all_grouped_appends() -> None: + nonlocal append_arrivals + async with append_lock: + append_arrivals += 1 + if append_arrivals == len(append_participants): + append_ready.set() + await append_ready.wait() + + delivery_ready = asyncio.Event() + delivery_lock = asyncio.Lock() + delivery_arrivals = 0 + + async def await_all_grouped_deliveries() -> None: + nonlocal delivery_arrivals + async with delivery_lock: + delivery_arrivals += 1 + if delivery_arrivals == len(grouped_terminal_events): + delivery_ready.set() + await delivery_ready.wait() + + async def persist_one_grouped_terminal_event( + grouped_terminal_event: tuple[Any, str, OpenAIEvent | None, Any, str | None, str | None], + ) -> None: + ( + grouped_request_state, + grouped_event_block, + _grouped_event, + _grouped_payload, + _grouped_event_type, + grouped_operation_state, + ) = grouped_terminal_event + if id(grouped_request_state) in append_participants: + await _persist_http_bridge_operation_event( + self, + session, grouped_request_state, - reason=grouped_error_reason, + grouped_event_block, + terminal=True, + terminal_state=grouped_operation_state, + terminal_event_queue=grouped_request_state.event_queue, + terminal_append_barrier=await_all_grouped_appends, + terminal_delivery_barrier=await_all_grouped_deliveries, ) + else: + await append_ready.wait() if grouped_request_state.event_queue is not None: await grouped_request_state.event_queue.put(grouped_event_block) await grouped_request_state.event_queue.put(None) - await self._finalize_websocket_request_state( + await await_all_grouped_deliveries() + await _persist_http_bridge_operation_event( + self, + session, + grouped_request_state, + grouped_event_block, + terminal=True, + terminal_state=grouped_operation_state, + ) + if grouped_operation_state is not None and grouped_operation_state != "failed": + await _update_http_bridge_operation_state( + self, + session, grouped_request_state, - account=session.account, - account_id_value=session.account.id, - event=grouped_event, - event_type=grouped_event_type, - payload=grouped_payload, - api_key=grouped_request_state.api_key, - upstream_control=session.upstream_control, - response_create_gate=session.response_create_gate, + state=grouped_operation_state, + response_id=_websocket_downstream_response_id(grouped_request_state), ) - finally: - # Grouped terminal errors settle detached/abandoned requests - # (event_queue is None) with no downstream stream finalizer - # left to run, so release the now-idle session's account - # stream lease here just like the single terminal path below. - await self._maybe_release_idle_http_bridge_session_lease(session) + + async def persist_grouped_terminal_events() -> Exception | None: + first_error: Exception | None = None + persistence_results = await asyncio.gather( + *(persist_one_grouped_terminal_event(item) for item in grouped_terminal_events), + return_exceptions=True, + ) + for persistence_result in persistence_results: + if isinstance(persistence_result, Exception) and first_error is None: + first_error = persistence_result + try: + for ( + grouped_request_state, + _grouped_event_block, + grouped_event, + grouped_payload, + grouped_event_type, + _grouped_operation_state, + ) in grouped_terminal_events: + try: + await self._finalize_websocket_request_state( + grouped_request_state, + account=session.account, + account_id_value=session.account.id, + event=grouped_event, + event_type=grouped_event_type, + payload=grouped_payload, + api_key=grouped_request_state.api_key, + upstream_control=session.upstream_control, + response_create_gate=session.response_create_gate, + ) + except Exception as exc: + if first_error is None: + first_error = exc + except Exception as exc: + if first_error is None: + first_error = exc + try: + # Grouped terminal errors settle detached/abandoned requests + # (event_queue is None) with no downstream stream finalizer + # left to run, so release the now-idle session's account + # stream lease here just like the single terminal path below. + await self._maybe_release_idle_http_bridge_session_lease(session) + except Exception as exc: + if first_error is None: + first_error = exc + return first_error + + grouped_settlement_task = asyncio.create_task( + persist_grouped_terminal_events(), + name=f"http-bridge-grouped-terminal-settlement-{session.durable_session_id}", + ) + grouped_error, grouped_cancellation = await _await_task_deferring_cancellation(grouped_settlement_task) + if grouped_cancellation is not None: + if grouped_error is not None: + logger.warning( + "Grouped HTTP bridge terminal finalization failed while preserving cancellation error=%r", + grouped_error, + ) + raise grouped_cancellation + if grouped_error is not None: + raise grouped_error return if len(grouped_previous_response_request_states) == 1 and terminal_request_state is None: @@ -1812,6 +2237,7 @@ async def _process_parsed_http_bridge_upstream_event( surface="http_bridge", ) + continuity_persistence_failed_after_ack = False if ( event_type == "response.completed" and terminal_request_state is not None @@ -1850,260 +2276,19 @@ async def _process_parsed_http_bridge_upstream_event( event_block = f"data: {rewritten_text}\n\n" if status_request_state is not None and is_previous_response_not_found_event: - capture_intent = status_request_state.rowless_recovery_capture_intent - if ( - capture_intent is not None - and response_id is None - and not has_other_pending_requests - and status_request_state.response_event_count == 0 - and status_request_state.previous_response_id is not None - and isinstance(status_request_state.request_text, str) - ): - projected_wire_text = rowless_projected_actual_wire_text( - status_request_state.request_text, - capture_intent.facts.projected_input, - ) - capture_facts = replace( - capture_intent.facts, - actual_wire_fingerprint=rowless_actual_wire_fingerprint(projected_wire_text), - ) - authority = None - try: - async with SessionLocal() as rowless_session: - repository = RowlessRecoveryRepository(rowless_session) - if capture_intent.automatic_live_recovery: - authority = await repository.capture_and_claim_automatic_preflight( - api_key_scope=capture_intent.api_key_scope, - session_key_kind=capture_intent.session_key_kind, - strong_session_hash=capture_intent.strong_session_hash, - stale_anchor_hash=durable_bridge_hash(status_request_state.previous_response_id), - selected_account_intent=session.account.id, - task_identity=capture_intent.task_identity, - session_identity=capture_intent.session_identity, - task_authority_digest=capture_intent.task_authority_digest, - facts=capture_facts, - request_id=status_request_state.request_id, - wire_request_fingerprint=capture_facts.actual_wire_fingerprint, - ) - # Persist the cleanup identity before session exit; - # cancellation during __aexit__ must not strand an - # unsent automatic claim in UNKNOWN. - status_request_state.rowless_recovery_authority_id = authority.id - status_request_state.rowless_recovery_generation = authority.generation - status_request_state.rowless_recovery_wire_fingerprint = authority.actual_wire_fingerprint - else: - authority = await repository.capture( - api_key_scope=capture_intent.api_key_scope, - session_key_kind=capture_intent.session_key_kind, - strong_session_hash=capture_intent.strong_session_hash, - stale_anchor_hash=durable_bridge_hash(status_request_state.previous_response_id), - selected_account_intent=session.account.id, - task_identity=capture_intent.task_identity, - session_identity=capture_intent.session_identity, - task_authority_digest=capture_intent.task_authority_digest, - facts=capture_facts, - ) - except asyncio.CancelledError: - if capture_intent.automatic_live_recovery and authority is not None: - rollback_task = asyncio.create_task( - self._rollback_rowless_preflight_setup_failure_if_unbound(status_request_state) - ) - await _await_task_deferring_cancellation(rollback_task) - raise - except RowlessRecoveryStateError: - if capture_intent.automatic_live_recovery: - status_request_state.error_http_status_override = 400 - payload = cast( - dict[str, JsonValue], - dict( - response_failed_event( - "rowless_automatic_recovery_proof_rejected", - "The live Codex turn could not prove a physically-unsent semantic rebase.", - error_type="invalid_request_error", - response_id=status_request_state.request_id, - ) - ), - ) - else: - logger.warning("Failed to persist legacy rowless recovery authority", exc_info=True) - status_request_state.error_http_status_override = 502 - payload = cast( - dict[str, JsonValue], - dict( - response_failed_event( - "bridge_continuity_persistence_failed", - "The semantic-rebase authority could not be persisted; retrying is unsafe.", - response_id=status_request_state.request_id, - ) - ), - ) - except Exception: - if capture_intent.automatic_live_recovery and authority is not None: - rollback_task = asyncio.create_task( - self._rollback_rowless_preflight_setup_failure_if_unbound(status_request_state) - ) - try: - _, rollback_cancellation = await _await_task_deferring_cancellation(rollback_task) - except Exception: - logger.warning( - "Failed to restore unsent rowless authority after persistence failure", - exc_info=True, - ) - else: - if rollback_cancellation is not None: - raise rollback_cancellation - logger.warning("Failed to persist rowless recovery authority", exc_info=True) - status_request_state.error_http_status_override = 502 - payload = cast( - dict[str, JsonValue], - dict( - response_failed_event( - "bridge_continuity_persistence_failed", - "The semantic-rebase authority could not be persisted; retrying is unsafe.", - response_id=status_request_state.request_id, - ) - ), - ) - else: - status_request_state.rowless_recovery_authority_id = authority.id - if capture_intent.automatic_live_recovery: - status_request_state.rowless_recovery_generation = authority.generation - status_request_state.rowless_recovery_task_authority_digest = ( - authority.captured_task_authority_digest - ) - status_request_state.rowless_recovery_wire_fingerprint = authority.actual_wire_fingerprint - status_request_state.fresh_upstream_request_text = projected_wire_text - status_request_state.fresh_upstream_request_is_retry_safe = True - status_request_state.fresh_upstream_request_is_account_neutral = True - status_request_state.fresh_upstream_request_responses_lite_model = ( - status_request_state.responses_lite_model - ) - status_request_state.preferred_account_id = authority.selected_account_intent - status_request_state.input_item_count = authority.captured_input_item_count - status_request_state.input_full_fingerprint = authority.captured_input_fingerprint - status_request_state.rowless_recovery_capture_intent = None - retry_consumer_attached = False - try: - async with session.pending_lock: - if ( - status_request_state.event_queue is not None - and not status_request_state.draining_until_terminal - ): - retry_consumer_attached = True - if status_request_state not in session.pending_requests: - session.pending_requests.appendleft(status_request_state) - session.queued_request_count += 1 - status_request_state.awaiting_response_created = True - status_request_state.response_id = None - retried = retry_consumer_attached and await self._retry_http_bridge_precreated_request( - session, - request_state=status_request_state, - ) - except asyncio.CancelledError: - cleanup_task = asyncio.create_task( - self._restore_unsent_rowless_retry_setup( - session, - status_request_state, - detach=True, - ) - ) - await _await_task_deferring_cancellation(cleanup_task) - raise - except Exception: - cleanup_task = asyncio.create_task( - self._restore_unsent_rowless_retry_setup( - session, - status_request_state, - detach=False, - ) - ) - await _await_task_deferring_cancellation(cleanup_task) - raise - if retried: - _log_http_bridge_event( - "rowless_automatic_live_rebase_submitted", - session.key, - account_id=session.account.id, - model=status_request_state.model, - detail=f"generation={authority.generation}", - cache_key_family=session.key.affinity_kind, - model_class=( - _extract_model_class(status_request_state.model) - if status_request_state.model - else None - ), - owner_check_applied=True, - ) - return - rollback_task = asyncio.create_task( - self._restore_unsent_rowless_retry_setup( - session, - status_request_state, - detach=True, - ) - ) - _, rollback_cancellation = await _await_task_deferring_cancellation(rollback_task) - if rollback_cancellation is not None: - raise rollback_cancellation - status_request_state.error_http_status_override = 502 - payload = cast( - dict[str, JsonValue], - dict( - response_failed_event( - "bridge_continuity_persistence_failed", - "The automatic semantic rebase could not be submitted safely.", - response_id=status_request_state.request_id, - ) - ), - ) - event_block = format_sse_event(payload) - event = parse_sse_event_payload(payload) - event_type = "response.failed" - capture_intent = None - else: - # The failed turn may have created a transient durable - # bridge row. Retire it so an approved retry must bind - # a fresh durable replacement. - session.closed = True - session.upstream_control.reconnect_requested = True - session.upstream_control.retire_after_drain = True - status_request_state.error_http_status_override = 400 - payload = cast( - dict[str, JsonValue], - dict( - response_failed_event( - "previous_response_recovery_authorization_required", - ( - "The saved response anchor no longer has a durable checkpoint. " - "A dashboard administrator must approve a same-turn semantic rebase." - ), - error_type="invalid_request_error", - response_id=status_request_state.request_id, - ) - ), - ) - response = payload.get("response") - if isinstance(response, dict): - error_detail = response.get("error") - if isinstance(error_detail, dict): - error_detail["action"] = "retry_same_turn_after_admin_approval" - event_block = format_sse_event(payload) - event = parse_sse_event_payload(payload) - event_type = "response.failed" - else: - status_request_state.error_http_status_override = 502 - status_request_state.previous_response_not_found_rewritten = ( - response_id is None and not has_other_pending_requests - ) - event, payload, event_type, rewritten_text = _maybe_rewrite_websocket_previous_response_not_found_event( - request_state=status_request_state, - event=event, - payload=payload, - event_type=event_type, - upstream_control=session.upstream_control, - original_text=text, - ) - event_block = f"data: {rewritten_text}\n\n" + status_request_state.error_http_status_override = 502 + status_request_state.previous_response_not_found_rewritten = ( + response_id is None and not has_other_pending_requests + ) + event, payload, event_type, rewritten_text = _maybe_rewrite_websocket_previous_response_not_found_event( + request_state=status_request_state, + event=event, + payload=payload, + event_type=event_type, + upstream_control=session.upstream_control, + original_text=text, + ) + event_block = f"data: {rewritten_text}\n\n" retry_error_code = _websocket_precreated_retry_error_code( status_request_state, @@ -2410,164 +2595,26 @@ async def _process_parsed_http_bridge_upstream_event( and completed_usage.output_tokens == 0 ) - completed_pending_tool_call_manifest: dict[str, str] | None = None - completed_response_transition_manifest: ResponseTransitionManifest | None = None if ( response_id is not None and matched_request_state is not None and event_type == "response.completed" and not completed_empty_prewarm ): - completed_pending_tool_call_manifest = _durable_pending_tool_call_manifest( - matched_request_state, - payload, + alias_registered = await self._register_http_bridge_previous_response_id( + session, + response_id, + input_item_count=( + matched_request_state.input_item_count if matched_request_state.input_item_count > 0 else None + ), + input_full_fingerprint=( + matched_request_state.input_full_fingerprint if matched_request_state.input_item_count > 0 else None + ), + pending_tool_calls=_durable_pending_tool_call_manifest(matched_request_state, payload), ) - if completed_pending_tool_call_manifest is not None: - transition_payload = _response_transition_payload( - matched_request_state, - payload, - ) - completed_response_transition_manifest = build_response_transition_manifest( - transition_payload, - pending_tool_calls=completed_pending_tool_call_manifest, - normalize_for_public_contract=matched_request_state.enforce_openai_sdk_contract, - ) - if matched_request_state.rowless_recovery_authority_id is not None: - rowless_settled = False - if ( - matched_request_state.rowless_recovery_generation is not None - and session.durable_session_id is not None - and session.durable_owner_epoch is not None - and matched_request_state.input_item_count > 0 - and matched_request_state.input_full_fingerprint is not None - and completed_pending_tool_call_manifest is not None - ): - try: - async with SessionLocal() as rowless_session: - rowless_settled = await RowlessRecoveryRepository(rowless_session).settle_completed( - authority_id=matched_request_state.rowless_recovery_authority_id, - generation=matched_request_state.rowless_recovery_generation, - replacement_session_id=session.durable_session_id, - owner_instance_id=_service_get_settings().http_responses_session_bridge_instance_id, - owner_epoch=session.durable_owner_epoch, - request_id=matched_request_state.request_id, - response_id=response_id, - input_item_count=matched_request_state.input_item_count, - input_full_fingerprint=matched_request_state.input_full_fingerprint, - pending_tool_calls=completed_pending_tool_call_manifest, - response_transition_manifest=completed_response_transition_manifest, - ) - except Exception: - logger.warning( - "Failed to atomically settle rowless recovery", - exc_info=True, - ) - # Rowless recovery owns its journal and authority as one - # transaction. Never let generic settlement consume an UNKNOWN - # row when that transaction failed. - matched_request_state.recovery_attempt_event_observed = True - if rowless_settled: - # The durable transaction above owns publication. This - # second idempotent call publishes the in-memory alias - # only after that transaction committed. - live_alias_registered = await self._register_http_bridge_previous_response_id( - session, - response_id, - input_item_count=matched_request_state.input_item_count, - input_full_fingerprint=matched_request_state.input_full_fingerprint, - pending_tool_calls=completed_pending_tool_call_manifest, - response_transition_manifest=completed_response_transition_manifest, - ) - if not live_alias_registered: - # Durable continuity is already atomically published. - # Retire the stale live view but deliver the successful - # response; the next turn resolves the durable alias. - session.closed = True - session.upstream_control.reconnect_requested = True - session.upstream_control.retire_after_drain = True - alias_registered = True - else: - alias_registered = False - elif matched_request_state.marker_recovery_terminal_settlement_required: - marker_session_id = matched_request_state.recovery_attempt_session_id - marker_owner_epoch = matched_request_state.recovery_attempt_owner_epoch - marker_fingerprint = matched_request_state.recovery_attempt_fingerprint - marker_claim_request_id = matched_request_state.marker_recovery_claim_request_id - marker_settled = False - if ( - marker_session_id is not None - and marker_owner_epoch is not None - and marker_fingerprint is not None - and marker_claim_request_id is not None - and matched_request_state.input_item_count > 0 - and matched_request_state.input_full_fingerprint is not None - and completed_pending_tool_call_manifest is not None - ): - try: - marker_settled = await self._durable_bridge.settle_marker_recovery_completed( - session_id=marker_session_id, - api_key_id=session.key.api_key_id, - instance_id=_service_get_settings().http_responses_session_bridge_instance_id, - owner_epoch=marker_owner_epoch, - account_id=session.account.id, - request_fingerprint=marker_fingerprint, - claim_request_id=marker_claim_request_id, - request_id=matched_request_state.request_id, - response_id=response_id, - input_item_count=matched_request_state.input_item_count, - input_full_fingerprint=matched_request_state.input_full_fingerprint, - pending_tool_calls=completed_pending_tool_call_manifest, - response_transition_manifest=completed_response_transition_manifest, - lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(), - ) - except Exception: - logger.warning( - "Failed to atomically settle durable-marker recovery", - exc_info=True, - ) - matched_request_state.recovery_attempt_event_observed = True - if marker_settled: - live_alias_registered = await self._register_http_bridge_previous_response_id( - session, - response_id, - input_item_count=matched_request_state.input_item_count, - input_full_fingerprint=matched_request_state.input_full_fingerprint, - pending_tool_calls=completed_pending_tool_call_manifest, - response_transition_manifest=completed_response_transition_manifest, - ) - if not live_alias_registered: - # The durable transaction is authoritative. Retire a - # stale in-memory view, but keep the successful - # completed response available through its durable - # replacement alias on the next request. - session.closed = True - session.upstream_control.reconnect_requested = True - session.upstream_control.retire_after_drain = True - alias_registered = True - else: - alias_registered = False - else: - alias_registered = await self._register_http_bridge_previous_response_id( - session, - response_id, - input_item_count=( - matched_request_state.input_item_count if matched_request_state.input_item_count > 0 else None - ), - input_full_fingerprint=( - matched_request_state.input_full_fingerprint - if matched_request_state.input_item_count > 0 - else None - ), - pending_tool_calls=completed_pending_tool_call_manifest, - response_transition_manifest=completed_response_transition_manifest, - ) - if not alias_registered and ( - matched_request_state.rowless_recovery_authority_id is not None - or matched_request_state.marker_recovery_terminal_settlement_required - or is_http_bridge_account_neutral_replay( - kind=session.key.affinity_kind, - key=session.key.affinity_key, - ) + if not alias_registered and is_http_bridge_account_neutral_replay( + kind=session.key.affinity_kind, + key=session.key.affinity_key, ): session.upstream_control.reconnect_requested = True session.upstream_control.retire_after_drain = True @@ -2585,9 +2632,37 @@ async def _process_parsed_http_bridge_upstream_event( event_block = format_sse_event(payload) event = parse_sse_event_payload(payload) event_type = "response.failed" + # The upstream response was already acknowledged. The local + # alias write failed, so expose a terminal error downstream + # but keep the durable operation acknowledged/ambiguous to + # prevent an identical retry from dispatching it again. + continuity_persistence_failed_after_ack = True completed_usage = None completed_empty_prewarm = False + operation_state = _http_bridge_operation_state_for_event(event_type) + if operation_state is not None: + operation_request_states: list[Any] = [] + for candidate in (matched_request_state, terminal_request_state): + if candidate is not None and candidate not in operation_request_states: + operation_request_states.append(candidate) + for operation_request_state in operation_request_states: + request_operation_state = operation_state + if continuity_persistence_failed_after_ack and operation_request_state is matched_request_state: + request_operation_state = "acknowledged" + if request_operation_state == "failed": + # Failure rows are exposed only by the terminal-event + # persistence path below, which appends the terminal SSE + # block and flips the operation state atomically. + continue + await _update_http_bridge_operation_state( + self, + session, + operation_request_state, + state=request_operation_state, + response_id=response_id, + ) + recovery_attempt_session_id = ( matched_request_state.recovery_attempt_session_id if matched_request_state is not None and matched_request_state.recovery_attempt_session_id is not None @@ -2601,14 +2676,15 @@ async def _process_parsed_http_bridge_upstream_event( if ( isinstance(event_type, str) - and event_type.startswith("response.") + and (event_type.startswith("response.") or event_type == "error") and matched_request_state is not None and matched_request_state.recovery_attempt_fingerprint is not None and recovery_attempt_session_id is not None and recovery_attempt_owner_epoch is not None - and matched_request_state.rowless_recovery_authority_id is None - and not matched_request_state.marker_recovery_terminal_settlement_required - and (event_type == "response.completed" or not matched_request_state.recovery_attempt_event_observed) + and ( + event_type in {"response.completed", "response.failed", "response.incomplete", "error"} + or not matched_request_state.recovery_attempt_event_observed + ) ): settlement_marked = False for settlement_attempt in range(3): @@ -2636,7 +2712,8 @@ async def _process_parsed_http_bridge_upstream_event( response_id=response_id, release_origin_lease=( recovery_attempt_session_id != session.durable_session_id - and event_type in {"response.completed", "response.failed"} + and event_type + in {"response.completed", "response.failed", "response.incomplete", "error"} ), ) except Exception: @@ -2653,14 +2730,15 @@ async def _process_parsed_http_bridge_upstream_event( response_id=response_id, release_origin_lease=( recovery_attempt_session_id != session.durable_session_id - and event_type in {"response.completed", "response.failed"} + and event_type + in {"response.completed", "response.failed", "response.incomplete", "error"} ), ) else: await asyncio.sleep(0.05 * (settlement_attempt + 1)) if ( settlement_marked - and event_type in {"response.completed", "response.failed"} + and event_type in {"response.completed", "response.failed", "response.incomplete", "error"} and recovery_attempt_session_id != session.durable_session_id ): try: @@ -2689,12 +2767,7 @@ async def _process_parsed_http_bridge_upstream_event( # pending so an anchored follow-up that omits their outputs # (interrupted turn) can receive synthetic interrupted # outputs instead of an upstream missing-tool-output 400. - session.last_pending_tool_call_manifest_invalid = _live_pending_tool_call_manifest_is_invalid( - terminal_request_state, - payload, - ) session.last_pending_tool_calls = dict(terminal_request_state.pending_tool_call_types) - session.last_response_transition_manifest = completed_response_transition_manifest # Prefix trimming is only meaningful for list-shaped inputs, so # keep the input-count / fingerprint update scoped to that path. if terminal_request_state.input_item_count > 0: @@ -2877,9 +2950,48 @@ async def _process_parsed_http_bridge_upstream_event( if matched_request_state is not None else None ) + matched_deferred_texts = ( + _pop_websocket_deferred_reasoning_downstream_texts(matched_request_state) + if matched_request_state is not None and not suppress_downstream_event + else [] + ) + matched_terminal_state = _http_bridge_operation_state_for_event(event_type) + if continuity_persistence_failed_after_ack and matched_request_state is not None: + # The upstream response was already accepted. The downstream + # failure only reports that its durable alias could not be + # persisted, so keep the operation fenced as acknowledged while + # retaining the failure SSE for the client. + matched_terminal_state = "acknowledged" + if matched_request_state is not None and not suppress_downstream_event: + for deferred_text in matched_deferred_texts: + await _persist_http_bridge_operation_event( + self, + session, + matched_request_state, + deferred_text, + terminal=False, + ) if matched_request_state is not None and matched_event_queue is not None and not suppress_downstream_event: - for deferred_text in _pop_websocket_deferred_reasoning_downstream_texts(matched_request_state): + for deferred_text in matched_deferred_texts: await matched_event_queue.put(deferred_text) + matched_terminal_enqueued = False + if matched_request_state is not None and not suppress_downstream_event: + matched_terminal_enqueued = await _persist_http_bridge_operation_event( + self, + session, + matched_request_state, + event_block, + terminal=event_type in {"response.completed", "response.failed", "response.incomplete", "error"}, + terminal_state=matched_terminal_state, + terminal_event_queue=matched_event_queue, + terminal_delivery_scope=(completed_delivery_scope if completed_event_queue_claimed else None), + ) + if ( + matched_request_state is not None + and matched_event_queue is not None + and not suppress_downstream_event + and matched_terminal_enqueued is not True + ): await matched_event_queue.put(event_block) if terminal_request_state is None: @@ -2888,12 +3000,40 @@ async def _process_parsed_http_bridge_upstream_event( terminal_event_queue = ( completed_event_queue if completed_event_queue_claimed else terminal_request_state.event_queue ) - if terminal_request_state is not matched_request_state and terminal_event_queue is not None: - for deferred_text in _pop_websocket_deferred_reasoning_downstream_texts(terminal_request_state): - await terminal_event_queue.put(deferred_text) - await terminal_event_queue.put(event_block) + terminal_enqueued = matched_terminal_enqueued if terminal_request_state is matched_request_state else False + if terminal_request_state is not matched_request_state: + deferred_texts = _pop_websocket_deferred_reasoning_downstream_texts(terminal_request_state) + for deferred_text in deferred_texts: + if not suppress_downstream_event: + await _persist_http_bridge_operation_event( + self, + session, + terminal_request_state, + deferred_text, + terminal=False, + ) + if terminal_event_queue is not None: + await terminal_event_queue.put(deferred_text) + if not suppress_downstream_event: + terminal_enqueued = await _persist_http_bridge_operation_event( + self, + session, + terminal_request_state, + event_block, + terminal=True, + terminal_state=( + "acknowledged" + if continuity_persistence_failed_after_ack and terminal_request_state is matched_request_state + else _http_bridge_operation_state_for_event(event_type) + ), + terminal_event_queue=terminal_event_queue, + terminal_delivery_scope=(completed_delivery_scope if completed_event_queue_claimed else None), + ) + if terminal_event_queue is not None and terminal_enqueued is not True: + await terminal_event_queue.put(event_block) if terminal_event_queue is not None: - await terminal_event_queue.put(None) + if terminal_enqueued is not True: + await terminal_event_queue.put(None) if completed_event_queue_claimed and completed_delivery_scope is not None: async with session.pending_lock: # Keep the completed claim authoritative after its producer diff --git a/app/modules/proxy/_service/streaming/helpers.py b/app/modules/proxy/_service/streaming/helpers.py index 6599eb516d..c37a1d1498 100644 --- a/app/modules/proxy/_service/streaming/helpers.py +++ b/app/modules/proxy/_service/streaming/helpers.py @@ -477,6 +477,31 @@ def _classify_upstream_close( return "transient" +def _is_account_neutral_transport_drop( + close_code: int | None, + *, + response_events_seen: int, +) -> bool: + """Return whether an upstream websocket ending is account-neutral evidence. + + An abrupt transport drop that carries no close frame and arrived before + any application-layer response event is the weakest possible evidence of + account ill-health: the account never spoke at the application layer for + this request. Charging the account lets a few infrastructure resets push + it into error backoff and 502 continuity-bound follow-ups while healthy + pool siblings idle (issue #1754). Any close frame — even a non-clean one — + is upstream-authored evidence and keeps the existing penalty semantics, as + does a drop after response events started streaming. + + Close code 1006 (abnormal closure) is reserved by RFC 6455 and can never + appear in an actual close frame: adapters synthesize it locally when the + socket dies without one (aiohttp stores 1006 on ``close_code`` for an + abnormal CLOSED), so it counts as frame-less here. + """ + + return close_code in (None, 1006) and response_events_seen == 0 + + def _should_infer_upstream_status_from_proxy_error(exc: ProxyResponseError, upstream_error_code: str | None) -> bool: if exc.failure_phase == "status": return True @@ -622,6 +647,36 @@ def _mark_downstream_stream_cancelled( ) +def _rewrite_malformed_stream_error_event( + *, + enforce_openai_sdk_contract: bool, + event: OpenAIEvent | None, + event_type: str | None, + event_payload: dict[str, JsonValue] | None, + response_id: str, +) -> tuple[str, OpenAIEvent | None, dict[str, JsonValue] | None, str | None] | None: + """Rewrite a schema-less upstream ``error`` frame under the SDK contract. + + A malformed frame like ``{"type":"error","message":"..."}`` classifies as + ``error`` but carries no error envelope (``event`` is None or has no + ``error``), so it must become a terminal ``response.failed`` instead of + leaking the raw frame with a success settlement. Returns None when the + frame is not a malformed error (well-formed errors keep their + envelope-driven handling). + """ + if not enforce_openai_sdk_contract or event_type != "error": + return None + if (event is not None and event.error is not None) or not isinstance(event_payload, dict): + return None + message_value = event_payload.get("message") + message = message_value.strip() if isinstance(message_value, str) and message_value.strip() else "Upstream error" + return _build_rewritten_stream_response_failed_event( + response_id=response_id, + error_code="upstream_error", + error_message=message, + ) + + def _build_rewritten_stream_response_failed_event( *, response_id: str, diff --git a/app/modules/proxy/_service/streaming/mixin.py b/app/modules/proxy/_service/streaming/mixin.py index 470e7a9376..5bab3d94b4 100644 --- a/app/modules/proxy/_service/streaming/mixin.py +++ b/app/modules/proxy/_service/streaming/mixin.py @@ -41,7 +41,11 @@ from app.core.errors import ( response_failed_event, ) -from app.core.openai.parsing import parse_sse_event_payload +from app.core.openai.parsing import ( + _LIFECYCLE_EVENT_TYPES, + classify_event_type, + parse_sse_event_payload, +) from app.core.openai.requests import ( ResponsesRequest, ) @@ -274,6 +278,7 @@ _mark_downstream_stream_cancelled, _mark_upstream_stream_incomplete, _raw_stream_error_code_or_upstream, + _rewrite_malformed_stream_error_event, ) from app.modules.proxy._service.streaming.helpers import ( _raw_stream_error_fields as _raw_error_fields, @@ -292,13 +297,13 @@ _WEBSOCKET_FULL_REPLAY_WAIT_MIN_ITEMS, # noqa: F401 _WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS, # noqa: F401 _ApiKeyReservationTouchState, - _event_type_from_payload, _finalize_ttft_latency_ms, _RequestLogFailureMetadata, _RetryableStreamError, _StreamSettlement, _TerminalStreamError, _ttft_event_latency_ms, + _verbatim_relay_event_type, _WebSocketUpstreamControl, ) from app.modules.proxy._service.support import ( @@ -521,6 +526,16 @@ async def _stream_once( response_create_lease = AdmissionLease(None, stage="response_create", request_id=request_id) account_response_create_lease: AccountLease | None = None api_key_reservation_touch_state = _ApiKeyReservationTouchState(last_touch_at=start) + + async def _touch_api_key_reservation() -> None: + api_key_reservation_touch_state.last_touch_at = await proxy._maybe_touch_api_key_reservation( + api_key=api_key, + reservation=api_key_reservation, + last_touch_at=api_key_reservation_touch_state.last_touch_at, + request_id=request_id, + surface="stream", + ) + api_key_reservation_heartbeat_stop = asyncio.Event() api_key_reservation_heartbeat_task: asyncio.Task[None] | None = None if api_key_reservation is not None: @@ -575,7 +590,7 @@ async def _stream_once( error_code = "stream_incomplete" error_message = "Upstream websocket closed before response.completed" settlement.record_success = False - settlement.account_health_error = True + terminal_event_seen = settlement.account_health_error = True settlement.error = {"message": error_message} yield format_sse_event( response_failed_event( @@ -593,7 +608,7 @@ async def _stream_once( error_code = "upstream_unavailable" error_message = str(exc) or "Request to upstream timed out" settlement.record_success = False - settlement.account_health_error = True + terminal_event_seen = settlement.account_health_error = True settlement.error = {"message": error_message} yield format_sse_event( response_failed_event( @@ -607,23 +622,21 @@ async def _stream_once( await proxy._load_balancer.release_account_lease(account_response_create_lease) account_response_create_lease = None first_payload = parse_sse_data_json(first) - event = parse_sse_event_payload(first_payload) - event_type = _event_type_from_payload(event, first_payload) - terminal_event_seen = event_type in { - "response.completed", - "response.failed", - "response.incomplete", - "error", - } + event_type = classify_event_type(first_payload) + event = parse_sse_event_payload(first_payload) if event_type in _LIFECYCLE_EVENT_TYPES else None + terminal_event_seen = False preserve_raw_sse_line = not enforce_openai_sdk_contract and event_type == "error" + malformed_error_rewrite = _rewrite_malformed_stream_error_event( + enforce_openai_sdk_contract=enforce_openai_sdk_contract, + event=event, + event_type=event_type, + event_payload=first_payload, + response_id=response_id, + ) + if malformed_error_rewrite is not None: + first, event, first_payload, event_type = malformed_error_rewrite if event_type not in {"response.completed", "response.failed", "response.incomplete", "error"}: - api_key_reservation_touch_state.last_touch_at = await proxy._maybe_touch_api_key_reservation( - api_key=api_key, - reservation=api_key_reservation, - last_touch_at=api_key_reservation_touch_state.last_touch_at, - request_id=request_id, - surface="stream", - ) + await _touch_api_key_reservation() event_service_tier = _facade()._service_tier_from_event_payload(first_payload) if event_service_tier is not None: actual_service_tier = event_service_tier @@ -754,6 +767,8 @@ async def _stream_once( else: if first_payload is not None and not preserve_raw_sse_line: first = format_sse_event(first_payload) + if event_type in {"response.completed", "response.failed", "response.incomplete", "error"}: + terminal_event_seen = True if latency_first_token_ms is None: latency_first_token_ms = _ttft_event_latency_ms( event_type, first_payload, ttft_reasoning_deltas, attempt_started_at @@ -765,37 +780,28 @@ async def _stream_once( if terminal_stream_error is not None: raise terminal_stream_error async for line in iterator: + if verbatim_type := _verbatim_relay_event_type(line, latency_first_token_ms, ttft_reasoning_deltas): + await _touch_api_key_reservation() + if verbatim_type in _facade()._TEXT_DELTA_EVENT_TYPES: + saw_text_delta = settlement.downstream_text_visible = True + settlement.downstream_visible = True + yield line + continue event_payload = parse_sse_data_json(line) - event = parse_sse_event_payload(event_payload) - event_type = _event_type_from_payload(event, event_payload) - if event_type in {"response.completed", "response.failed", "response.incomplete", "error"}: - terminal_event_seen = True + event_type = classify_event_type(event_payload) + event = parse_sse_event_payload(event_payload) if event_type in _LIFECYCLE_EVENT_TYPES else None preserve_raw_sse_line = not enforce_openai_sdk_contract and event_type == "error" - if ( - enforce_openai_sdk_contract - and event_type == "error" - and (event is None or event.error is None) - and isinstance(event_payload, dict) - ): - message_value = event_payload.get("message") - message = ( - message_value.strip() - if isinstance(message_value, str) and message_value.strip() - else "Upstream error" - ) - line, event, event_payload, event_type = _facade()._build_rewritten_stream_response_failed_event( - response_id=response_id, - error_code="upstream_error", - error_message=message, - ) + malformed_error_rewrite = _rewrite_malformed_stream_error_event( + enforce_openai_sdk_contract=enforce_openai_sdk_contract, + event=event, + event_type=event_type, + event_payload=event_payload, + response_id=response_id, + ) + if malformed_error_rewrite is not None: + line, event, event_payload, event_type = malformed_error_rewrite if event_type not in {"response.completed", "response.failed", "response.incomplete", "error"}: - api_key_reservation_touch_state.last_touch_at = await proxy._maybe_touch_api_key_reservation( - api_key=api_key, - reservation=api_key_reservation, - last_touch_at=api_key_reservation_touch_state.last_touch_at, - request_id=request_id, - surface="stream", - ) + await _touch_api_key_reservation() event_service_tier = _facade()._service_tier_from_event_payload(event_payload) if event_service_tier is not None: actual_service_tier = event_service_tier @@ -940,6 +946,8 @@ async def _stream_once( settlement.downstream_visible = True if event_type in _facade()._TEXT_DELTA_EVENT_TYPES: settlement.downstream_text_visible = True + if event_type in {"response.completed", "response.failed", "response.incomplete", "error"}: + terminal_event_seen = True yield line if not terminal_event_seen: status, error_code, error_message, failure_metadata = _mark_upstream_stream_incomplete(settlement) @@ -1005,7 +1013,8 @@ async def _stream_once( except _TerminalStreamError: raise except (asyncio.CancelledError, GeneratorExit): - status, error_code, error_message, failure_metadata = _mark_downstream_stream_cancelled(settlement) + if not terminal_event_seen: + status, error_code, error_message, failure_metadata = _mark_downstream_stream_cancelled(settlement) raise except Exception: if settlement.downstream_visible: diff --git a/app/modules/proxy/_service/streaming/retry.py b/app/modules/proxy/_service/streaming/retry.py index f09346a162..4e725f0911 100644 --- a/app/modules/proxy/_service/streaming/retry.py +++ b/app/modules/proxy/_service/streaming/retry.py @@ -7,9 +7,10 @@ import sys import time from dataclasses import replace -from typing import Any, AsyncIterator, Mapping, cast +from typing import Any, AsyncGenerator, AsyncIterator, Mapping, TypeVar, cast import aiohttp +import anyio from app.core.auth.refresh import RefreshError, is_transient_refresh_contention, refresh_contention_kind from app.core.balancer import failover_decision @@ -47,6 +48,7 @@ _request_log_client_fields, _RetryableStreamError, _signal_propagated_capacity_startup_wait, + _signal_propagated_responses_service_cleanup_ready, _stream_settlement_error_payload, _StreamSettlement, _TerminalStreamError, @@ -63,6 +65,7 @@ _sticky_key_for_responses_request, _sticky_key_from_session_header, _sticky_key_from_turn_state_header, + _websocket_continuity_key_from_headers, ) from app.modules.proxy.api_key_usage import estimate_api_key_request_usage from app.modules.proxy.continuity import resolve_required_account_id @@ -76,6 +79,7 @@ is_upstream_model_capacity_error, ) from app.modules.proxy.load_balancer import AccountLease, AccountSelection +from app.modules.proxy.replay_safety import responses_payload_is_account_neutral_fresh_replay from app.modules.proxy.selection_errors import USAGE_LIMIT_REACHED, selection_failure_response _REQUEST_TRANSPORT_HTTP = "http" @@ -84,6 +88,27 @@ _HTTP_DOWNSTREAM_TRANSPORT_POLICIES = frozenset({"smart", "always_http", "always_websocket", "pinned"}) logger = logging.getLogger(__name__) +_TaskResultT = TypeVar("_TaskResultT") + + +async def _await_task_deferring_cancellation( + task: asyncio.Task[_TaskResultT], +) -> tuple[_TaskResultT, asyncio.CancelledError | None]: + """Finish critical cleanup while preserving the caller's cancellation.""" + + cancellation: asyncio.CancelledError | None = None + # The anyio shield keeps a level-cancelled Starlette scope from re-raising + # into every ``await``, which would otherwise busy-spin this loop until the + # owned task completes. + with anyio.CancelScope(shield=True): + while True: + try: + return await asyncio.shield(task), cancellation + except asyncio.CancelledError as exc: + if task.cancelled(): + raise + cancellation = cancellation or exc + raise RuntimeError("unreachable shielded cancellation-deferral state") def _facade() -> Any: @@ -130,7 +155,7 @@ def _verified_cross_transport_fresh_replay( input_items = cast(list[Any], input_value) if not _websocket_input_items_are_self_contained_fresh_replay(input_items): return None - session_id = _owner_lookup_session_id_from_headers(headers) + session_id = _websocket_continuity_key_from_headers(headers) if session_id is None: return None api_key_id = api_key.id if api_key is not None else None @@ -153,7 +178,10 @@ def _verified_cross_transport_fresh_replay( stored_fingerprint=continuity_state.last_completed_input_prefix_fingerprint, ): return None - return payload.model_copy(update={"previous_response_id": None}) + fresh_payload = payload.model_copy(update={"previous_response_id": None}) + if not responses_payload_is_account_neutral_fresh_replay(fresh_payload.to_replay_safety_payload()): + return None + return fresh_payload def _effective_http_downstream_transport_policy( @@ -247,6 +275,7 @@ async def _stream_with_retry( suppress_text_done_events: bool, request_transport: str, rewritten_file_account_id: str | None = None, + file_account_resolution_complete: bool = False, upstream_stream_transport_override: str | None = None, client_ip: str | None = None, enforce_openai_sdk_contract: bool = True, @@ -311,7 +340,7 @@ async def _stream_with_retry( upstream_stream_transport, request_id, ) - if rewritten_file_account_id is None: + if rewritten_file_account_id is None and not file_account_resolution_complete: proxy._raise_for_unsupported_input_image_references(payload) rewritten_file_account_id = await proxy._resolve_file_account_for_responses(payload, headers) had_prompt_cache_key = _prompt_cache_key_from_request_model(payload) is not None @@ -336,7 +365,9 @@ async def _stream_with_retry( fail_on_missing=not _is_synthesized_turn_state(turn_state), ) sticky_key_source = "none" - if affinity.kind == StickySessionKind.CODEX_SESSION: + if affinity.codex_session_source == "thread_header": + sticky_key_source = "thread_header" + elif affinity.kind == StickySessionKind.CODEX_SESSION: sticky_key_source = "session_header" elif affinity.key: sticky_key_source = "payload" if had_prompt_cache_key else "derived" @@ -370,6 +401,7 @@ async def _stream_with_retry( deferred_capacity_account: Account | None = None deferred_capacity_lease: AccountLease | None = None preferred_account_id: str | None = None + payload_replay_required_account_id: str | None = None file_preferred_account_id: str | None = rewritten_file_account_id require_preferred_account = False last_retryable_stream_error: _RetryableStreamError | None = None @@ -479,6 +511,32 @@ async def _drain_pending_post_refresh_penalty_on_terminal( return settled return True + async def _finalize_terminal_settlement_after_downstream_close( + current_settlement: _StreamSettlement, + account: Account, + ) -> None: + nonlocal settled + + async def _finalize() -> None: + nonlocal settled + if not settled: + settled = await _settle_stream_usage_before_pending_penalty(current_settlement) + if not settled: + return + if current_settlement.account_health_error: + await proxy._handle_stream_error( + account, + _stream_settlement_error_payload(current_settlement), + current_settlement.error_code or "upstream_error", + ) + elif current_settlement.record_success: + await proxy._load_balancer.record_success(account) + + finalize_task = asyncio.create_task(_finalize(), name=f"stream-terminal-settlement-{request_id}") + _, cancellation = await _await_task_deferring_cancellation(finalize_task) + if cancellation is not None: + raise cancellation + async def _wait_for_process_network_recovery( account: Account, *, @@ -525,18 +583,39 @@ async def _settle_process_network_budget_exhaustion( ) settled = await _settle_stream_usage_before_pending_penalty(settlement) + def _authorize_payload_dispatch(account: Account) -> bool: + required_account_id = payload_replay_required_account_id + if required_account_id is not None and required_account_id != account.id: + raise ProxyResponseError( + 502, + openai_error( + "previous_response_owner_unavailable", + "Request payload owner account is unavailable; retry later.", + error_type="server_error", + ), + ) + return required_account_id is None and not responses_payload_is_account_neutral_fresh_replay( + payload.to_replay_safety_payload() + ) + def _move_verified_fresh_replay_from_owner(*, account_id: str, outcome: str) -> bool: # Only a proxy-injected owner anchor with locally verified full # input may move; the failed owner stays excluded so sticky # selection cannot immediately loop back to it. - nonlocal affinity, payload, preferred_account_id, require_preferred_account, verified_fresh_replay_payload + nonlocal affinity, payload, payload_replay_required_account_id + nonlocal preferred_account_id, require_preferred_account, verified_fresh_replay_payload if not ( require_preferred_account and preferred_account_id == account_id and verified_fresh_replay_payload is not None ): return False + if not responses_payload_is_account_neutral_fresh_replay( + verified_fresh_replay_payload.to_replay_safety_payload() + ): + return False payload = verified_fresh_replay_payload + payload_replay_required_account_id = None verified_fresh_replay_payload = None excluded_account_ids.add(account_id) preferred_account_id = None @@ -556,35 +635,45 @@ async def _stream_post_refresh_with_capacity_recovery( settlement: _StreamSettlement, can_try_other_account: bool, tool_call_dedupe: _WebSocketUpstreamControl, - ) -> AsyncIterator[str]: + ) -> AsyncGenerator[str, None]: nonlocal last_transient_exc transient_retries = 0 - async def _iter_stream_once() -> AsyncIterator[str]: + async def _iter_stream_once() -> AsyncGenerator[str, None]: + inner_stream = proxy._stream_once( + account, + payload, + headers, + request_id, + False, + request_started_at=start, + allow_transient_retry=True, + api_key=api_key, + api_key_reservation=api_key_reservation, + settlement=settlement, + suppress_text_done_events=suppress_text_done_events, + upstream_stream_transport=upstream_stream_transport, + request_transport=request_transport, + concurrency_caps=concurrency_caps, + useragent=useragent, + useragent_group=useragent_group, + conversation_id=conversation_id, + client_ip=client_ip, + tool_call_dedupe=tool_call_dedupe, + enforce_openai_sdk_contract=enforce_openai_sdk_contract, + ) try: - async for line in proxy._stream_once( - account, - payload, - headers, - request_id, - False, - request_started_at=start, - allow_transient_retry=True, - api_key=api_key, - api_key_reservation=api_key_reservation, - settlement=settlement, - suppress_text_done_events=suppress_text_done_events, - upstream_stream_transport=upstream_stream_transport, - request_transport=request_transport, - concurrency_caps=concurrency_caps, - useragent=useragent, - useragent_group=useragent_group, - conversation_id=conversation_id, - client_ip=client_ip, - tool_call_dedupe=tool_call_dedupe, - enforce_openai_sdk_contract=enforce_openai_sdk_contract, - ): - yield line + try: + async for line in inner_stream: + yield line + finally: + close_task = asyncio.create_task( + inner_stream.aclose(), + name=f"stream-post-refresh-inner-close-{request_id}", + ) + _, close_cancellation = await _await_task_deferring_cancellation(close_task) + if close_cancellation is not None: + raise close_cancellation except ProxyResponseError as exc: if is_confirmed_pre_dispatch_transport_error(exc): # Keep dispatch provenance intact for the outer account @@ -620,8 +709,28 @@ async def _iter_stream_once() -> AsyncIterator[str]: _facade()._remaining_budget_seconds(deadline) ) try: - async for line in _iter_stream_once(): - yield line + attempt_stream = _iter_stream_once() + try: + try: + async for line in attempt_stream: + yield line + finally: + close_task = asyncio.create_task( + attempt_stream.aclose(), + name=f"stream-post-refresh-close-{request_id}", + ) + _, close_cancellation = await _await_task_deferring_cancellation(close_task) + if close_cancellation is not None: + raise close_cancellation + except (asyncio.CancelledError, GeneratorExit): + # A terminal frame may already have been yielded when + # downstream cancellation is delivered on the next + # generator resume. Finalize that terminal usage and + # health result before propagating cancellation so the + # reservation is not released as abandoned. + if settlement.status in {"success", "error"} and not settled: + await _finalize_terminal_settlement_after_downstream_close(settlement, account) + raise network_recovery.log_recovered() return except _TerminalStreamError: @@ -857,6 +966,10 @@ async def _retry_account_model_rejection( return True try: + # From this exact point the service finalizer below owns reservation + # settlement/release. Preflight failures before this boundary are + # still owned by the originating API startup guard. + _signal_propagated_responses_service_cleanup_ready() if payload.previous_response_id is not None: previous_response_lookup_session_id = _owner_lookup_session_id_from_headers(headers) preferred_account_id = await proxy._resolve_websocket_previous_response_owner( @@ -951,6 +1064,13 @@ async def _retry_account_model_rejection( yield format_sse_event(_facade()._proxy_request_timeout_event(request_id)) return while True: + effective_preferred_account_id = resolve_required_account_id( + ("continuation", preferred_account_id), + ("dispatched payload", payload_replay_required_account_id), + ) + effective_require_preferred_account = ( + require_preferred_account or payload_replay_required_account_id is not None + ) try: selection = await proxy._select_account_with_budget_compatible( deadline, @@ -964,7 +1084,7 @@ async def _retry_account_model_rejection( model=payload.model, service_tier=payload.service_tier, exclude_account_ids=excluded_account_ids, - preferred_account_id=preferred_account_id, + preferred_account_id=effective_preferred_account_id, require_security_work_authorized=require_security_work_authorized, lease_kind="stream", estimated_lease_tokens=estimated_lease_tokens, @@ -972,7 +1092,7 @@ async def _retry_account_model_rejection( # verified-fresh replay branch below removes its # anchor before it permits cross-account movement. fallback_on_preferred_account_unavailable=not ( - require_preferred_account or file_required_preferred_account + effective_require_preferred_account or file_required_preferred_account ), ) except ProxyResponseError as exc: @@ -1760,7 +1880,8 @@ async def _retry_account_model_rejection( ) try: settlement = _StreamSettlement() - async for line in proxy._stream_once( + register_payload_owner = _authorize_payload_dispatch(account) + inner_stream = proxy._stream_once( account, payload, headers, @@ -1799,8 +1920,40 @@ async def _retry_account_model_rejection( ), tool_call_dedupe=tool_call_dedupe, enforce_openai_sdk_contract=enforce_openai_sdk_contract, - ): - yield line + ) + try: + try: + async for line in inner_stream: + if register_payload_owner: + payload_replay_required_account_id = account.id + register_payload_owner = False + yield line + if register_payload_owner: + payload_replay_required_account_id = account.id + except BaseException as exc: + if register_payload_owner and not ( + isinstance(exc, ProxyResponseError) + and is_confirmed_pre_dispatch_transport_error(exc) + ): + payload_replay_required_account_id = account.id + raise + finally: + close_task = asyncio.create_task( + inner_stream.aclose(), + name=f"stream-inner-close-{request_id}", + ) + _, close_cancellation = await _await_task_deferring_cancellation(close_task) + if close_cancellation is not None: + raise close_cancellation + except (asyncio.CancelledError, GeneratorExit): + # A terminal frame may already have been yielded when + # downstream cancellation is delivered on the next + # generator resume. Finalize that terminal usage and + # health result before propagating cancellation so the + # reservation is not released as abandoned. + if settlement.status in {"success", "error"} and not settled: + await _finalize_terminal_settlement_after_downstream_close(settlement, account) + raise except (_TransientStreamError, ProxyResponseError) as tex: if account.id == account_model_replacement_account_id: # Account/model routing gets exactly one selected @@ -1864,7 +2017,6 @@ async def _retry_account_model_rejection( account.id, error_code, ) - yield format_sse_event(event) settlement.record_success = False settlement.error_code = error_code settlement.error_message = error_message @@ -1873,6 +2025,11 @@ async def _retry_account_model_rejection( else: settlement.error = tex.error settlement.account_health_error = _facade()._should_penalize_stream_error(error_code) + try: + yield format_sse_event(event) + except (asyncio.CancelledError, GeneratorExit): + await _finalize_terminal_settlement_after_downstream_close(settlement, account) + raise settled = await _settle_stream_usage_before_pending_penalty(settlement) if settled and settlement.account_health_error: await proxy._handle_stream_error( @@ -2466,13 +2623,27 @@ async def _retry_account_model_rejection( and account.id != file_preferred_account_id and attempt < max_attempts - 1 ) - async for line in _stream_post_refresh_with_capacity_recovery( + post_refresh_stream = _stream_post_refresh_with_capacity_recovery( account, settlement=settlement, can_try_other_account=can_try_other_account, tool_call_dedupe=tool_call_dedupe, - ): - yield line + ) + try: + async for line in post_refresh_stream: + yield line + finally: + # Closing this generator runs its internal + # cancellation-safe close/terminal finalization; + # without an owned aclose() the child would stay + # suspended after a downstream disconnect. + close_task = asyncio.create_task( + post_refresh_stream.aclose(), + name=f"stream-post-refresh-outer-close-{request_id}", + ) + _, close_cancellation = await _await_task_deferring_cancellation(close_task) + if close_cancellation is not None: + raise close_cancellation except ProxyResponseError as retry_exc: if _facade()._is_proxy_budget_exhausted_error(retry_exc): await _settle_process_network_budget_exhaustion(account, settlement) diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index 3e9dc047f9..8a79957e72 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -22,12 +22,13 @@ from app.core.errors import OpenAIErrorEnvelope, openai_error from app.core.openai.model_registry import get_model_registry from app.core.openai.models import OpenAIEvent -from app.core.openai.public_output import strip_blank_html_comment_lines as _strip_blank_html_comment_lines +from app.core.openai.parsing import classify_event_type from app.core.plan_types import account_plan_matches_allowed from app.core.resilience.network_recovery import PROCESS_NETWORK_UNAVAILABLE_CODE from app.core.resilience.overload import is_local_overload_error_code from app.core.types import JsonValue from app.core.upstream_proxy import ResolvedUpstreamRoute +from app.core.utils.sse import sse_event_type_from_block from app.db.models import Account from app.modules.api_keys.service import ( ApiKeyData, @@ -40,8 +41,6 @@ AccountSelection, CatalogOmissionQuotaAdmission, ) -from app.modules.proxy.response_transition_manifest import ResponseTransitionManifest -from app.modules.proxy.rowless_recovery import RowlessRecoveryCaptureIntent from app.modules.proxy.tool_call_dedupe import ToolCallDedupeKey from app.modules.proxy.work_admission import AdmissionLease @@ -49,6 +48,7 @@ _REQUEST_TRANSPORT_HTTP = "http" _REQUEST_TRANSPORT_WEBSOCKET = "websocket" +_REASONING_SUMMARY_BLANK_HTML_COMMENT_RE = re.compile(r"(?m)^[ \t]*[ \t]*(?:\r?\n|\Z)") _TTFT_EVENT_TYPES = frozenset( { "response.output_text.delta", @@ -74,6 +74,7 @@ { "turn_state_header", "session_header", + "thread_header", "internal_unanchored_parallel", "internal_model_parallel", "internal_request_parallel", @@ -97,6 +98,31 @@ "propagated_capacity_startup_ready", default=None, ) +_PROPAGATED_RESPONSES_SERVICE_CLEANUP_READY: ContextVar[asyncio.Event | None] = ContextVar( + "propagated_responses_service_cleanup_ready", + default=None, +) +_PROPAGATED_RESPONSES_OWNER_FORWARD_DISPATCHED: ContextVar[asyncio.Event | None] = ContextVar( + "propagated_responses_owner_forward_dispatched", + default=None, +) +_PROPAGATED_RESPONSES_OWNER_FORWARD_REJECTED: ContextVar[asyncio.Event | None] = ContextVar( + "propagated_responses_owner_forward_rejected", + default=None, +) + + +def _strip_blank_html_comment_lines(text: str) -> str: + terminal_match = None + for match in _REASONING_SUMMARY_BLANK_HTML_COMMENT_RE.finditer(text): + if match.end() == len(text): + terminal_match = match + cleaned, count = _REASONING_SUMMARY_BLANK_HTML_COMMENT_RE.subn("", text) + if count == 0: + return text + if terminal_match is not None: + return cleaned.rstrip("\r\n") + return cleaned def _reasoning_summary_delta_key(payload: Mapping[str, JsonValue]) -> tuple[str | None, int | None, int | None]: @@ -242,6 +268,58 @@ def _finalize_ttft_latency_ms( return _ttft_latency_ms_from_visible_at(_finalize_ttft_reasoning_deltas(pending_reasoning_deltas), started_at) +# Stream frames whose parsed payload feeds a real per-event consumer: +# lifecycle/terminal handling and usage settlement (created/in_progress/ +# completed/failed/incomplete/error), parallel tool-call rewrite + duplicate +# side-effect suppression (response.output_item.*), and text-done suppression +# (response.output_text.done / response.content_part.done). Canonically framed +# frames of any other type can relay upstream bytes verbatim once the TTFT +# window is settled and no service-tier snapshot is present. +_MUST_PARSE_STREAM_EVENT_TYPES = frozenset( + { + "response.created", + "response.in_progress", + "response.completed", + "response.failed", + "response.incomplete", + "error", + "response.output_item.added", + "response.output_item.done", + "response.output_text.done", + "response.content_part.done", + } +) +# Raw-line gate for service-tier attribution: response snapshots carry +# `"service_tier"` in their JSON, and a false positive (the substring inside +# delta text) merely takes the full-parse path. +_SERVICE_TIER_MARKER = '"service_tier"' + + +def _verbatim_relay_event_type( + line: str, + latency_first_token_ms: int | None, + pending_reasoning_deltas: dict[tuple[str | None, int | None, int | None], _TTFTReasoningDeltaState], +) -> str | None: + """Return the cheap event type when a stream frame can relay verbatim. + + Eligible frames are canonically framed (leading ``event: `` line, + single JSON-object ``data:`` line — see ``sse_event_type_from_block``), + outside the must-parse set, past the TTFT first-token window (including a + pending reasoning-delta window), and free of the service-tier marker. + Everything else returns ``None`` and takes the full parse + + ``format_sse_event`` path, preserving the EventSource framing guarantee + for data-only blocks. + """ + if latency_first_token_ms is None or pending_reasoning_deltas: + return None + if _SERVICE_TIER_MARKER in line: + return None + event_type = sse_event_type_from_block(line) + if event_type is None or event_type in _MUST_PARSE_STREAM_EVENT_TYPES: + return None + return event_type + + def _bind_propagated_capacity_startup_wait(event: asyncio.Event) -> Token[asyncio.Event | None]: return _PROPAGATED_CAPACITY_STARTUP_WAIT.set(event) @@ -276,6 +354,48 @@ def _signal_propagated_capacity_startup_ready() -> None: event.set() +def _bind_propagated_responses_service_cleanup_ready(event: asyncio.Event) -> Token[asyncio.Event | None]: + return _PROPAGATED_RESPONSES_SERVICE_CLEANUP_READY.set(event) + + +def _reset_propagated_responses_service_cleanup_ready(token: Token[asyncio.Event | None]) -> None: + _PROPAGATED_RESPONSES_SERVICE_CLEANUP_READY.reset(token) + + +def _signal_propagated_responses_service_cleanup_ready() -> None: + event = _PROPAGATED_RESPONSES_SERVICE_CLEANUP_READY.get() + if event is not None: + event.set() + + +def _bind_propagated_responses_owner_forward_dispatched(event: asyncio.Event) -> Token[asyncio.Event | None]: + return _PROPAGATED_RESPONSES_OWNER_FORWARD_DISPATCHED.set(event) + + +def _reset_propagated_responses_owner_forward_dispatched(token: Token[asyncio.Event | None]) -> None: + _PROPAGATED_RESPONSES_OWNER_FORWARD_DISPATCHED.reset(token) + + +def _signal_propagated_responses_owner_forward_dispatched() -> None: + event = _PROPAGATED_RESPONSES_OWNER_FORWARD_DISPATCHED.get() + if event is not None: + event.set() + + +def _bind_propagated_responses_owner_forward_rejected(event: asyncio.Event) -> Token[asyncio.Event | None]: + return _PROPAGATED_RESPONSES_OWNER_FORWARD_REJECTED.set(event) + + +def _reset_propagated_responses_owner_forward_rejected(token: Token[asyncio.Event | None]) -> None: + _PROPAGATED_RESPONSES_OWNER_FORWARD_REJECTED.reset(token) + + +def _signal_propagated_responses_owner_forward_rejected() -> None: + event = _PROPAGATED_RESPONSES_OWNER_FORWARD_REJECTED.get() + if event is not None: + event.set() + + def _account_selection_recovery_sleep_seconds_from_message( message: str | None, *, @@ -723,12 +843,6 @@ def _consume_api_key_reservation_heartbeat_result(task: asyncio.Task[None]) -> N logger.warning("API key reservation heartbeat task failed during cancellation", exc_info=True) -@dataclass(frozen=True, slots=True) -class _FilePinEntry: - account_id: str - expires_at: float - - @dataclass(frozen=True, slots=True) class _RequestLogFailureMetadata: failure_phase: str | None = None @@ -758,6 +872,34 @@ class _DeferredAccountBackoffTracker: current_lifecycle: _DeferredAccountBackoffLifecycle | None = None +@dataclass(eq=False, slots=True) +class _HTTPBridgeResponseCreateAttempt: + ordinal: int + disarmed: bool = False + response_observed: bool = False + retry_circuit_failure_recorded: bool = False + retry_circuit_failure_settled: anyio.Event | None = None + + +@dataclass(frozen=True, slots=True) +class _HTTPBridgeRetryCircuitAttemptSelection: + kind: Literal["absent", "eligible", "recorded", "settled", "ineligible"] + attempts: tuple[_HTTPBridgeResponseCreateAttempt, ...] = () + + def __post_init__(self) -> None: + carries_attempts = self.kind in {"eligible", "recorded", "settled"} + if carries_attempts != bool(self.attempts): + raise ValueError(f"invalid retry-circuit attempt selection: {self.kind}") + + @property + def attempt(self) -> _HTTPBridgeResponseCreateAttempt | None: + return self.attempts[0] if len(self.attempts) == 1 else None + + @property + def ambiguous(self) -> bool: + return len(self.attempts) > 1 + + @dataclass class _WebSocketRequestState: request_id: str @@ -780,19 +922,8 @@ class _WebSocketRequestState: # send. Retries replace this value so admission wait and prior attempts do # not age a fresh send into the eventless owner deadline. response_create_sent_at: float | None = None - # Set immediately before the one-shot replacement path invokes its send - # primitive. A cancellation while reconnecting is still proven unsent and - # may roll back a reversible recovery alias; cancellation after this point - # is ambiguous and must retain the alias/fail closed. - fresh_upstream_send_primitive_reached: bool = False - # Set immediately before the initial rowless semantic-rebase send helper - # is invoked. Cleanup may restore the durable attempt only while this is - # false; after the helper is entered, cancellation is ambiguous. - rowless_recovery_send_primitive_reached: bool = False - # True only after the initial rowless send returned the transport's exact - # closed-before-send proof. Generic socket-only reconnects must never set - # this bit: replay_count alone is not physical non-delivery evidence. - rowless_recovery_first_send_proven_unsent: bool = False + response_create_attempt_count: int = 0 + response_create_attempt: _HTTPBridgeResponseCreateAttempt | None = None bridge_queue_wait_started_at: float | None = None # Monotonic deadline of the original bridge request budget. Retry and # recovery paths re-prepare request states with a fresh started_at, so @@ -816,6 +947,24 @@ class _WebSocketRequestState: connection_request_kind: str | None = None generate_false_prewarm: bool = False api_key: ApiKeyData | None = None + # The client's requested model captured before api-key enforcement + # normalized aliases (``gpt-5-high`` -> ``gpt-5``), with the key's + # ``enforced_model`` substituted and the fast-mode correction applied, + # exactly like the HTTP path's ``raw_source_model``. Consumed only by the + # WebSocket source-ownership guards; it must never reach the upstream + # wire payload. ``None`` on request states that were not built by + # ``_prepare_websocket_response_create_request`` (replays, archives), + # which keeps those on the normalized-model check. + raw_source_model: str | None = None + # True when the HTTP route would exclude this request from model-source + # routing (``responses_source_route_excluded``: a terminal compaction + # trigger, or ``input_file`` references pinned to the uploading + # subscription account). The WebSocket source-ownership guards skip such + # requests so the owner-routing logic can dispatch them to a subscription + # account, exactly like HTTP. ``False`` on request states that were not + # built by ``_prepare_websocket_response_create_request``, which keeps + # the guards active for those. + source_route_excluded: bool = False request_usage_budget: ApiKeyRequestUsageBudget | None = None request_text: str | None = None replay_count: int = 0 @@ -870,11 +1019,6 @@ class _WebSocketRequestState: # on, and dropping the anchor there would silently turn a continuation into # a context-free fresh turn. fresh_upstream_request_is_retry_safe: bool = False - # True only when the retained fresh body is also proven free of - # account-scoped identifiers (for example conversation, prompt, hosted - # input items, or uploaded files). Replay safety proves context - # completeness; it does not by itself authorize changing accounts. - fresh_upstream_request_is_account_neutral: bool = False # Stable fingerprint used by the durable recovery-attempt journal. It is # populated only for a proof-gated fresh replay candidate. recovery_attempt_fingerprint: str | None = None @@ -888,21 +1032,32 @@ class _WebSocketRequestState: # claimed recovery journal; an attempted send must remain consumed. recovery_attempt_dispatched: bool = False recovery_attempt_event_observed: bool = False - # True only for the proof-gated automatic recovery of an active durable - # rejected-anchor marker. Its response.completed checkpoint must publish - # the replacement anchor, alias, marker clear, and recovery journal in one - # durable transaction before downstream success is delivered. - marker_recovery_terminal_settlement_required: bool = False - # True only after this concrete request successfully claimed the active - # marker generation. Cleanup must not attempt a rollback before this flips. - marker_recovery_claimed: bool = False - # The outer HTTP request that owns the durable marker claim. This differs - # from request_id, which identifies the concrete upstream dispatch journal. - marker_recovery_claim_request_id: str | None = None - # Plaintext anchor is kept only in request memory so the delayed marker - # claim can revalidate the exact durable generation immediately before the - # recovery journal is written. - marker_recovery_rejected_response_id: str | None = None + # Durable operation identity for a continuity-bound response.create. It is + # stable across client reconnects with the same parent response and body. + operation_id: str | None = None + operation_fingerprint: str | None = None + operation_parent_response_id: str | None = None + operation_registered: bool = False + # Account-neutral durable recovery keeps the original operation identity + # while asking request submission to rebind it to the replacement session. + operation_rebind_required: bool = False + # True after an existing UNKNOWN operation is claimed for this attempt. + # If admission fails before send, cleanup must restore UNKNOWN rather than + # treating the pre-existing row like a newly-created operation. + operation_recovery_claimed: bool = False + # True only when this request created the durable operation row. A + # pre-dispatch admission failure may remove that row; an existing row + # represents an ambiguous upstream attempt and must remain fenced. + operation_created: bool = False + operation_replay: bool = False + operation_dispatched: bool = False + # Immutable durable attempt generation. Recovery claims increment the + # operation's dispatch count before sending a replacement attempt. + operation_attempt_generation: int = 0 + # Last response identity successfully written to the durable operation. + # Retry setup may clear the active response before a replacement is + # acknowledged, but fallback settlement must still fence against this ID. + operation_persisted_response_id: str | None = None # Responses-Lite model advertised by ``fresh_upstream_request_text``. A # fresh replay built from a trusted marker-only frame has the reserved # marker stripped, so swapping to the fresh body must also swap this onto @@ -911,6 +1066,9 @@ class _WebSocketRequestState: fresh_upstream_request_responses_lite_model: str | None = None request_stage: str = "first_turn" preferred_account_id: str | None = None + # Once an account-bound body has been dispatched, retries remain pinned to + # that owner even when stale-anchor recovery removes previous_response_id. + replay_required_account_id: str | None = None require_security_work_authorized: bool = False durable_capability_lineage_required: bool = False file_required_preferred_account: bool = False @@ -927,11 +1085,6 @@ class _WebSocketRequestState: last_upstream_activity_at: float | None = None upstream_model_output_seen: bool = False previous_response_not_found_rewritten: bool = False - rowless_recovery_capture_intent: RowlessRecoveryCaptureIntent | None = None - rowless_recovery_authority_id: str | None = None - rowless_recovery_generation: int | None = None - rowless_recovery_task_authority_digest: str | None = None - rowless_recovery_wire_fingerprint: str | None = None previous_response_owner_lookup_source: str | None = None previous_response_owner_lookup_outcome: str | None = None previous_response_owner_requested_at: datetime | None = None @@ -944,14 +1097,13 @@ class _WebSocketRequestState: account_response_create_release: Callable[[AccountLease | None], Coroutine[Any, Any, None]] | None = None websocket_stream_lease: AccountLease | None = None affinity_policy: _AffinityPolicy = field(default_factory=_AffinityPolicy) + thread_affinity_last_touch_at: float = field(default_factory=time.monotonic) suppressed_downstream_tool_call: bool = False suppressed_duplicate_tool_call: bool = False pending_function_call_ids: list[str] = field(default_factory=list) pending_tool_call_types: dict[str, str] = field(default_factory=dict) added_tool_call_types: dict[str, str] = field(default_factory=dict) tool_call_manifest_invalid: bool = False - response_output_items: dict[int, dict[str, JsonValue]] = field(default_factory=dict) - response_output_items_invalid: bool = False seen_tool_call_keys: dict[ToolCallDedupeKey, None] = field(default_factory=dict) input_item_count: int = 0 input_full_fingerprint: str | None = None @@ -1063,18 +1215,16 @@ class _HTTPBridgeSession: last_completed_response_account_id: str | None = None last_completed_input_prefix_fingerprint: str | None = None last_pending_tool_calls: dict[str, str] = field(default_factory=dict) - last_response_transition_manifest: ResponseTransitionManifest | None = None - # A false value means the completed response's pending-tool manifest was - # physically verified (including the valid empty-manifest case). Keep an - # invalid/unrepresentable live manifest distinct from an empty one so a - # later full-history resend cannot mint an unanchored replay proof. - last_pending_tool_call_manifest_invalid: bool = False durable_session_id: str | None = None durable_owner_epoch: int | None = None upstream_reader: asyncio.Task[None] | None = None last_upstream_close_code: int | None = None last_upstream_close_generation: int = 0 closed: bool = False + # ``closed`` is only an admission fence. Resource teardown is single-flight + # through this task so invalidation and shutdown can distinguish a rejected + # session from one whose socket and leases actually have a close owner. + resource_close_task: asyncio.Task[None] | None = None # ``closed`` rejects new admissions but is written by many unrelated # retirement paths; it never proves that a sender owns pending settlement. # Only the submitter may claim this, while holding ``lifecycle_lock``, when @@ -1113,16 +1263,6 @@ def claim_liveness_settlement(self) -> bool: return self.liveness_settlement_owner == "send" -def _clear_http_bridge_session_response_checkpoint(session: _HTTPBridgeSession) -> None: - session.last_completed_response_id = None - session.last_completed_response_account_id = None - session.last_completed_input_count = 0 - session.last_completed_input_prefix_fingerprint = None - session.last_pending_tool_call_manifest_invalid = False - session.last_pending_tool_calls.clear() - session.last_response_transition_manifest = None - - def _complete_http_bridge_handoff( session: _HTTPBridgeSession, inflight_sessions: dict[_HTTPBridgeSessionKey, asyncio.Future[_HTTPBridgeSession]], @@ -1327,9 +1467,21 @@ def _clear_websocket_deferred_reasoning_downstream_texts(request_state: _WebSock request_state.deferred_reasoning_downstream_texts = [] +def _mark_response_create_attempt_observed( + request_state: _WebSocketRequestState | None, + event_type: str | None, +) -> None: + if request_state is None or event_type is None or not event_type.startswith("response."): + return + attempt = request_state.response_create_attempt + if attempt is not None: + attempt.response_observed = True + + def _record_response_event(request_state: _WebSocketRequestState | None, event_type: str | None) -> None: if request_state is None or event_type is None or not event_type.startswith("response."): return + _mark_response_create_attempt_observed(request_state, event_type) request_state.last_upstream_activity_at = time.monotonic() if event_type in {"response.failed", "response.incomplete"}: return @@ -1341,19 +1493,6 @@ def _websocket_request_can_replay_before_visible_output( *, allow_clean_close_retry: bool = False, ) -> bool: - if request_state.rowless_recovery_authority_id is not None: - # Automatic authorization is claimed before its first physical send. - # Admit only that retained in-memory projection; once the send marker - # is durable, every outcome is ambiguous and remains non-replayable. - return bool( - not request_state.rowless_recovery_send_primitive_reached - and request_state.replay_count == 0 - and request_state.response_event_count == 0 - and not request_state.downstream_visible - and not request_state.upstream_model_output_seen - and request_state.fresh_upstream_request_is_retry_safe - and request_state.fresh_upstream_request_text is not None - ) if not request_state.request_text: return False if request_state.transport == _REQUEST_TRANSPORT_WEBSOCKET and request_state.response_create_sent_at is None: @@ -1471,14 +1610,7 @@ class _WebSocketReceiveTimeout: def _event_type_from_payload(event: OpenAIEvent | None, payload: dict[str, JsonValue] | None) -> str | None: if event is not None: return event.type - if payload is None: - return None - payload_type = payload.get("type") - if isinstance(payload_type, str): - return payload_type - if isinstance(payload.get("error"), dict): - return "error" - return None + return classify_event_type(payload) async def _wait_for_websocket_continuity_gap( @@ -1520,6 +1652,7 @@ def _is_account_neutral_error_code(code: str | None) -> bool: PROCESS_NETWORK_UNAVAILABLE_CODE, "proxy_unavailable", "responses_compact_input_too_large", + "stream_idle_timeout", } @@ -1606,7 +1739,4 @@ def _openai_error_envelope_from_response_failed_payload( resets_in = error_payload.get("resets_in_seconds") if isinstance(resets_in, int | float): error_detail["resets_in_seconds"] = resets_in - action = error_payload.get("action") - if isinstance(action, str) and action.strip(): - error_detail["action"] = action.strip() return envelope diff --git a/app/modules/proxy/_service/warmup.py b/app/modules/proxy/_service/warmup.py index f6c611345c..862117e307 100644 --- a/app/modules/proxy/_service/warmup.py +++ b/app/modules/proxy/_service/warmup.py @@ -248,7 +248,6 @@ async def _submit_account_warmup(account: _WarmupAccountSnapshot) -> _WarmupSubm headers=filtered_headers, warmup_model=effective_model, prohibit_fast_mode=prohibit_fast_mode, - allow_pre_submit_errors_as_result=len(accounts_to_submit) > 1, ) submission_results = await asyncio.gather(*(_submit_account_warmup(account) for account in accounts_to_submit)) @@ -300,7 +299,6 @@ async def _submit_warmup_request( headers: Mapping[str, str], warmup_model: str, prohibit_fast_mode: bool, - allow_pre_submit_errors_as_result: bool = False, ) -> _WarmupSubmitResult: started_at = time.monotonic() useragent, useragent_group, conversation_id = _request_log_client_fields(headers) @@ -423,13 +421,9 @@ async def _submit_warmup_request( except ProxyAuthError as exc: error_code = "auth_error" error_message = str(exc) or "Warmup authentication failed" - if not allow_pre_submit_errors_as_result: - raise except ProxyRateLimitError as exc: error_code = "rate_limit_exceeded" error_message = str(exc) or "Warmup request was rate limited" - if not allow_pre_submit_errors_as_result: - raise except Exception as exc: error_code = "upstream_error" error_message = str(exc) or "Warmup request failed" diff --git a/app/modules/proxy/_service/websocket/helpers.py b/app/modules/proxy/_service/websocket/helpers.py index 9bb2bc99d9..83858cc645 100644 --- a/app/modules/proxy/_service/websocket/helpers.py +++ b/app/modules/proxy/_service/websocket/helpers.py @@ -5,7 +5,7 @@ import sys import time from collections import deque -from collections.abc import Sequence +from collections.abc import Awaitable, Sequence from dataclasses import dataclass from typing import Any, cast @@ -339,12 +339,94 @@ from app.modules.proxy.http_bridge_forwarding import ( OwnerForwardRelayFailure as OwnerForwardRelayFailure, ) +from app.modules.proxy.replay_safety import responses_payload_is_account_neutral_fresh_replay def _facade() -> Any: return sys.modules["app.modules.proxy.service"] +# A confirmed stale previous-response anchor can otherwise cause every client +# reconnect to repeat the same owner lookup and doomed upstream connection. +# Keep this local and short-lived: an owner record may still be committed by a +# concurrent request, so discovery always invalidates the negative entry. +_WEBSOCKET_STALE_PREVIOUS_RESPONSE_CACHE_TTL_SECONDS = 60.0 +_WEBSOCKET_STALE_PREVIOUS_RESPONSE_CACHE_LIMIT = 4096 +_websocket_stale_previous_response_index: dict[tuple[str, str | None], float] = {} + + +def _clear_websocket_stale_previous_response_cache() -> None: + """Drop process-local negative entries when a proxy service is created. + + The cache intentionally is not durable: it only suppresses repeated + lookups during a short recovery window. Clearing it with the service + lifecycle prevents entries from one app/test instance from affecting a + later instance that happens to receive the same synthetic response id. + """ + _websocket_stale_previous_response_index.clear() + + +def _prune_websocket_stale_previous_response_cache(now: float | None = None) -> None: + current_time = time.monotonic() if now is None else now + for cache_key, expires_at in tuple(_websocket_stale_previous_response_index.items()): + if expires_at <= current_time: + _websocket_stale_previous_response_index.pop(cache_key, None) + while len(_websocket_stale_previous_response_index) > _WEBSOCKET_STALE_PREVIOUS_RESPONSE_CACHE_LIMIT: + _websocket_stale_previous_response_index.pop(next(iter(_websocket_stale_previous_response_index))) + + +def _remember_websocket_stale_previous_response( + *, + previous_response_id: str | None, + api_key_id: str | None, +) -> None: + if previous_response_id is None: + return + response_id = previous_response_id.strip() + if not response_id: + return + now = time.monotonic() + _prune_websocket_stale_previous_response_cache(now) + cache_key = (response_id, api_key_id) + _websocket_stale_previous_response_index.pop(cache_key, None) + _websocket_stale_previous_response_index[cache_key] = now + _WEBSOCKET_STALE_PREVIOUS_RESPONSE_CACHE_TTL_SECONDS + _prune_websocket_stale_previous_response_cache(now) + + +def _forget_websocket_stale_previous_response( + *, + previous_response_id: str | None, + api_key_id: str | None, +) -> None: + if previous_response_id is None: + return + response_id = previous_response_id.strip() + if not response_id: + return + _websocket_stale_previous_response_index.pop((response_id, api_key_id), None) + + +def _is_websocket_stale_previous_response( + *, + previous_response_id: str | None, + api_key_id: str | None, +) -> bool: + if previous_response_id is None: + return False + response_id = previous_response_id.strip() + if not response_id: + return False + now = time.monotonic() + _prune_websocket_stale_previous_response_cache(now) + expires_at = _websocket_stale_previous_response_index.get((response_id, api_key_id)) + if expires_at is None: + return False + if expires_at <= now: + _websocket_stale_previous_response_index.pop((response_id, api_key_id), None) + return False + return True + + def _prepare_websocket_request_state_for_visible_output_replay( request_state: "_WebSocketRequestState", ) -> str | None: @@ -378,37 +460,75 @@ def _websocket_owner_switch_has_other_pending_requests( return any(pending is not request_state for pending in pending_requests) -def _prepare_websocket_request_state_for_account_switch( +def _websocket_request_text_is_account_neutral_fresh_replay(request_text: str | None) -> bool: + if not isinstance(request_text, str): + return False + try: + payload = json.loads(request_text) + except json.JSONDecodeError: + return False + if not isinstance(payload, dict): + return False + event_type = payload.get("type") + if event_type is not None and event_type != "response.create": + return False + payload.pop("type", None) + return responses_payload_is_account_neutral_fresh_replay(cast(dict[str, JsonValue], payload)) + + +def _bind_websocket_request_dispatch_owner( request_state: "_WebSocketRequestState", + *, + account_id: str, + exact_request_text: str, +) -> bool: + required_account_id = request_state.replay_required_account_id + if _websocket_request_text_is_account_neutral_fresh_replay(exact_request_text): + return required_account_id is None or required_account_id == account_id + if required_account_id is not None and required_account_id != account_id: + return False + request_state.preferred_account_id = account_id + request_state.replay_required_account_id = account_id + return True + + +def _install_verified_fresh_replay( + request_state: "_WebSocketRequestState", + *, + require_proxy_injected_previous_response_id: bool = True, + require_account_neutral: bool = True, ) -> str | None: - """Return an unsent request body only when moving accounts is proven safe.""" - if request_state.previous_response_id is None: - return request_state.request_text - if not ( - request_state.proxy_injected_previous_response_id - and request_state.fresh_upstream_request_is_retry_safe - and request_state.fresh_upstream_request_text - ): + if not (request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text): return None - try: - fresh_payload = json.loads(request_state.fresh_upstream_request_text) - except (TypeError, json.JSONDecodeError): + if require_proxy_injected_previous_response_id and not request_state.proxy_injected_previous_response_id: return None - fresh_input = fresh_payload.get("input") - if extract_input_file_ids(fresh_input): - # A retained full body can be replay-safe for text continuity while - # still naming an account-scoped uploaded file. Keep its injected - # anchor instead of moving that file reference to another account. + fresh_request_text = request_state.fresh_upstream_request_text + account_neutral = _websocket_request_text_is_account_neutral_fresh_replay(fresh_request_text) + if require_account_neutral and not account_neutral: return None - - request_state.request_text = request_state.fresh_upstream_request_text + replay_required_account_id = request_state.replay_required_account_id or request_state.preferred_account_id + if not account_neutral and replay_required_account_id is None: + return None + request_state.request_text = fresh_request_text request_state.previous_response_id = None request_state.preferred_account_id = None + request_state.replay_required_account_id = None if account_neutral else replay_required_account_id request_state.proxy_injected_previous_response_id = False request_state.fresh_upstream_request_is_retry_safe = False request_state.responses_lite_model = request_state.fresh_upstream_request_responses_lite_model _refresh_websocket_request_input_fingerprint_from_text(request_state) - return request_state.request_text + return fresh_request_text + + +def _prepare_websocket_request_state_for_account_switch( + request_state: "_WebSocketRequestState", +) -> str | None: + """Return an unsent request body only when moving accounts is proven safe.""" + if request_state.previous_response_id is None: + if not _websocket_request_text_is_account_neutral_fresh_replay(request_state.request_text): + return None + return request_state.request_text + return _install_verified_fresh_replay(request_state) def _websocket_continuity_anchor_for_payload( @@ -595,11 +715,21 @@ def _rewrite_websocket_downstream_response_id( if downstream_response_id is None: return payload + direct_response_id = payload.get("response_id") + rewrite_direct = isinstance(direct_response_id, str) and direct_response_id != downstream_response_id + response = payload.get("response") + nested_response_id = response.get("id") if isinstance(response, dict) else None + rewrite_nested = isinstance(nested_response_id, str) and nested_response_id != downstream_response_id + if not rewrite_direct and not rewrite_nested: + # Identity fast-path contract: callers skip re-serialization when the + # original payload object comes back, so an already-aligned frame must + # not be copied into an equal-but-new dict. + return payload + rewritten = dict(payload) - if isinstance(rewritten.get("response_id"), str): + if rewrite_direct: rewritten["response_id"] = downstream_response_id - response = rewritten.get("response") - if isinstance(response, dict) and isinstance(response.get("id"), str): + if rewrite_nested and isinstance(response, dict): rewritten["response"] = {**response, "id": downstream_response_id} return rewritten @@ -809,35 +939,43 @@ def _websocket_auth_request_can_switch_account(request_state: _WebSocketRequestS if request_state.file_required_preferred_account: return False if request_state.previous_response_id is None: - return True + return request_state.request_text is None or _websocket_request_text_is_account_neutral_fresh_replay( + request_state.request_text + ) if not ( request_state.proxy_injected_previous_response_id and request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text ): return False - return not _websocket_fresh_request_blocks_account_switch(request_state) + return _websocket_request_text_is_account_neutral_fresh_replay( + request_state.fresh_upstream_request_text + ) and not _websocket_fresh_request_blocks_account_switch(request_state) def _prepare_websocket_request_state_for_auth_replay( request_state: _WebSocketRequestState, + *, + current_account_id: str | None = None, ) -> str | None: if request_state.last_downstream_sequence_number is not None: return None - if not _websocket_auth_request_can_switch_account(request_state): + can_switch_account = _websocket_auth_request_can_switch_account(request_state) + can_retry_bound_owner = ( + request_state.auth_replay_count == 0 + and current_account_id is not None + and request_state.replay_required_account_id == current_account_id + and isinstance(request_state.request_text, str) + ) + if not can_switch_account and not can_retry_bound_owner: return None - if ( + if can_switch_account and ( request_state.proxy_injected_previous_response_id and request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text ): - request_state.request_text = request_state.fresh_upstream_request_text - request_state.previous_response_id = None - request_state.preferred_account_id = None - request_state.proxy_injected_previous_response_id = False - request_state.fresh_upstream_request_is_retry_safe = False - request_state.responses_lite_model = request_state.fresh_upstream_request_responses_lite_model - _refresh_websocket_request_input_fingerprint_from_text(request_state) + if _install_verified_fresh_replay(request_state) is None: + return None request_text = request_state.request_text if not isinstance(request_text, str): return None @@ -1027,7 +1165,10 @@ def _maybe_rewrite_websocket_previous_response_not_found_event( upstream_control: _WebSocketUpstreamControl, original_text: str, ) -> tuple[OpenAIEvent | None, dict[str, JsonValue] | None, str | None, str]: - error_code = _websocket_event_error_code(event_type, payload) + error_code = _normalize_error_code( + _websocket_event_error_code(event_type, payload), + _websocket_event_error_type(event_type, payload), + ) error_param = _websocket_event_error_param(event_type, payload) error_message = _websocket_event_error_message(event_type, payload) should_rewrite = _facade()._is_previous_response_not_found_error( @@ -1154,6 +1295,11 @@ def _record_websocket_stale_anchor_failure( request_state.failure_phase_override = "upstream" request_state.failure_detail_override = _websocket_stale_anchor_failure_detail(diagnostics) request_state.upstream_error_code_override = upstream_error_code + if not diagnostics.fresh_replay_available: + _remember_websocket_stale_previous_response( + previous_response_id=request_state.previous_response_id, + api_key_id=request_state.api_key.id if request_state.api_key is not None else None, + ) _record_continuity_fail_closed( surface=surface, reason="previous_response_not_found", @@ -1575,6 +1721,7 @@ async def _release_websocket_response_create_gate( request_state: _WebSocketRequestState, response_create_gate: asyncio.Semaphore, ) -> None: + cancellation: asyncio.CancelledError | None = None account_response_create_lease = request_state.account_response_create_lease account_response_create_release = request_state.account_response_create_release request_state.account_response_create_lease = None @@ -1583,13 +1730,36 @@ async def _release_websocket_response_create_gate( request_state.response_create_admission.release() request_state.response_create_admission = None if account_response_create_lease is not None and account_response_create_release is not None: - await account_response_create_release(account_response_create_lease) + cancellation = await _await_cleanup_deferring_cancellation( + account_response_create_release(account_response_create_lease) + ) request_state.awaiting_response_created = False request_state.response_create_gate = None if not request_state.response_create_gate_acquired: + if cancellation is not None: + raise cancellation return request_state.response_create_gate_acquired = False response_create_gate.release() + if cancellation is not None: + raise cancellation + + +async def _await_cleanup_deferring_cancellation(awaitable: Awaitable[object]) -> asyncio.CancelledError | None: + """Finish response-create lease cleanup before propagating cancellation.""" + + task = asyncio.ensure_future(awaitable) + cancellation: asyncio.CancelledError | None = None + with anyio.CancelScope(shield=True): + while True: + try: + await asyncio.shield(task) + break + except asyncio.CancelledError as exc: + cancellation = cancellation or exc + if task.cancelled(): + raise + return cancellation def _pop_terminal_websocket_request_state( @@ -1744,9 +1914,12 @@ def _is_websocket_response_create(payload: dict[str, JsonValue]) -> bool: def _app_error_to_websocket_event(exc: AppError) -> dict[str, JsonValue]: + payload = openai_error(exc.code, exc.message, error_type=getattr(exc, "error_type", "server_error")) + if exc.param is not None: + payload["error"]["param"] = exc.param return _wrapped_websocket_error_event( exc.status_code, - openai_error(exc.code, exc.message, error_type=getattr(exc, "error_type", "server_error")), + payload, ) diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index 371654853b..1c268f7ea9 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -7,7 +7,7 @@ import time from collections import deque from contextlib import contextmanager -from dataclasses import replace +from dataclasses import dataclass, replace from datetime import datetime from typing import Any, Iterator, Mapping, NoReturn, cast @@ -71,7 +71,11 @@ from app.core.exceptions import AppError, ProxyAuthError from app.core.openai.exceptions import ClientPayloadError from app.core.openai.models import OpenAIEvent -from app.core.openai.parsing import parse_sse_event +from app.core.openai.parsing import ( + _LIFECYCLE_EVENT_TYPES, + classify_event_type, + parse_sse_event_payload, +) from app.core.openai.requests import ( ResponsesRequest, ) @@ -84,7 +88,7 @@ from app.core.upstream_proxy import UpstreamProxyRouteError from app.core.utils.request_id import get_request_id, reset_request_id, set_request_id from app.core.utils.sse import CODEX_KEEPALIVE_FRAME as CODEX_KEEPALIVE_FRAME # noqa: F401 -from app.core.utils.sse import format_sse_event, parse_sse_data_json +from app.core.utils.sse import format_sse_event from app.core.utils.time import utcnow as utcnow from app.db.models import ( Account, @@ -96,6 +100,10 @@ ApiKeyInvalidError, ApiKeysService, ) +from app.modules.model_sources.selection import ( + effective_model_for_api_key, + responses_model_is_source_owned, +) from app.modules.proxy._service.api_key_usage import ( _API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS, ) @@ -319,7 +327,6 @@ _clear_websocket_precreated_replay_fallback, _clear_websocket_request_error_overrides, _DownstreamWebSocketActivity, - _event_type_from_payload, _finalize_ttft_reasoning_deltas, _PreparedWebSocketRequest, _record_response_event, @@ -376,8 +383,12 @@ from app.modules.proxy._service.websocket.helpers import ( _app_error_to_websocket_event, _assign_websocket_response_id, + _bind_websocket_request_dispatch_owner, _find_websocket_request_state_by_response_id, + _forget_websocket_stale_previous_response, + _install_verified_fresh_replay, _is_websocket_response_create, + _is_websocket_stale_previous_response, _match_websocket_request_state_for_anonymous_event, _matching_websocket_request_states_for_missing_tool_output_error, _matching_websocket_request_states_for_previous_response_error, @@ -431,9 +442,11 @@ _is_synthesized_turn_state, _owner_lookup_session_id_from_headers, _prompt_cache_key_from_request_model, + _request_allows_unavailable_legacy_owner_abandonment, _sticky_key_for_responses_request, _sticky_key_from_session_header, # noqa: F401 _sticky_key_from_turn_state_header, + _websocket_continuity_aliases_from_headers, ) from app.modules.proxy.api_key_usage import estimate_api_key_request_usage from app.modules.proxy.capability_routing import ( @@ -467,11 +480,14 @@ from app.modules.proxy.request_policy import ( apply_api_key_enforcement, apply_enforced_service_tier_model_fallback, + model_alias_requests_fast_mode, normalize_responses_request_payload, openai_client_payload_error, openai_invalid_payload_error, openai_validation_error, + responses_source_route_excluded, validate_model_access, + validate_top_level_compaction_trigger_input_shape, ) from app.modules.proxy.selection_errors import USAGE_LIMIT_REACHED, selection_failure_response from app.modules.proxy.tool_call_dedupe import ( @@ -490,6 +506,9 @@ def _facade() -> Any: logger = logging.getLogger(__name__) _WEBSOCKET_PINNED_REFRESH_UNAVAILABLE_MESSAGE = "Account refresh is temporarily unavailable; retry later." +# Scope teardown coordinates several request/lease finalizers; keep its normal +# observation budget separate from the short generic child-task cancel bound. +_WEBSOCKET_SCOPE_CLEANUP_TIMEOUT_SECONDS = 5.0 _CAPABILITY_REQUIRED_NO_AUTHORIZED_ACCOUNTS_MESSAGE = ( "This request requires Trusted Access for Cyber, but no eligible account is marked as " "security-work-authorized. codex-lb did not fall back to an ordinary account." @@ -532,23 +551,24 @@ async def _reject_websocket_owner_switch_blocked( api_key: ApiKeyData | None, response_create_gate: asyncio.Semaphore, downstream_activity: _DownstreamWebSocketActivity, -) -> None: - error_message = ( + error_code: str = "previous_response_owner_unavailable", + error_message: str = ( "Previous response owner differs while another response is still streaming; retry after the terminal frame." - ) + ), +) -> None: await proxy._release_websocket_request_state_reservation(request_state) await proxy._write_websocket_connect_failure( account_id=account.id, api_key=api_key, request_state=request_state, - error_code="previous_response_owner_unavailable", + error_code=error_code, error_message=error_message, ) await proxy._emit_websocket_terminal_error( websocket, client_send_lock=client_send_lock, request_state=request_state, - error_code="previous_response_owner_unavailable", + error_code=error_code, error_message=error_message, downstream_activity=downstream_activity, ) @@ -679,35 +699,52 @@ def _websocket_archive_request_state_for_payload( ) +@dataclass(frozen=True, slots=True) +class _ParsedUpstreamWebSocketFrame: + payload: dict[str, JsonValue] | None + event_type: str | None + event: OpenAIEvent | None + + +def _parse_upstream_websocket_text_frame(text: str) -> _ParsedUpstreamWebSocketFrame: + """Decode an upstream websocket text frame exactly once. + + The payload is json-decoded a single time, the event type is classified + from the parsed dict, and pydantic validation runs only for lifecycle + frames (the only events whose validated model fields the proxy consumes). + """ + try: + raw_payload = json.loads(text) + except json.JSONDecodeError: + raw_payload = None + payload = cast(dict[str, JsonValue], raw_payload) if isinstance(raw_payload, dict) else None + event_type = classify_event_type(payload) + event = parse_sse_event_payload(payload) if event_type in _LIFECYCLE_EVENT_TYPES else None + return _ParsedUpstreamWebSocketFrame(payload=payload, event_type=event_type, event=event) + + async def _websocket_archive_request_id_for_message( message: Any, *, pending_requests: deque[_WebSocketRequestState], pending_lock: anyio.Lock, + parsed_frame: _ParsedUpstreamWebSocketFrame | None = None, ) -> str | None: if message.kind != "text" or message.text is None: async with pending_lock: if len(pending_requests) == 1: return pending_requests[0].archive_request_id return None - event_block = f"data: {message.text}\n\n" - payload = parse_sse_data_json(event_block) - if payload is None: - try: - raw_payload = json.loads(message.text) - except json.JSONDecodeError: - raw_payload = None - if isinstance(raw_payload, dict): - payload = cast(dict[str, JsonValue], raw_payload) - event_block = format_sse_event(payload) - event = parse_sse_event(event_block) - event_type = _event_type_from_payload(event, payload) + # Archive attribution only needs the payload dict (response ids and error + # fields are read from it directly), so reuse the caller's parsed frame + # when provided and never re-validate non-lifecycle deltas. + frame = parsed_frame if parsed_frame is not None else _parse_upstream_websocket_text_frame(message.text) async with pending_lock: request_state = _websocket_archive_request_state_for_payload( pending_requests, - event=event, - payload=payload, - event_type=event_type, + event=frame.event, + payload=frame.payload, + event_type=frame.event_type, ) return None if request_state is None else request_state.archive_request_id @@ -785,6 +822,59 @@ def _discard_owned_task(_done_task: asyncio.Task[Any]) -> None: task.add_done_callback(_discard_owned_task) +_WEBSOCKET_UPSTREAM_CLOSE_CLEANUP_TIMEOUT_SECONDS = 0.25 + + +async def _close_websocket_upstream_for_cleanup( + proxy: _WebSocketServiceProtocol, + upstream: UpstreamWebSocket, + *, + timeout_seconds: float, +) -> None: + """Close an upstream socket without letting a stuck close block cleanup. + + Some websocket implementations can wait for a close handshake after the + peer has already disappeared. The close operation remains tracked so it + can finish asynchronously, while scope finalization continues releasing + request ownership and leases within its bounded cleanup budget. + """ + + close_task = asyncio.create_task( + upstream.close(), + name="proxy-websocket-upstream-close", + ) + _track_websocket_owned_task(proxy, close_task) + effective_timeout = min( + max(float(timeout_seconds), 0.0), + _WEBSOCKET_UPSTREAM_CLOSE_CLEANUP_TIMEOUT_SECONDS, + ) + + async def cancel_close_task() -> None: + try: + await _facade()._await_cancelled_task( + close_task, + timeout_seconds=effective_timeout, + label="proxy websocket upstream close", + cleanup_tasks=proxy._background_cleanup_tasks, + ) + except Exception: + _facade().logger.debug("Failed to cancel upstream websocket close task", exc_info=True) + + if effective_timeout <= 0: + await cancel_close_task() + return + try: + await asyncio.wait_for(asyncio.shield(close_task), timeout=effective_timeout) + except TimeoutError: + _facade().logger.debug( + "Upstream websocket close continued after cleanup budget timeout_seconds=%.3f", + effective_timeout, + ) + await cancel_close_task() + except Exception: + _facade().logger.debug("Failed to close upstream websocket during scope cleanup", exc_info=True) + + async def _await_owned_websocket_task_after_reader_cancellation( task: asyncio.Task[Any], *, @@ -872,10 +962,12 @@ async def _process_and_forward_upstream_websocket_text( continuity_state: _WebSocketContinuityState | None, codex_session_affinity: bool, ) -> bool: + parsed_frame = _parse_upstream_websocket_text_frame(text) archive_request_id = await _websocket_archive_request_id_for_message( message, pending_requests=pending_requests, pending_lock=pending_lock, + parsed_frame=parsed_frame, ) _archive_received_websocket_message( upstream, @@ -884,6 +976,7 @@ async def _process_and_forward_upstream_websocket_text( ) downstream_text = await proxy._process_upstream_websocket_text( text, + parsed_frame=parsed_frame, account=account, account_id_value=account_id_value, pending_requests=pending_requests, @@ -1158,6 +1251,45 @@ async def _process_upstream_websocket_transport_end( class _WebSocketMixin: + async def _touch_active_websocket_thread_affinity( + self, + request_state: _WebSocketRequestState, + account: Account, + ) -> None: + """Refresh bounded thread locality without turning it into ownership.""" + + proxy = cast(_WebSocketServiceProtocol, self) + policy = request_state.affinity_policy + if ( + policy.codex_session_source != "thread_header" + or policy.selection_key is None + or policy.kind != StickySessionKind.PROMPT_CACHE + or policy.max_age_seconds is None + ): + return + now = time.monotonic() + touch_interval = max(1.0, min(float(policy.max_age_seconds) / 2.0, 60.0)) + if now - request_state.thread_affinity_last_touch_at < touch_interval: + return + try: + # A response can outlive the selection TTL. Throttled event-time + # touches keep reconnect locality current, while exact response or + # bridge ownership remains the hard authority for this turn. + async with proxy._repo_factory() as repos: + await repos.sticky_sessions.upsert( + policy.selection_key, + account.id, + kind=policy.kind, + ) + except Exception: + _facade().logger.warning( + "Failed to refresh active Codex thread affinity account_id=%s", + account.id, + exc_info=True, + ) + return + request_state.thread_affinity_last_touch_at = now + def _websocket_continuity_state_for_request( self, headers: Mapping[str, str], @@ -1170,28 +1302,37 @@ def _websocket_continuity_state_for_request( _ = proxy if not codex_session_affinity: return _WebSocketContinuityState() - session_id = _owner_lookup_session_id_from_headers(headers, synthesized_turn_state=synthesized_turn_state) api_key_id = api_key.id if api_key is not None else None - cache_keys: list[tuple[str, str | None]] = [] - if session_id is not None: - cache_keys.append((session_id, api_key_id)) - if synthesized_turn_state is not None: - generated_key = (synthesized_turn_state, api_key_id) - if generated_key not in cache_keys: - cache_keys.append(generated_key) + cache_keys = [ + (continuity_key, api_key_id) + for continuity_key in _websocket_continuity_aliases_from_headers( + headers, + synthesized_turn_state=synthesized_turn_state, + ) + ] if not cache_keys: return _WebSocketContinuityState() + explicit_turn_state = _sticky_key_from_turn_state_header(headers) + exact_client_turn = explicit_turn_state is not None and explicit_turn_state != synthesized_turn_state + # An exact client turn state is hard continuity. If its alias is + # unknown, do not borrow retained response/tool state from the broader + # thread key; the turn may have a different owner. Once the exact alias + # resolves, publishing that same state under the thread key is safe and + # keeps a later unanchored reconnect thread-local. + lookup_keys = cache_keys[:1] if exact_client_turn else cache_keys continuity_state = next( ( existing_state - for key in cache_keys + for key in lookup_keys if (existing_state := proxy._websocket_continuity_index.get(key)) is not None ), None, ) + exact_alias_resolved = continuity_state is not None if continuity_state is None: continuity_state = _WebSocketContinuityState() - for key in cache_keys: + publish_keys = cache_keys if not exact_client_turn or exact_alias_resolved else lookup_keys + for key in publish_keys: proxy._websocket_continuity_index.pop(key, None) proxy._websocket_continuity_index[key] = continuity_state while len(proxy._websocket_continuity_index) > _facade()._WEBSOCKET_CONTINUITY_CACHE_LIMIT: @@ -1238,7 +1379,14 @@ async def proxy_responses_websocket( account_lease: AccountLease | None = None upstream_requires_security_work_authorized: bool | None = None upstream_turn_state: str | None = _sticky_key_from_turn_state_header(headers) - client_turn_state_header: str | None = _sticky_key_from_turn_state_header(filtered_headers) + # The API inserts its generated downstream turn state into ``headers`` + # before entering this service. Preserve a turn-state header as + # client-owned only when no synthesized value accompanied it; otherwise + # account-switch cleanup must remain able to remove the old account's + # generated token from ``filtered_headers``. + client_turn_state_header: str | None = ( + _sticky_key_from_turn_state_header(filtered_headers) if synthesized_turn_state is None else None + ) upstream_account_id: str | None = None downstream_activity = _DownstreamWebSocketActivity() replay_request_state: _WebSocketRequestState | None = None @@ -1457,6 +1605,7 @@ def take_reader_replay_request_state() -> _WebSocketRequestState | None: if not await proxy._downstream_websocket_is_idle( pending_requests, pending_lock=pending_lock, + upstream_control=upstream_control, downstream_activity=downstream_activity, idle_timeout_seconds=downstream_idle_timeout_seconds, ): @@ -1466,6 +1615,7 @@ def take_reader_replay_request_state() -> _WebSocketRequestState | None: if await proxy._downstream_websocket_is_idle( pending_requests, pending_lock=pending_lock, + upstream_control=upstream_control, downstream_activity=downstream_activity, idle_timeout_seconds=downstream_idle_timeout_seconds, ): @@ -1590,6 +1740,81 @@ def take_reader_replay_request_state() -> _WebSocketRequestState | None: request_state = prepared_request.request_state request_affinity = prepared_request.affinity_policy text_data = prepared_request.text_data + if ( + upstream is not None + and account is not None + # A reader that has already finished means the upstream is + # gone but the cleanup that nulls it runs further below, so + # without this the turn would take the reuse path (terminal + # error) when it should reconnect and take the connect path + # (503, which the client transparently falls back from). + and upstream_reader is not None + and not upstream_reader.done() + # Requests the HTTP route excludes from + # source routing (a terminal compaction + # trigger, ``input_file`` references) + # must stay on subscription accounts even + # when their model is also source-owned; + # the owner-routing below dispatches them + # to the pinned account instead of this + # guard failing the turn. + and not request_state.source_route_excluded + and await responses_model_is_source_owned( + request_state.model, + request_state.api_key or api_key, + # The raw client model, before enforcement + # normalized aliases: an alias-only source + # (``gpt-5-high``) is invisible in the + # normalized ``request_state.model``. + raw_model=request_state.raw_source_model, + ) + ): + # Socket reuse bypasses connect-time selection, so a later + # response.create that switches to a source-owned model + # would otherwise be forwarded to the subscription account + # already attached to the open upstream. Model sources are + # only reachable from the HTTP request path. + # + # Gated on an existing upstream on purpose: a first turn has + # no socket yet and must fall through to the connect guard, + # which fails with a service-level 503 so the client falls + # back to HTTP. Emitting a terminal error here would preempt + # that fallback and make source models unreachable. + source_model = request_state.raw_source_model or request_state.model + source_message = ( + f"Model {source_model!r} is served by an " + "OpenAI-compatible model source, which is only reachable " + "over the HTTP transport; retry the request over HTTPS." + ) + _facade().logger.info( + "Websocket model source requires http transport " + "request_id=%s model=%s raw_model=%s stage=response_create", + request_state.request_log_id or request_state.request_id, + request_state.model, + request_state.raw_source_model, + ) + await proxy._release_websocket_request_state_reservation(request_state) + # The prepared request already owns a request-log row; without + # this the row is never finalized, so the same logical failure + # is only visible in request logs when it happens on the first + # turn (where the connect path writes it). + await proxy._write_websocket_connect_failure( + account_id=account.id, + api_key=request_state.api_key or api_key, + request_state=request_state, + error_code="model_source_requires_http_transport", + error_message=source_message, + ) + await proxy._emit_websocket_terminal_error( + websocket, + client_send_lock=client_send_lock, + request_state=request_state, + error_code="model_source_requires_http_transport", + error_message=source_message, + error_type="invalid_request_error", + downstream_activity=downstream_activity, + ) + continue except ProxyResponseError as exc: ( status_code, @@ -1792,6 +2017,53 @@ def take_reader_replay_request_state() -> _WebSocketRequestState | None: payload = None continue + if ( + request_state is not None + and upstream is not None + and account is not None + and request_state.affinity_policy.abandon_unavailable_legacy_owner + ): + # Reusing the existing socket would bypass sticky + # selection, so the unavailable raw owner would never be + # compared, tombstoned, or replaced. A restart is movable + # only before dispatch and cannot retire a socket that + # still owns another response. + async with pending_lock: + restart_switch_blocked = _websocket_owner_switch_has_other_pending_requests( + request_state, + pending_requests, + ) + if restart_switch_blocked: + await _reject_websocket_owner_switch_blocked( + proxy, + websocket, + client_send_lock=client_send_lock, + request_state=request_state, + account=account, + api_key=api_key, + response_create_gate=response_create_gate, + downstream_activity=downstream_activity, + error_code="stream_incomplete", + error_message=( + "Goal restart cannot switch accounts while another response is still streaming; " + "retry after the terminal frame." + ), + ) + request_state = None + text_data = None + payload = None + continue + await retire_current_upstream() + upstream_turn_state = None + if client_turn_state_header is None: + # Provenance was captured before this switch: absence + # here means the API synthesized the forwarded token. + # Such account-local state must die with its upstream; + # an actual client anchor remains fail-closed instead. + filtered_headers = { + key: value for key, value in filtered_headers.items() if key.lower() != "x-codex-turn-state" + } + if ( request_state is not None and upstream is not None @@ -2305,6 +2577,19 @@ def take_reader_replay_request_state() -> _WebSocketRequestState | None: if text_data is not None: archive_request_id = None if request_state is None else request_state.archive_request_id if request_state is not None and payload is not None and _is_websocket_response_create(payload): + if account is None or not _bind_websocket_request_dispatch_owner( + request_state, + account_id=account.id, + exact_request_text=text_data, + ): + raise ProxyResponseError( + 502, + openai_error( + "previous_response_owner_unavailable", + "Request payload owner account is unavailable; retry later.", + error_type="server_error", + ), + ) request_state.response_create_sent_at = time.monotonic() with _websocket_archive_request_context(archive_request_id): await upstream.send_text(text_data) @@ -2453,11 +2738,19 @@ def take_reader_replay_request_state() -> _WebSocketRequestState | None: scope_cancelled = True raise finally: - cleanup_timeout = shutdown_state.remaining_drain_timeout_seconds() - if cleanup_timeout is None: - cleanup_timeout = _facade()._TASK_CANCEL_TIMEOUT_SECONDS + remaining_drain_timeout = shutdown_state.remaining_drain_timeout_seconds() + cleanup_timeout = ( + _WEBSOCKET_SCOPE_CLEANUP_TIMEOUT_SECONDS + if remaining_drain_timeout is None + else max(float(remaining_drain_timeout), 0.0) + ) + task_cleanup_timeout = ( + _facade()._TASK_CANCEL_TIMEOUT_SECONDS if remaining_drain_timeout is None else cleanup_timeout + ) + cleanup_phase = "not_started" async def finalize_websocket_scope() -> None: + nonlocal cleanup_phase nonlocal replay_request_state nonlocal request_state_failure_task nonlocal request_state_to_fail @@ -2470,16 +2763,20 @@ async def finalize_websocket_scope() -> None: # release that wait. reader_to_await.cancel() if upstream is not None: - try: - await upstream.close() - except Exception: - _facade().logger.debug("Failed to close upstream websocket", exc_info=True) + cleanup_phase = "upstream_close" + await _close_websocket_upstream_for_cleanup( + proxy, + upstream, + timeout_seconds=task_cleanup_timeout, + ) if reader_to_await is not None: try: + cleanup_phase = "upstream_reader" await _facade()._await_cancelled_task( reader_to_await, label="proxy websocket upstream reader", cancel=False, + cleanup_tasks=proxy._background_cleanup_tasks, ) except Exception: # Reader failure must not skip lease release or the @@ -2491,9 +2788,10 @@ async def finalize_websocket_scope() -> None: upstream_reader = None if retired_create_lease_release_task is not None: try: + cleanup_phase = "retired_create_lease" await _facade()._await_cancelled_task( retired_create_lease_release_task, - timeout_seconds=cleanup_timeout, + timeout_seconds=task_cleanup_timeout, label="proxy websocket retired create lease release", cancel=False, ) @@ -2505,9 +2803,10 @@ async def finalize_websocket_scope() -> None: retired_create_lease_release_task = None if request_state_failure_task is not None: try: + cleanup_phase = "unsent_request" await _facade()._await_cancelled_task( request_state_failure_task, - timeout_seconds=cleanup_timeout, + timeout_seconds=task_cleanup_timeout, label="proxy websocket unsent request finalization", cancel=False, ) @@ -2521,6 +2820,7 @@ async def finalize_websocket_scope() -> None: replay_request_state = upstream_control.replay_request_state upstream_control.replay_request_state = None if request_state_to_fail is not None: + cleanup_phase = "unsent_request" await proxy._fail_pending_websocket_requests( account=None, account_id_value=account.id if account is not None else upstream_account_id, @@ -2539,6 +2839,7 @@ async def finalize_websocket_scope() -> None: ) request_state_to_fail = None if replay_request_state is not None: + cleanup_phase = "replay_request" await proxy._fail_pending_websocket_requests( account=None, account_id_value=account.id if account is not None else upstream_account_id, @@ -2556,6 +2857,7 @@ async def finalize_websocket_scope() -> None: penalize_account=False, ) client_disconnected = downstream_activity.disconnected + cleanup_phase = "pending_requests" await proxy._fail_pending_websocket_requests( account=None if client_disconnected or scope_cancelled else account, account_id_value=account.id if account is not None else upstream_account_id, @@ -2578,6 +2880,7 @@ async def finalize_websocket_scope() -> None: penalize_account=not (client_disconnected or scope_cancelled), ) try: + cleanup_phase = "connection_lease" await release_current_account_lease() except Exception: # Connection-lease cleanup must never replace cancellation @@ -2586,6 +2889,7 @@ async def finalize_websocket_scope() -> None: "Failed to release websocket connection lease during scope cleanup", exc_info=True, ) + cleanup_phase = "complete" cleanup_task = asyncio.create_task( finalize_websocket_scope(), @@ -2609,7 +2913,13 @@ def log_scope_cleanup_failure(done_task: asyncio.Task[None]) -> None: timeout=max(float(cleanup_timeout), 0.0), ) if not done: - _facade().logger.warning("Websocket scope cleanup exceeded its remaining drain budget") + _facade().logger.warning( + "Websocket scope cleanup exceeded its cleanup budget " + "timeout_seconds=%.3f cleanup_phase=%s background_cleanup_tasks=%d", + max(float(cleanup_timeout), 0.0), + cleanup_phase, + sum(1 for task in proxy._background_cleanup_tasks if not task.done()), + ) async def _prepare_websocket_response_create_request( self, @@ -2641,19 +2951,44 @@ async def _prepare_websocket_response_create_request( header_values=capability_header_values, client_metadata_values=_websocket_capability_metadata_values(payload), ) + validate_top_level_compaction_trigger_input_shape(payload) responses_payload = normalize_responses_request_payload( payload, openai_compat=openai_cache_affinity, ) + # The client's raw model, captured before enforcement normalizes + # aliases (``gpt-5-high`` -> ``gpt-5``). The source-ownership guards + # must judge the raw alias too, or an alias-only model source is + # missed on the WebSocket paths while the HTTP path routes the same + # request via ``raw_source_model``. Mirrors ``api.py::responses`` + # exactly, including the enforced-model substitution here and the + # fast-mode correction after enforcement below. + raw_source_model = effective_model_for_api_key(refreshed_api_key, responses_payload.model) + # The effort the normalizer replaced is discarded here on purpose: the + # WebSocket transport never reaches a model source, so the rewrite that + # works around the backend hang must stick. service_tier_was_enforced = apply_api_key_enforcement( responses_payload, refreshed_api_key, prohibit_fast_mode=prohibit_fast_mode, - ) + ).service_tier_was_enforced + if prohibit_fast_mode and model_alias_requests_fast_mode(raw_source_model): + raw_source_model = responses_payload.model apply_enforced_service_tier_model_fallback( responses_payload, service_tier_was_enforced=service_tier_was_enforced, ) + # Judged on the full client input, before the websocket-specific + # trimming and anchor injection below rewrite it — the same payload + # the HTTP route evaluates for its source-selection gate. + try: + source_route_excluded = responses_source_route_excluded(responses_payload) + except ClientPayloadError: + # HTTP rejects a malformed compaction trigger with a 400; the + # WebSocket path has always forwarded such frames verbatim, so a + # parse failure keeps the source guards active instead of + # changing that behavior here. + source_route_excluded = False normalized_payload = responses_payload.to_payload() stripped_client_metadata = strip_capability_metadata(normalized_payload.get("client_metadata")) if stripped_client_metadata is not normalized_payload.get("client_metadata"): @@ -2724,11 +3059,21 @@ async def _prepare_websocket_response_create_request( original_full_resend_payload: ResponsesRequest | None = None original_input_item_count: int | None = None original_input_fingerprint: str | None = None - session_anchor = _websocket_continuity_anchor_for_payload( - continuity_state, - responses_payload=responses_payload, - codex_session_affinity=codex_session_affinity, - ) + # Classify restart authority from the complete normalized client body, + # before ordinary direct-WebSocket continuity injects a + # ``previous_response_id`` and trims historical input. That injected + # anchor is account-owned and would both erase the restart capability + # and make the payload unsafe for the replacement account. A proven + # goal restart must retain the complete resend through selection. + goal_restart_full_resend = _request_allows_unavailable_legacy_owner_abandonment(responses_payload) + restart_affinity_payload = responses_payload + session_anchor = None + if not goal_restart_full_resend: + session_anchor = _websocket_continuity_anchor_for_payload( + continuity_state, + responses_payload=responses_payload, + codex_session_affinity=codex_session_affinity, + ) if session_anchor is not None: original_input_items = cast(list[JsonValue], responses_payload.input) original_input_item_count = len(original_input_items) @@ -2809,6 +3154,8 @@ async def _prepare_websocket_response_create_request( request_state.useragent_group = useragent_group request_state.conversation_id = conversation_id request_state.client_ip = client_ip + request_state.raw_source_model = raw_source_model + request_state.source_route_excluded = source_route_excluded request_state.responses_lite_model = next_responses_lite_model request_state.expose_stale_previous_response_classifier = codex_session_affinity request_state.require_security_work_authorized = capability_route.require_security_work_authorized @@ -2892,7 +3239,10 @@ async def _prepare_websocket_response_create_request( request_state.input_item_count, ) affinity_policy = _sticky_key_for_responses_request( - responses_payload, + # Only the proven restart uses the pre-injection body. Ordinary + # full resends must be classified after anchor injection so they + # cannot accidentally gain soft-session mobility. + restart_affinity_payload if goal_restart_full_resend else responses_payload, headers, codex_session_affinity=codex_session_affinity, openai_cache_affinity=openai_cache_affinity, @@ -2902,7 +3252,9 @@ async def _prepare_websocket_response_create_request( synthesized_turn_state=synthesized_turn_state, ) sticky_key_source = "none" - if affinity_policy.kind == StickySessionKind.CODEX_SESSION: + if affinity_policy.codex_session_source == "thread_header": + sticky_key_source = "thread_header" + elif affinity_policy.kind == StickySessionKind.CODEX_SESSION: turn_state_key = _sticky_key_from_turn_state_header(headers) if turn_state_key is not None and turn_state_key == synthesized_turn_state: sticky_key_source = "generated_turn_state" @@ -3038,6 +3390,65 @@ async def _record_or_defer_confirmed_route_backoff(account: Account) -> None: request_transport="websocket", ), ) + # Model sources are only reachable from the HTTP request path. Fail the + # WebSocket connect instead of dispatching a source-owned model to a + # subscription account, which the upstream rejects with "The '' + # model is not supported when using Codex with a ChatGPT account." + # Codex clients fall back to the HTTP transport when a WebSocket + # connect fails, and that path routes to the source correctly. + # + # Evaluated once per connect series rather than inside the failover + # loop below: source ownership is a property of the requested model, so + # re-resolving it per attempt would only repeat the same lookup. The + # per-request api key is used (rather than the session key) so a policy + # refresh mid-session cannot make this disagree with the equivalent + # check on the prepared-request path. + # + # Requests the HTTP route excludes from source routing (a terminal + # compaction trigger, ``input_file`` references pinned to the + # uploading account) skip the guard: they must land on a subscription + # account either way, and the owner-required selection below routes + # them there instead of bouncing the turn to HTTP. + if not request_state.source_route_excluded and await responses_model_is_source_owned( + model, + request_state.api_key or api_key, + # ``model`` is the session loop's post-enforcement + # ``request_state.model``; the raw client alias captured at + # preparation is what an alias-only source is registered under. + raw_model=request_state.raw_source_model, + ): + source_model = request_state.raw_source_model or model + message = ( + f"Model {source_model!r} is served by an OpenAI-compatible model source, which is only " + "reachable over the HTTP transport; retry the request over HTTPS." + ) + _facade().logger.info( + "Websocket model source requires http transport request_id=%s model=%s raw_model=%s api_key_present=%s", + request_state.request_log_id or request_state.request_id, + model, + request_state.raw_source_model, + (request_state.api_key or api_key) is not None, + ) + await proxy._emit_websocket_connect_failure( + websocket, + client_send_lock=client_send_lock, + account_id=None, + api_key=request_state.api_key or api_key, + request_state=request_state, + # 503 (not 4xx) is deliberate: Codex clients only fall back to + # the HTTP transport when a WebSocket connect fails at the + # service level. A 4xx is treated as terminal and surfaces to + # the user instead of retrying over HTTPS. + status_code=503, + payload=openai_error( + "model_source_requires_http_transport", + message, + error_type="server_error", + ), + error_code="model_source_requires_http_transport", + error_message=message, + ) + return None, None max_attempts = _facade()._WEBSOCKET_MAX_ACCOUNT_ATTEMPTS excluded_account_ids: set[str] = set(request_state.excluded_account_ids) last_failover_exc: ProxyResponseError | None = None @@ -3045,13 +3456,18 @@ async def _record_or_defer_confirmed_route_backoff(account: Account) -> None: for attempt in range(max_attempts): is_retry = attempt > 0 forced_refresh_account_id = request_state.force_refresh_account_id - preferred_account_id = forced_refresh_account_id or request_state.preferred_account_id + preferred_account_id = ( + request_state.replay_required_account_id + or forced_refresh_account_id + or request_state.preferred_account_id + ) turn_state_owner_required = ( request_state.affinity_policy.codex_session_source == "turn_state" and request_state.preferred_account_id is not None ) require_preferred_account = ( (request_state.previous_response_id is not None and request_state.preferred_account_id is not None) + or request_state.replay_required_account_id is not None or request_state.file_required_preferred_account or turn_state_owner_required ) @@ -3292,7 +3708,11 @@ async def _select_websocket_connect_account( reallocate_sticky=reallocate_sticky, sticky_source=request_state.affinity_policy.codex_session_source, legacy_sticky_key=request_state.affinity_policy.legacy_selection_key, + legacy_continuity_source=request_state.affinity_policy.legacy_continuity_source, + sticky_seed_key=request_state.affinity_policy.seed_selection_key, + sticky_seed_kind=request_state.affinity_policy.seed_selection_kind, spill_bare_session_on_account_cap=request_state.affinity_policy.spill_on_account_cap, + abandon_unavailable_legacy_owner=(request_state.affinity_policy.abandon_unavailable_legacy_owner), require_unambiguous_account=request_state.affinity_policy.require_unambiguous_account, sticky_max_age_seconds=sticky_max_age_seconds, prefer_earlier_reset_accounts=prefer_earlier_reset, @@ -3361,6 +3781,14 @@ async def _heartbeat(remaining_seconds: float) -> None: break account = selection.account + if ( + account is not None + and request_state.replay_required_account_id is None + and request_state.request_text is not None + and not _facade()._websocket_request_text_is_account_neutral_fresh_replay(request_state.request_text) + ): + request_state.preferred_account_id = account.id + request_state.replay_required_account_id = account.id if ( account is not None and require_preferred_account @@ -4097,6 +4525,10 @@ def _remember_websocket_previous_response_owner( account_id_value = account_id.strip() if not account_id_value: return + _forget_websocket_stale_previous_response( + previous_response_id=response_id, + api_key_id=api_key_id, + ) cache_keys = [(response_id, api_key_id, None)] normalized_session_id = _facade()._normalize_session_id(session_id) if normalized_session_id is not None: @@ -4133,6 +4565,7 @@ async def _resolve_websocket_previous_response_owner( session_id: str | None = None, surface: str, request_state: _WebSocketRequestState | None = None, + force_request_log_lookup: bool = False, ) -> str | None: proxy = cast(_WebSocketServiceProtocol, self) _ = proxy @@ -4151,6 +4584,24 @@ def _record_lookup_metadata( request_state.previous_response_owner_requested_at = requested_at request_state.previous_response_owner_session_id = owner_session_id + def _raise_stale_response_cache_suppression(*, outcome: str) -> NoReturn: + _record_lookup_metadata(source="stale_response_cache", outcome=outcome) + _record_continuity_owner_resolution( + surface=surface, + source="stale_response_cache", + outcome=outcome, + previous_response_id=response_id, + session_id=session_id_value, + ) + raise ProxyResponseError( + 502, + openai_error( + "stream_incomplete", + "Previous response is temporarily unavailable; retrying is suppressed for the recovery window.", + error_type="server_error", + ), + ) + if previous_response_id is None: return None response_id = previous_response_id.strip() @@ -4158,8 +4609,19 @@ def _record_lookup_metadata( return None api_key_id = api_key.id if api_key is not None else None session_id_value = _facade()._normalize_session_id(session_id) + stale_cache_hit = ( + request_state is not None + and not force_request_log_lookup + and not request_state.fresh_upstream_request_is_retry_safe + and _is_websocket_stale_previous_response( + previous_response_id=response_id, + api_key_id=api_key_id, + ) + ) cache_key = (response_id, api_key_id, session_id_value) - cached_account_id = proxy._websocket_previous_response_account_index.get(cache_key) + cached_account_id = ( + None if force_request_log_lookup else proxy._websocket_previous_response_account_index.get(cache_key) + ) if cached_account_id is not None: _record_lookup_metadata(source="request_cache", outcome="hit") _record_continuity_owner_resolution( @@ -4171,9 +4633,13 @@ def _record_lookup_metadata( ) return cached_account_id fallback_account_id = ( - proxy._websocket_previous_response_account_index.get((response_id, api_key_id, None)) - if session_id_value is not None - else None + None + if force_request_log_lookup + else ( + proxy._websocket_previous_response_account_index.get((response_id, api_key_id, None)) + if session_id_value is not None + else None + ) ) try: async with proxy._repo_factory() as repos: @@ -4183,6 +4649,8 @@ def _record_lookup_metadata( session_id=session_id_value, ) except Exception as exc: + if stale_cache_hit: + _raise_stale_response_cache_suppression(outcome="lookup_failed") if fallback_account_id is not None: _record_lookup_metadata(source="request_cache_fallback", outcome="hit") _record_continuity_owner_resolution( @@ -4217,6 +4685,12 @@ def _record_lookup_metadata( _facade()._previous_response_owner_lookup_failed_error_envelope(), ) from exc if owner_record is None: + if stale_cache_hit: + _raise_stale_response_cache_suppression(outcome="hit") + if force_request_log_lookup: + proxy._websocket_previous_response_account_index.pop(cache_key, None) + if session_id_value is not None: + proxy._websocket_previous_response_account_index.pop((response_id, api_key_id, None), None) if fallback_account_id is not None: _record_lookup_metadata(source="request_cache_fallback", outcome="hit") _record_continuity_owner_resolution( @@ -4619,21 +5093,15 @@ async def _process_upstream_websocket_text( response_create_gate: asyncio.Semaphore, continuity_state: "_WebSocketContinuityState | None" = None, codex_session_affinity: bool = False, + parsed_frame: _ParsedUpstreamWebSocketFrame | None = None, ) -> str: proxy = cast(_WebSocketServiceProtocol, self) _ = proxy - event_block = f"data: {text}\n\n" - payload = parse_sse_data_json(event_block) - if payload is None: - try: - raw_payload = json.loads(text) - except json.JSONDecodeError: - raw_payload = None - if isinstance(raw_payload, dict): - payload = cast(dict[str, JsonValue], raw_payload) - event_block = format_sse_event(payload) - event = parse_sse_event(event_block) - event_type = _event_type_from_payload(event, payload) + if parsed_frame is None: + parsed_frame = _parse_upstream_websocket_text_frame(text) + payload = parsed_frame.payload + event_type = parsed_frame.event_type + event = parsed_frame.event response_id = _websocket_response_id(event, payload) error_message = _websocket_event_error_message(event_type, payload) is_typeless_error_event = ( @@ -4658,10 +5126,14 @@ async def _process_upstream_websocket_text( message=error_message, ) previous_response_id_hint = _facade()._previous_response_id_from_not_found_message(error_message) + # The returned event block is unused here; the rewrite helper rebuilds + # its own canonical block on the (rare) changed path, so avoid the + # per-frame ``format_sse_event`` re-encode and pass the raw framing. text, payload, event, event_type, _event_block = rewrite_parallel_tool_call_text( text, payload, - event_block=format_sse_event(payload) if payload is not None else f"data: {text}\n\n", + event_block=f"data: {text}\n\n", + event=event, ) async with pending_lock: @@ -4745,8 +5217,10 @@ async def _process_upstream_websocket_text( request_state.suppress_next_created_downstream = False upstream_control.suppress_downstream_event = True if payload is not None: - payload = _rewrite_websocket_downstream_response_id(payload, request_state) - text = json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + rewritten_payload = _rewrite_websocket_downstream_response_id(payload, request_state) + if rewritten_payload is not payload: + payload = rewritten_payload + text = json.dumps(payload, ensure_ascii=True, separators=(",", ":")) sequence_number = payload.get("sequence_number") if isinstance(sequence_number, int) and not isinstance(sequence_number, bool): upstream_control.downstream_sequence_request_state = request_state @@ -4861,6 +5335,9 @@ async def _process_upstream_websocket_text( if event_type == "response.created" and release_create_gate and created_request_state is not None: await _release_websocket_response_create_gate(created_request_state, response_create_gate) + if request_state is not None: + await proxy._touch_active_websocket_thread_affinity(request_state, account) + if len(grouped_previous_response_request_states) > 1: upstream_control.reconnect_requested = True downstream_texts: list[str] = [] @@ -5095,18 +5572,21 @@ async def _process_upstream_websocket_text( # transparently retried. retry_error_code = None else: - upstream_control.reconnect_requested = True - request_state.request_text = request_state.fresh_upstream_request_text - request_state.previous_response_id = None - request_state.proxy_injected_previous_response_id = False - request_state.fresh_upstream_request_is_retry_safe = False - request_state.responses_lite_model = request_state.fresh_upstream_request_responses_lite_model - request_state.replay_count += 1 - request_state.awaiting_response_created = True - request_state.response_id = None - _clear_websocket_request_error_overrides(request_state) - upstream_control.suppress_downstream_event = True - upstream_control.replay_request_state = request_state + replay_text = _install_verified_fresh_replay( + request_state, + require_proxy_injected_previous_response_id=False, + require_account_neutral=False, + ) + if replay_text is None: + retry_error_code = None + else: + upstream_control.reconnect_requested = True + request_state.replay_count += 1 + request_state.awaiting_response_created = True + request_state.response_id = None + _clear_websocket_request_error_overrides(request_state) + upstream_control.suppress_downstream_event = True + upstream_control.replay_request_state = request_state else: upstream_control.reconnect_requested = True request_state.replay_count += 1 @@ -5229,10 +5709,31 @@ async def _handle_precreated_websocket_auth_failure( ) -> bool: proxy = cast(_WebSocketServiceProtocol, self) _ = proxy - if _prepare_websocket_request_state_for_auth_replay(request_state) is None: + bound_to_current_account = request_state.replay_required_account_id == account.id + requires_reauth = _websocket_auth_failure_requires_reauth(error_message) + if bound_to_current_account and ( + requires_reauth or request_state.auth_replay_counts_by_account.get(account.id, 0) > 0 + ): + failure_code = ( + _facade()._WEBSOCKET_SESSION_EXPIRED_FAILURE_CODE + if requires_reauth + else _facade()._WEBSOCKET_AUTH_INVALIDATED_FAILURE_CODE + ) + await proxy._load_balancer.mark_permanent_failure(account, failure_code) + request_state.force_refresh_account_id = None + request_state.preferred_account_id = None + request_state.excluded_account_ids.add(account.id) + return False + if ( + _prepare_websocket_request_state_for_auth_replay( + request_state, + current_account_id=account.id, + ) + is None + ): return False - if _websocket_auth_failure_requires_reauth(error_message): + if requires_reauth: failure_code = _facade()._WEBSOCKET_SESSION_EXPIRED_FAILURE_CODE elif request_state.auth_replay_counts_by_account.get(account.id, 0) == 0: request_state.auth_replay_counts_by_account[account.id] = 1 @@ -5329,11 +5830,16 @@ async def _downstream_websocket_is_idle( pending_requests: deque[_WebSocketRequestState], *, pending_lock: anyio.Lock, + upstream_control: _WebSocketUpstreamControl | None = None, downstream_activity: _DownstreamWebSocketActivity, idle_timeout_seconds: float, ) -> bool: proxy = cast(_WebSocketServiceProtocol, self) _ = proxy + if upstream_control is not None: + terminal_task = upstream_control.terminal_message_task + if terminal_task is not None and not terminal_task.done(): + return False async with pending_lock: if pending_requests: return False @@ -5830,10 +6336,10 @@ async def _fail_pending_websocket_requests( status: str = "error", penalize_account: bool = True, suppress_sequenced_downstream_errors: bool = False, - ) -> None: + ) -> bool: proxy = cast(_WebSocketServiceProtocol, self) _ = proxy - finalization_task: asyncio.Task[None] | None = None + finalization_task: asyncio.Task[bool] | None = None await pending_lock.acquire() try: remaining = list(pending_requests) @@ -5865,10 +6371,10 @@ async def _fail_pending_websocket_requests( pending_lock.release() if finalization_task is None: - return + return True try: - await asyncio.shield(finalization_task) + settlement_succeeded = await asyncio.shield(finalization_task) except asyncio.CancelledError: remaining_timeout = shutdown_state.remaining_drain_timeout_seconds() timeout_seconds = ( @@ -5881,6 +6387,7 @@ async def _fail_pending_websocket_requests( # the claimed states and remains visible to lifespan draining. await asyncio.wait({finalization_task}, timeout=timeout_seconds) raise + return settlement_succeeded async def _finalize_claimed_websocket_requests( self, @@ -5898,7 +6405,7 @@ async def _finalize_claimed_websocket_requests( status: str, penalize_account: bool, suppress_sequenced_downstream_errors: bool, - ) -> None: + ) -> bool: proxy = cast(_WebSocketServiceProtocol, self) _ = proxy @@ -5967,6 +6474,7 @@ async def _finalize_claimed_websocket_requests( request_state.request_log_id or request_state.request_id, exc_info=True, ) + if response_create_gate is not None: await _release_websocket_response_create_ownership_for_cleanup( request_state, @@ -6133,6 +6641,8 @@ async def _finalize_claimed_websocket_requests( exc_info=True, ) + return reservation_release_succeeded + async def _emit_websocket_terminal_error( self, websocket: WebSocket, diff --git a/app/modules/proxy/_service/websocket/protocol.py b/app/modules/proxy/_service/websocket/protocol.py index 519cd7c868..f0dcd2c343 100644 --- a/app/modules/proxy/_service/websocket/protocol.py +++ b/app/modules/proxy/_service/websocket/protocol.py @@ -59,6 +59,7 @@ class _WebSocketServiceProtocol(Protocol): _settle_stream_api_key_usage: Any _start_request_state_api_key_reservation_heartbeat: Any _try_open_websocket_connect_attempt: Any + _touch_active_websocket_thread_affinity: Any _websocket_continuity_index: Any _websocket_continuity_state_for_request: Any _websocket_previous_response_account_index: Any diff --git a/app/modules/proxy/_support.py b/app/modules/proxy/_support.py index b9d401a16f..44a28f5ca0 100644 --- a/app/modules/proxy/_support.py +++ b/app/modules/proxy/_support.py @@ -36,9 +36,6 @@ from app.modules.proxy._service.support import ( _event_type_from_payload as _event_type_from_payload, ) -from app.modules.proxy._service.support import ( - _FilePinEntry as _FilePinEntry, -) from app.modules.proxy._service.support import ( _HTTPBridgeOwnerForward as _HTTPBridgeOwnerForward, ) diff --git a/app/modules/proxy/affinity.py b/app/modules/proxy/affinity.py index 52e941c45f..03fe030d33 100644 --- a/app/modules/proxy/affinity.py +++ b/app/modules/proxy/affinity.py @@ -13,17 +13,23 @@ from collections.abc import Mapping from dataclasses import dataclass, replace from hashlib import sha256 -from typing import Literal, cast +from typing import Literal, TypedDict, cast from uuid import uuid4 from app.core.config.settings import get_settings -from app.core.openai.requests import ResponsesCompactRequest, ResponsesRequest, extract_input_file_ids +from app.core.openai.requests import ( + ResponsesCompactRequest, + ResponsesRequest, + extract_input_file_ids, + responses_request_contains_goal_continuation_context, +) from app.db.models import StickySessionKind from app.modules.api_keys.service import ApiKeyData +from app.modules.proxy.replay_safety import responses_payload_is_account_neutral_fresh_replay # This typed provenance is a routing capability: callers must never recover it # from key text, because a client-controlled turn state can mimic any prefix. -_CodexSessionSource = Literal["session_header", "turn_state"] +_CodexSessionSource = Literal["session_header", "thread_header", "turn_state"] # Request headers are stripped and HTTP forbids CR/LF, while PostgreSQL/SQLite # text keys can safely retain LF. This sentinel makes the internal namespace # structurally unreachable by every legacy raw header, even if its digest is @@ -31,6 +37,21 @@ _CODEX_SELECTION_KEY_PREFIX = "\ncodex-lb-affinity-v1" +class _AffinitySelectionKwargs(TypedDict): + sticky_key: str | None + sticky_kind: StickySessionKind | None + reallocate_sticky: bool + sticky_source: _CodexSessionSource | None + legacy_sticky_key: str | None + legacy_continuity_source: _CodexSessionSource | None + sticky_seed_key: str | None + sticky_seed_kind: StickySessionKind | None + spill_bare_session_on_account_cap: bool + abandon_unavailable_legacy_owner: bool + require_unambiguous_account: bool + sticky_max_age_seconds: int | None + + @dataclass(frozen=True, slots=True) class _AffinityPolicy: key: str | None = None @@ -39,8 +60,26 @@ class _AffinityPolicy: # Source capability only. Shared selection still revokes spillover for a # required owner or any stage that may carry account-local state. spill_on_account_cap: bool = False + # An explicit, self-contained Codex goal restart may retire only a raw + # compatibility owner whose durable account status is unavailable. + abandon_unavailable_legacy_owner: bool = False max_age_seconds: int | None = None codex_session_source: _CodexSessionSource | None = None + # A thread row is soft locality, but old replicas may have persisted the + # raw process/session value as hard CODEX_SESSION ownership. Keep that + # compatibility lookup explicit instead of trying to reconstruct it from + # the new opaque thread key. + legacy_codex_session_key: str | None = None + # Interpretation used when consulting that raw key. Process-session text + # is session_header even on a thread-scoped request; a thread-only raw + # key stays thread_header so a session_header tombstone cannot hide it. + legacy_continuity_source: _CodexSessionSource | None = None + # A previously unseen thread should inherit the healthy process preference + # once, then persist its own bounded row. This is never ownership: a + # missing process default may be initialized once by insert-if-absent, but + # no thread request may update or delete an established process row. + seed_selection_key: str | None = None + seed_selection_kind: StickySessionKind | None = None # ``conversation`` has no dedicated owner index. Preserve that provenance # until selection can prove one hard owner or a one-account pool. require_unambiguous_account: bool = False @@ -59,8 +98,33 @@ def legacy_selection_key(self) -> str | None: # Old replicas persisted bare session headers as raw CODEX_SESSION # keys. Always consult this alongside the soft row: any raw hit may be # hard turn-state ownership and therefore takes precedence. + if self.legacy_codex_session_key is not None: + return self.legacy_codex_session_key return self.key if self.codex_session_source == "session_header" else None + def selection_kwargs(self) -> _AffinitySelectionKwargs: + """Expand routing policy once at the account-selection boundary.""" + + # Keep the compatibility edge from silently omitting new policy + # fields. In particular, thread locality is incomplete if callers pass + # its row but forget the process seed or legacy hard-owner lookup. + return { + "sticky_key": self.selection_key, + "sticky_kind": self.kind, + "reallocate_sticky": self.reallocate_sticky, + "sticky_source": self.codex_session_source, + "legacy_sticky_key": self.legacy_selection_key, + "legacy_continuity_source": ( + None if self.legacy_selection_key is None else (self.legacy_continuity_source or "session_header") + ), + "sticky_seed_key": self.seed_selection_key, + "sticky_seed_kind": self.seed_selection_kind, + "spill_bare_session_on_account_cap": self.spill_on_account_cap, + "abandon_unavailable_legacy_owner": self.abandon_unavailable_legacy_owner, + "require_unambiguous_account": self.require_unambiguous_account, + "sticky_max_age_seconds": self.max_age_seconds, + } + @staticmethod def cap_spillover_allowed( capability: bool, @@ -86,7 +150,7 @@ def preferred_owner_sticky_inputs( _CodexSessionSource | None, str | None, ]: - if sticky_source != "session_header": + if sticky_source not in {"session_header", "thread_header"}: return ( sticky_key, sticky_kind, @@ -95,10 +159,12 @@ def preferred_owner_sticky_inputs( sticky_source, legacy_sticky_key, ) - # A resolved response/file/bridge owner bypasses the new soft row, but - # the raw compatibility row still has to be checked for conflicting - # legacy hard ownership. Selection receives no writable sticky key, so - # a raw miss cannot manufacture or rebind a mapping. + # A resolved response/file/bridge owner bypasses the current-Codex + # soft row (process-session or thread PROMPT_CACHE). The raw + # compatibility row still has to be checked for conflicting legacy + # hard ownership. Selection receives no writable sticky key, so a + # raw miss cannot manufacture or rebind a mapping. The caller also + # deliberately omits any broader process seed in this exact-owner path. return None, StickySessionKind.CODEX_SESSION, False, sticky_max_age_seconds, sticky_source, legacy_sticky_key @@ -109,6 +175,73 @@ def _codex_session_selection_key(key: str) -> str: return f"{_CODEX_SELECTION_KEY_PREFIX}:session_header:{digest}" +@dataclass(frozen=True, slots=True) +class _CodexBackendIdentity: + """Independently parsed process-tree and logical-thread identities.""" + + process_session: str | None + thread_id: str | None + + @property + def thread_selection_key(self) -> str | None: + if self.thread_id is None: + return None + # The explicit scope tag prevents the thread-only compatibility form + # from colliding with (process, thread). Length framing keeps distinct + # client tuples distinct even if a future non-HTTP caller admits NULs + # or other delimiters. The LF namespace remains unreachable by headers. + if self.process_session is None: + parts = ("thread-only", self.thread_id) + scope = "thread_only" + else: + parts = ("process-thread", self.process_session, self.thread_id) + scope = "process_thread" + encoded_parts = (part.encode() for part in parts) + framed = b"".join(len(part).to_bytes(8, "big") + part for part in encoded_parts) + digest = sha256(framed).hexdigest() + return f"{_CODEX_SELECTION_KEY_PREFIX}:thread_header:{scope}:{digest}" + + +_CODEX_PROCESS_SESSION_HEADERS = ( + "session_id", + "session-id", + "x-codex-session-id", + "x-codex-conversation-id", +) + + +def _normalized_header_value(headers: Mapping[str, str], names: tuple[str, ...]) -> str | None: + normalized = {key.lower(): value for key, value in headers.items()} + for name in names: + value = normalized.get(name) + if not isinstance(value, str): + continue + stripped = value.strip() + if stripped: + return stripped + return None + + +def _process_session_key_from_headers(headers: Mapping[str, str]) -> str | None: + return _normalized_header_value(headers, _CODEX_PROCESS_SESSION_HEADERS) + + +def _thread_id_from_headers(headers: Mapping[str, str]) -> str | None: + return _normalized_header_value(headers, ("thread-id",)) + + +def _codex_backend_identity( + headers: Mapping[str, str], + *, + thread_id: str | None = None, +) -> _CodexBackendIdentity: + normalized_thread_id = thread_id.strip() if isinstance(thread_id, str) and thread_id.strip() else None + return _CodexBackendIdentity( + process_session=_process_session_key_from_headers(headers), + thread_id=normalized_thread_id if thread_id is not None else _thread_id_from_headers(headers), + ) + + def _prompt_cache_key_from_request_model(payload: ResponsesRequest | ResponsesCompactRequest) -> str | None: typed_value = getattr(payload, "prompt_cache_key", None) if isinstance(typed_value, str) and typed_value: @@ -252,15 +385,10 @@ def _sticky_key_from_payload(payload: ResponsesRequest) -> str | None: def _sticky_key_from_session_header(headers: Mapping[str, str]) -> str | None: - normalized = {key.lower(): value for key, value in headers.items()} - for key in ("session_id", "session-id", "x-codex-session-id", "x-codex-conversation-id", "thread-id"): - value = normalized.get(key) - if not isinstance(value, str): - continue - stripped = value.strip() - if stripped: - return stripped - return None + # Legacy owner/request-log callers still need the historical alias order. + # New account, bridge, and replay locality MUST use the typed process/thread + # helpers above; otherwise a shared process id silently hides thread-id. + return _process_session_key_from_headers(headers) or _thread_id_from_headers(headers) def _sticky_key_from_turn_state_header(headers: Mapping[str, str]) -> str | None: @@ -291,6 +419,37 @@ def _bare_codex_session_affinity( ) +def _thread_codex_session_affinity( + headers: Mapping[str, str], + *, + enabled: bool, + max_age_seconds: int, + thread_id: str | None = None, +) -> _AffinityPolicy | None: + if not enabled: + return None + identity = _codex_backend_identity(headers, thread_id=thread_id) + thread_key = identity.thread_selection_key + if thread_key is None: + return None + # Current Codex shares process session and prompt_cache_key across a root + # tree. Thread locality therefore reuses the bounded PROMPT_CACHE lifecycle + # but does not rewrite the upstream cache hint or create durable child rows. + legacy_key = identity.process_session or identity.thread_id + return _AffinityPolicy( + key=thread_key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=max_age_seconds, + codex_session_source="thread_header", + legacy_codex_session_key=legacy_key, + legacy_continuity_source=("session_header" if identity.process_session is not None else "thread_header"), + seed_selection_key=( + _codex_session_selection_key(identity.process_session) if identity.process_session is not None else None + ), + seed_selection_kind=(StickySessionKind.CODEX_SESSION if identity.process_session is not None else None), + ) + + def _request_allows_bare_session_cap_spillover( payload: ResponsesRequest | ResponsesCompactRequest, ) -> bool: @@ -312,6 +471,25 @@ def _request_allows_bare_session_cap_spillover( ) +def _request_allows_unavailable_legacy_owner_abandonment(payload: ResponsesRequest) -> bool: + # Intent and safety are separate proofs. The internal goal marker says the + # client deliberately restarted, while the replay classifier proves that + # this particular body carries no account-scoped state. Never collapse this + # into a marker-only or missing-previous-response shortcut. + if not responses_request_contains_goal_continuation_context(payload): + return False + # Classify the same canonical body that subscription egress would send. + # Raw model dumps retain accepted compatibility-only controls, which are + # not account state and must not make equivalent request forms disagree. + replay_payload = dict(payload.to_replay_safety_payload()) + # ``type=response.create`` belongs to the direct-WebSocket envelope, not + # the HTTP Responses body. Remove only that exact discriminator after + # canonicalization; unknown envelope values remain fail-closed below. + if replay_payload.get("type") == "response.create": + replay_payload.pop("type") + return responses_payload_is_account_neutral_fresh_replay(replay_payload) + + def _affinity_with_payload_continuity( policy: _AffinityPolicy, payload: ResponsesRequest | ResponsesCompactRequest, @@ -347,6 +525,37 @@ def _sticky_key_for_codex_control_request( return _AffinityPolicy() +def _sticky_key_for_thread_goal_request( + payload: Mapping[str, object], + headers: Mapping[str, str], + codex_session_affinity: bool, + max_age_seconds: int, +) -> _AffinityPolicy: + turn_state_key = _sticky_key_from_turn_state_header(headers) + if turn_state_key is not None: + return _AffinityPolicy( + key=turn_state_key, + kind=StickySessionKind.CODEX_SESSION, + codex_session_source="turn_state", + ) + payload_thread_id = payload.get("threadId") + if isinstance(payload_thread_id, str) and payload_thread_id.strip(): + thread_affinity = _thread_codex_session_affinity( + headers, + enabled=codex_session_affinity, + max_age_seconds=max_age_seconds, + thread_id=payload_thread_id, + ) + if thread_affinity is not None: + return thread_affinity + # Routing only consumes a valid nonblank identity. The upstream thread-goal + # protocol remains authoritative for payload validation and error shape. + return _sticky_key_for_codex_control_request( + headers, + codex_session_affinity=codex_session_affinity, + ) + + def _owner_lookup_session_id_from_headers( headers: Mapping[str, str], *, @@ -363,6 +572,55 @@ def _owner_lookup_session_id_from_headers( return _sticky_key_from_session_header(headers) +def _websocket_continuity_key_from_headers( + headers: Mapping[str, str], + *, + synthesized_turn_state: str | None = None, +) -> str | None: + """Return the primary count-bounded direct-WebSocket continuity key.""" + + explicit_turn_state = _sticky_key_from_turn_state_header(headers) + if explicit_turn_state is not None and explicit_turn_state != synthesized_turn_state: + # Exact client continuation must outrank broader thread locality. A + # synthesized handshake placeholder is only an alias for the current + # connection and therefore does not gain this hard precedence. + return explicit_turn_state + identity = _codex_backend_identity(headers) + if identity.thread_selection_key is not None: + return identity.thread_selection_key + return _owner_lookup_session_id_from_headers( + headers, + synthesized_turn_state=synthesized_turn_state, + ) + + +def _websocket_continuity_aliases_from_headers( + headers: Mapping[str, str], + *, + synthesized_turn_state: str | None = None, +) -> tuple[str, ...]: + """Keep exact turn aliases without restoring process-wide thread state.""" + + aliases: list[str] = [] + primary = _websocket_continuity_key_from_headers( + headers, + synthesized_turn_state=synthesized_turn_state, + ) + if primary is not None: + aliases.append(primary) + thread_key = _codex_backend_identity(headers).thread_selection_key + if thread_key is not None: + # When an exact turn resolved first, refresh the thread alias to that + # same state so a later unanchored reconnect remains thread-local. + aliases.append(thread_key) + explicit_turn_state = _sticky_key_from_turn_state_header(headers) + if explicit_turn_state is not None and explicit_turn_state != synthesized_turn_state: + aliases.append(explicit_turn_state) + if synthesized_turn_state is not None: + aliases.append(synthesized_turn_state) + return tuple(dict.fromkeys(aliases)) + + # Pattern matching turn-state values synthesized by the helpers below. # A 32-char lowercase hex (uuid4().hex) suffix follows the prefix. _SYNTHESIZED_TURN_STATE_PATTERN = re.compile(r"^(?:http_)?turn_[0-9a-f]{32}$") @@ -450,6 +708,14 @@ def _sticky_key_for_responses_request( kind=StickySessionKind.CODEX_SESSION, codex_session_source="turn_state", ) + elif ( + thread_affinity := _thread_codex_session_affinity( + headers, + enabled=codex_session_affinity, + max_age_seconds=openai_cache_affinity_max_age_seconds, + ) + ) is not None: + policy = thread_affinity elif ( session_affinity := _bare_codex_session_affinity( headers, @@ -478,4 +744,17 @@ def _sticky_key_for_responses_request( ) else: policy = _AffinityPolicy() + if ( + # The raw row this escape hatch retires is the process-session key. + # Current Codex also sends thread-id, so locality source is often + # thread_header; that must not hide the process-session exception. + # An explicit turn-state header stays hard even with the same marker. + policy.codex_session_source in {"session_header", "thread_header"} + and ( + policy.codex_session_source == "session_header" + or _codex_backend_identity(headers).process_session is not None + ) + and _request_allows_unavailable_legacy_owner_abandonment(payload) + ): + policy = replace(policy, abandon_unavailable_legacy_owner=True) return _affinity_with_payload_continuity(policy, payload) diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index 186bc72cca..97340a6564 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -5,12 +5,12 @@ import logging import math import time -from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mapping +from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterable, Mapping from contextlib import asynccontextmanager from dataclasses import dataclass, replace from datetime import datetime, timezone from json import JSONDecodeError -from typing import Any, Final, Literal, Protocol, cast +from typing import Any, Final, Literal, Protocol, TypeVar, cast from uuid import uuid4 import anyio @@ -26,16 +26,17 @@ WebSocket, ) from fastapi.responses import JSONResponse, StreamingResponse -from pydantic import ValidationError +from pydantic import BaseModel, ConfigDict, ValidationError from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from starlette.convertors import Convertor, register_url_convertor +from starlette.datastructures import Headers from starlette.websockets import WebSocketState from app.core import usage as usage_core from app.core.auth.dependencies import ( set_openai_error_format, - validate_codex_usage_identity, + validate_codex_provider_usage_identity, validate_proxy_api_key, validate_proxy_api_key_authorization, validate_required_proxy_api_key, @@ -122,29 +123,12 @@ OpenAIError, OpenAIResponsePayload, OpenAIResponseResult, + normalize_compaction_item_id, ) from app.core.openai.models import ( OpenAIErrorEnvelope as OpenAIErrorEnvelopeModel, ) from app.core.openai.parsing import parse_response_payload -from app.core.openai.public_output import ( - PUBLIC_RESPONSE_TEXT_PART_TYPES as _PUBLIC_RESPONSE_TEXT_PART_TYPES, -) -from app.core.openai.public_output import ( - collect_public_output_item_event as _collect_public_output_item_event_shared, -) -from app.core.openai.public_output import ( - extract_public_output_item_text as _extract_public_output_item_text, -) -from app.core.openai.public_output import ( - is_public_passthrough_output_item_type as _is_public_passthrough_output_item_type, -) -from app.core.openai.public_output import ( - merge_public_response_output_items as _merge_public_response_output_items, -) -from app.core.openai.public_output import ( - normalize_public_output_item as _normalize_public_output_item, -) from app.core.openai.requests import ( ResponsesCompactRequest, ResponsesRequest, @@ -161,7 +145,7 @@ resolve_request_client_host, ) from app.core.resilience.overload import is_local_overload_error_code, merge_retry_after_headers -from app.core.runtime_logging import log_error_response +from app.core.runtime_logging import log_error_response, safe_log_field from app.core.types import JsonValue from app.core.upstream_proxy import ResolvedUpstreamRoute, UpstreamProxyRouteError, resolve_upstream_route from app.core.utils.json_guards import is_json_list, is_json_mapping @@ -210,6 +194,9 @@ from app.modules.model_sources.forwarding import ( forward_audio_transcription as forward_source_audio_transcription, ) +from app.modules.model_sources.forwarding import ( + forward_embeddings as forward_source_embeddings, +) from app.modules.model_sources.forwarding import ( forward_responses as forward_source_responses, ) @@ -220,18 +207,29 @@ stream_responses as stream_source_responses, ) from app.modules.model_sources.repository import ModelSourcesRepository +from app.modules.model_sources.selection import ( + allowed_source_ids_for_api_key, + effective_model_for_api_key, + select_responses_model_source, +) from app.modules.proxy import affinity as proxy_affinity_module from app.modules.proxy import images_service as images_service_module from app.modules.proxy import service as proxy_service_module from app.modules.proxy._service.support import ( _bind_propagated_capacity_startup_ready, _bind_propagated_capacity_startup_wait, + _bind_propagated_responses_owner_forward_dispatched, + _bind_propagated_responses_owner_forward_rejected, + _bind_propagated_responses_service_cleanup_ready, _could_be_blank_html_comment_line, _is_reasoning_summary_interleavable_event, _reasoning_summary_delta_key, _request_log_client_fields, _reset_propagated_capacity_startup_ready, _reset_propagated_capacity_startup_wait, + _reset_propagated_responses_owner_forward_dispatched, + _reset_propagated_responses_owner_forward_rejected, + _reset_propagated_responses_service_cleanup_ready, _strip_blank_html_comment_lines, ) from app.modules.proxy.account_cache import get_account_selection_cache @@ -240,6 +238,7 @@ from app.modules.proxy.http_bridge_forwarding import parse_forwarded_request from app.modules.proxy.images_observability import ( IMAGE_ROUTE_MODEL_STATE, + IMAGE_ROUTE_STARTED_AT_STATE, IMAGE_ROUTE_STREAM_STATE, record_images_route_observability, ) @@ -251,12 +250,17 @@ enforce_strict_text_format, model_alias_requests_fast_mode, normalize_responses_request_payload, + normalize_source_reasoning_aliases, openai_client_payload_error, openai_validation_error, resolve_model_alias, + resolve_wire_reasoning_effort, + responses_source_route_excluded, + restore_source_reasoning_effort, sanitize_source_chat_payload, strip_terminal_compaction_trigger_input, validate_model_access, + validate_top_level_compaction_trigger_input_shape, ) from app.modules.proxy.schemas import ( AccountPoolUsageResponse, @@ -298,6 +302,7 @@ from app.modules.usage.updater import UsageUpdater logger = logging.getLogger(__name__) +_T = TypeVar("_T") _REASONING_SUMMARY_DELTA_TYPES = frozenset({"response.reasoning_summary_text.delta"}) _REASONING_SUMMARY_DONE_TYPES = frozenset( @@ -307,6 +312,23 @@ } ) +_PUBLIC_RESPONSE_OUTPUT_ITEM_TYPES = frozenset( + { + "message", + "compaction", + "function_call", + "function_call_output", + "reasoning", + "web_search_call", + "file_search_call", + "computer_call", + "code_interpreter_call", + "mcp_approval_request", + "mcp_list_tools", + "output_image", + } +) +_PUBLIC_RESPONSE_TEXT_PART_TYPES = frozenset({"output_text", "input_text", "text", "refusal"}) _PUBLIC_RESPONSE_STREAM_TERMINAL_TYPES = frozenset( {"response.completed", "response.incomplete", "response.failed", "error"} ) @@ -690,6 +712,31 @@ def _is_openai_sdk_request( return _accepts_event_stream(request) or payload.messages is not None +async def _capture_raw_compaction_trigger_error(request: Request) -> None: + """Validate top-level compaction triggers before Pydantic normalization. + + The typed request models intentionally hoist trailing system/developer + messages into ``instructions``. Keep that behavior for runtime parsing and + OpenAPI, but remember a raw trigger-placement error for the endpoint to + render after FastAPI has supplied the typed body. + """ + try: + raw_payload = await request.json() + except (JSONDecodeError, UnicodeDecodeError, ValueError): + return + if not is_json_mapping(raw_payload): + return + try: + validate_top_level_compaction_trigger_input_shape(raw_payload) + except ClientPayloadError as exc: + request.state.compaction_trigger_error = exc + + +def _raw_compaction_trigger_error(request: Request) -> ClientPayloadError | None: + error = getattr(request.state, "compaction_trigger_error", None) + return error if isinstance(error, ClientPayloadError) else None + + async def _thread_goal_payload_from_request(request: Request) -> dict[str, JsonValue]: if request.method.upper() == "GET": return {key: value for key, value in request.query_params.multi_items()} @@ -710,6 +757,9 @@ async def _thread_goal_proxy( context: ProxyContext, api_key: ApiKeyData | None, ) -> Response: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial payload = await _thread_goal_payload_from_request(request) try: response = await context.service.thread_goal_request( @@ -875,7 +925,12 @@ async def _codex_control_proxy( api_key: ApiKeyData | None, *, adapter: _CodexControlAdapter = _PASSTHROUGH_CODEX_CONTROL_ADAPTER, + enforce_required_capability_transport: bool = True, ) -> Response: + if enforce_required_capability_transport: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial try: response = await context.service.codex_control_request( path, @@ -997,10 +1052,19 @@ async def wham_agent_identities_jwks( context: ProxyContext = Depends(get_proxy_context), api_key: ApiKeyData | None = Security(validate_proxy_api_key), ) -> Response: - return await _codex_control_proxy(request, "wham/agent-identities/jwks", context, api_key) + return await _codex_control_proxy( + request, + "wham/agent-identities/jwks", + context, + api_key, + enforce_required_capability_transport=False, + ) -@router.post("/responses/", include_in_schema=False) +@router.post( + "/responses/", + include_in_schema=False, +) @router.post( "/responses", responses={ @@ -1019,11 +1083,15 @@ async def responses( context: ProxyContext = Depends(get_proxy_context), api_key: ApiKeyData | None = Security(validate_proxy_api_key), ) -> Response: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial explicit_openai_sdk_marker = _has_explicit_openai_sdk_marker(request) openai_sdk_request = _is_openai_sdk_request(request, payload) native_codex_heartbeat = _is_native_codex_request(request.headers) and not explicit_openai_sdk_marker openai_compat_payload = _has_openai_responses_shape(payload) try: + validate_top_level_compaction_trigger_input_shape(payload) responses_payload = normalize_responses_request_payload( payload, openai_compat=openai_compat_payload, @@ -1036,19 +1104,25 @@ async def responses( return _logged_error_json_response(request, 400, error) raw_source_model = _effective_optional_model_for_api_key(api_key, responses_payload.model) - prohibit_fast_mode, service_tier_was_enforced = await _apply_api_key_enforcement_with_fast_mode_policy( - responses_payload, api_key - ) + ( + prohibit_fast_mode, + service_tier_was_enforced, + pre_normalization_effort, + ) = await _apply_api_key_enforcement_with_fast_mode_policy(responses_payload, api_key) if prohibit_fast_mode and _is_fast_mode_model_alias(raw_source_model): raw_source_model = responses_payload.model validate_model_access(api_key, responses_payload.model) try: - compact_trigger_input = strip_terminal_compaction_trigger_input(responses_payload) + # Terminal compaction triggers run the upstream compact flow on the + # turn's owner account, and file-referencing requests are pinned to + # the account that received the upload; the shared predicate keeps + # this gate and the WebSocket source-ownership guards in agreement. + source_route_excluded = responses_source_route_excluded(responses_payload) except ClientPayloadError as exc: error = openai_client_payload_error(exc) return _logged_error_json_response(request, 400, error) source = None - if compact_trigger_input is None and not extract_input_file_ids(responses_payload.input): + if not source_route_excluded: source_selection = await _select_responses_model_source( responses_payload.model, api_key, @@ -1070,6 +1144,7 @@ async def responses( source=source, api_key=api_key, rate_limit_headers=rate_limit_headers, + pre_normalization_effort=pre_normalization_effort, ) apply_enforced_service_tier_model_fallback( @@ -1085,6 +1160,7 @@ async def responses( codex_session_affinity=True, openai_cache_affinity=True, prefer_http_bridge=True, + api_key_policy_already_applied=True, prohibit_fast_mode=prohibit_fast_mode, # The Codex CLI consumes codex.* vendor events and the upstream's # native event ordering, while OpenAI SDK clients pointed at this @@ -1102,6 +1178,9 @@ async def opportunistic_admission( context: ProxyContext = Depends(get_proxy_context), api_key: ApiKeyData | None = Security(validate_proxy_api_key), ) -> Response: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial denial = await _opportunistic_admission_denial(request, context, api_key, model=model) if denial is not None: return denial @@ -1113,7 +1192,12 @@ async def responses_websocket( websocket: WebSocket, context: ProxyContext = Depends(get_proxy_websocket_context), ) -> None: - api_key, denial = await _validate_proxy_websocket_request(websocket) + capability_header_values = _required_capability_values(websocket.headers) + api_key, denial = await _validate_proxy_websocket_request( + websocket, + allow_required_capability=True, + require_api_key=bool(capability_header_values), + ) if denial is not None: await websocket.send_denial_response(denial) return @@ -1131,11 +1215,15 @@ async def responses_websocket( api_key=api_key, client_ip=resolve_request_client_host(websocket), synthesized_turn_state=turn_state if client_turn_state is None else None, - capability_header_values=tuple(websocket.headers.getlist(CODEX_LB_REQUIRED_CAPABILITY_HEADER)), + capability_header_values=capability_header_values, ) -@v1_router.post("/responses/", response_model=OpenAIResponseResult, include_in_schema=False) +@v1_router.post( + "/responses/", + response_model=OpenAIResponseResult, + include_in_schema=False, +) @v1_router.post( "/responses", response_model=OpenAIResponseResult, @@ -1152,9 +1240,16 @@ async def responses_websocket( async def v1_responses( request: Request, payload: V1ResponsesRequest = Body(...), + _raw_trigger_validation: None = Depends(_capture_raw_compaction_trigger_error), context: ProxyContext = Depends(get_proxy_context), api_key: ApiKeyData | None = Security(validate_proxy_api_key), ) -> Response: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial + raw_trigger_error = _raw_compaction_trigger_error(request) + if raw_trigger_error is not None: + return _logged_error_json_response(request, 400, openai_client_payload_error(raw_trigger_error)) try: responses_payload = payload.to_responses_request() enforce_strict_text_format(responses_payload) @@ -1166,9 +1261,11 @@ async def v1_responses( error = openai_validation_error(exc) return _logged_error_json_response(request, 400, error) raw_source_model = _effective_optional_model_for_api_key(api_key, responses_payload.model) - prohibit_fast_mode, service_tier_was_enforced = await _apply_api_key_enforcement_with_fast_mode_policy( - responses_payload, api_key - ) + ( + prohibit_fast_mode, + service_tier_was_enforced, + pre_normalization_effort, + ) = await _apply_api_key_enforcement_with_fast_mode_policy(responses_payload, api_key) if prohibit_fast_mode and _is_fast_mode_model_alias(raw_source_model): raw_source_model = responses_payload.model validate_model_access(api_key, responses_payload.model) @@ -1199,6 +1296,7 @@ async def v1_responses( source=source, api_key=api_key, rate_limit_headers=rate_limit_headers, + pre_normalization_effort=pre_normalization_effort, ) apply_enforced_service_tier_model_fallback( responses_payload, @@ -1213,6 +1311,7 @@ async def v1_responses( codex_session_affinity=False, openai_cache_affinity=True, prefer_http_bridge=True, + api_key_policy_already_applied=True, prohibit_fast_mode=prohibit_fast_mode, ) else: @@ -1224,6 +1323,7 @@ async def v1_responses( codex_session_affinity=False, openai_cache_affinity=True, prefer_http_bridge=True, + api_key_policy_already_applied=True, prohibit_fast_mode=prohibit_fast_mode, ) return _mark_subscription_prompt_cache_fallback(response, responses_payload) @@ -1258,6 +1358,9 @@ async def internal_bridge_responses( api_key, auth_error = await _validate_internal_bridge_api_key(request) if auth_error is not None: return auth_error + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial if forwarded_request_context.context.signature_version is None: try: await context.service.validate_http_bridge_legacy_forward_anchor( @@ -1283,6 +1386,7 @@ async def internal_bridge_responses( codex_session_affinity=forwarded_request_context.context.codex_session_affinity, openai_cache_affinity=True, prefer_http_bridge=True, + api_key_policy_already_applied=True, skip_limit_enforcement=skip_limit_enforcement, api_key_reservation_override=forwarded_request_context.context.reservation, include_rate_limit_headers=False, @@ -1420,7 +1524,12 @@ async def v1_responses_websocket( websocket: WebSocket, context: ProxyContext = Depends(get_proxy_websocket_context), ) -> None: - api_key, denial = await _validate_proxy_websocket_request(websocket) + capability_header_values = _required_capability_values(websocket.headers) + api_key, denial = await _validate_proxy_websocket_request( + websocket, + allow_required_capability=True, + require_api_key=bool(capability_header_values), + ) if denial is not None: await websocket.send_denial_response(denial) return @@ -1438,7 +1547,7 @@ async def v1_responses_websocket( api_key=api_key, client_ip=resolve_request_client_host(websocket), synthesized_turn_state=turn_state if client_turn_state is None else None, - capability_header_values=tuple(websocket.headers.getlist(CODEX_LB_REQUIRED_CAPABILITY_HEADER)), + capability_header_values=capability_header_values, ) @@ -1646,7 +1755,9 @@ async def _ensure_v1_reset_credit_account_fresh(account_id: str) -> _V1ResetCred async with get_background_session() as session: repo = AccountsRepository(session) account = await repo.get_by_id(account_id) - if account is None: + # An account marked for background deletion is already deleted from + # every consumer's point of view (its credentials are wiped). + if account is None or account.delete_requested_at is not None: raise HTTPException(status_code=404, detail="Account not found") auth_manager = AuthManager( repo, @@ -1673,13 +1784,25 @@ async def v1_reset_credit( return response -@usage_router.post("/v1/reset-credit", response_model=V1ResetCreditRedeemResponse) +@usage_router.post( + "/v1/reset-credit", + response_model=V1ResetCreditRedeemResponse, +) async def v1_redeem_reset_credit( + request: Request, payload: V1ResetCreditRedeemRequest, api_key: ApiKeyData = Security(validate_usage_api_key), -) -> V1ResetCreditRedeemResponse: +) -> V1ResetCreditRedeemResponse | JSONResponse: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial async with get_background_session() as session: account = await AccountsRepository(session).get_by_id(payload.account_id) + # A pending-deletion account is gone (credentials wiped): treat it + # exactly like an account outside the pool. ``getattr`` because pool + # membership tests stub the account with plain namespaces. + if account is not None and getattr(account, "delete_requested_at", None) is not None: + account = None if not _is_reset_credit_account_in_api_key_pool(account, api_key): raise HTTPException(status_code=403, detail="Account is outside the API key pool") if account is None: @@ -1782,6 +1905,9 @@ async def _run_v1_warmup( *, mode: str, ) -> Response: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial if mode not in _WARMUP_MODES: return _logged_error_json_response( request, @@ -1959,14 +2085,18 @@ async def _hide_upstream_quota_for_api_key_clients(api_key: ApiKeyData | None) - async def _apply_api_key_enforcement_with_fast_mode_policy( payload: ResponsesRequest | ResponsesCompactRequest, api_key: ApiKeyData | None, -) -> tuple[bool, bool]: +) -> tuple[bool, bool, str | None]: prohibit_fast_mode = await _prohibit_fast_mode_enabled() - service_tier_was_enforced = apply_api_key_enforcement( + enforcement = apply_api_key_enforcement( payload, api_key, prohibit_fast_mode=prohibit_fast_mode, ) - return prohibit_fast_mode, service_tier_was_enforced + return ( + prohibit_fast_mode, + enforcement.service_tier_was_enforced, + enforcement.pre_normalization_reasoning_effort, + ) async def _prohibit_fast_mode_enabled() -> bool: @@ -1990,26 +2120,44 @@ async def _rate_limit_headers_for_request( async def _release_reservation_deferring_cancellation( reservation: ApiKeyUsageReservationData, ) -> None: + await _await_cleanup_deferring_cancellation(_release_reservation(reservation)) + + +async def _await_result_deferring_cancellation(awaitable: Awaitable[_T]) -> tuple[_T, bool]: + """Finish an owned awaitable despite repeated cancellation and report whether cancellation arrived.""" + + task = asyncio.ensure_future(awaitable) + cancellation_deferred = False with anyio.CancelScope(shield=True): - task = asyncio.create_task(_release_reservation(reservation)) while True: try: - await asyncio.shield(task) - return + return await asyncio.shield(task), cancellation_deferred except asyncio.CancelledError: if task.cancelled(): raise + cancellation_deferred = True + raise RuntimeError("unreachable shielded cancellation-deferral state") + + +async def _await_cleanup_deferring_cancellation(awaitable: Awaitable[object]) -> None: + """Finish a required cleanup operation despite repeated cancellation delivery.""" + + await _await_result_deferring_cancellation(awaitable) async def _rate_limit_headers_with_reservation_cleanup( context: ProxyContext, api_key: ApiKeyData | None, owned_reservation: ApiKeyUsageReservationData | None, + *, + reservation_cleanup: _ResponsesReservationCleanup | None = None, ) -> dict[str, str]: try: return await _rate_limit_headers_for_request(context, api_key) except BaseException: - if owned_reservation is not None: + if reservation_cleanup is not None: + await reservation_cleanup.release(action="rate limit headers") + elif owned_reservation is not None: try: await _release_reservation_deferring_cancellation(owned_reservation) except (Exception, asyncio.CancelledError): @@ -2020,6 +2168,55 @@ async def _rate_limit_headers_with_reservation_cleanup( raise +@dataclass(slots=True) +class _ResponsesReservationCleanup: + owns_reservation: bool + reservation: ApiKeyUsageReservationData | None + scheduler: _ResponsesCleanupScheduler | None + request_id: str + released: bool = False + + async def release(self, *, action: str) -> None: + if not self.owns_reservation or self.released: + return + self.released = True + await _release_reservation_best_effort( + self.reservation, + action=action, + scheduler=self.scheduler, + request_id=self.request_id, + ) + + +class _ResponsesCleanupScheduler(Protocol): + def _schedule_cancel_safe_cleanup( + self, + coro: Coroutine[Any, Any, None], + *, + action: str, + request_id: str, + ) -> asyncio.Task[None]: ... + + +def _responses_origin_may_release_reservation( + *, + service_cleanup_ready_event: asyncio.Event, + owner_forward_dispatched_event: asyncio.Event | None = None, + owner_forward_rejected_event: asyncio.Event | None = None, +) -> bool: + if service_cleanup_ready_event.is_set(): + return False + if owner_forward_dispatched_event is None or not owner_forward_dispatched_event.is_set(): + return True + return owner_forward_rejected_event is not None and owner_forward_rejected_event.is_set() + + +def _responses_cleanup_scheduler(service: object) -> _ResponsesCleanupScheduler | None: + if callable(getattr(service, "_schedule_cancel_safe_cleanup", None)): + return cast(_ResponsesCleanupScheduler, service) + return None + + def _select_codex_usage_limit( limits: list[V1UsageLimitResponse], window: str, @@ -2160,6 +2357,9 @@ async def backend_transcribe( context: ProxyContext = Depends(get_proxy_context), api_key: ApiKeyData | None = Security(validate_proxy_api_key), ) -> JSONResponse: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial multipart = await _parse_transcription_multipart(request, require_model=False) return await _transcribe_request( request=request, @@ -2192,6 +2392,9 @@ async def backend_files_create( apply here -- upstream caps file size at 512 MiB which we enforce in ``FileCreateRequest``. """ + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial reservation = await _enforce_request_limits( api_key, request_model=_FILES_CREATE_LIMIT_MODEL, @@ -2236,6 +2439,9 @@ async def backend_files_finalize( polls upstream for up to 30 s while ``status == "retry"``; we return the final payload verbatim so the caller sees what upstream saw. """ + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial reservation = await _enforce_request_limits( api_key, request_model=_FILES_FINALIZE_LIMIT_MODEL, @@ -2275,6 +2481,9 @@ async def v1_audio_transcriptions( context: ProxyContext = Depends(get_proxy_context), api_key: ApiKeyData | None = Security(validate_proxy_api_key), ) -> Response: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial multipart = await _parse_transcription_multipart(request, require_model=True) assert multipart.model is not None model = multipart.model @@ -2304,14 +2513,81 @@ async def v1_audio_transcriptions( ) -@router.post("/images/generations", response_model=None, include_in_schema=False) -@v1_router.post("/images/generations", response_model=None) +class V1EmbeddingsRequest(BaseModel): + """OpenAI-compatible embeddings request. + + Only ``model`` and ``input`` are validated; other OpenAI params + (``encoding_format``, ``dimensions``, ``user``, …) pass through to the + model source verbatim. + """ + + model_config = ConfigDict(extra="allow") + + model: str + input: str | list[str] | list[int] | list[list[int]] + + +@v1_router.post("/embeddings") +async def v1_embeddings( + request: Request, + payload: V1EmbeddingsRequest = Body(...), + context: ProxyContext = Depends(get_proxy_context), + api_key: ApiKeyData | None = Security(validate_proxy_api_key), +) -> Response: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial + model = payload.model + rate_limit_headers = await _rate_limit_headers_for_request(context, api_key) + source = await _select_embeddings_model_source(model, api_key) + if source is None: + # Embeddings have no subscription-backed fallback: only configured + # model sources can serve them. + return _logged_error_json_response( + request, + status_code=404, + content=openai_error( + "model_not_found", + f"The model '{model}' does not exist or no enabled model source supports embeddings for it", + error_type="invalid_request_error", + ), + headers=rate_limit_headers, + ) + validate_model_access(api_key, model) + return await _source_embeddings_response( + request=request, + model=model, + payload=payload, + source=source, + api_key=api_key, + rate_limit_headers=rate_limit_headers, + ) + + +@router.post( + "/images/generations", + response_model=None, + include_in_schema=False, +) +@v1_router.post( + "/images/generations", + response_model=None, +) async def v1_images_generations( request: Request, payload: V1ImagesGenerationsRequest = Body(...), context: ProxyContext = Depends(get_proxy_context), api_key: ApiKeyData | None = Security(validate_proxy_api_key), ) -> Response: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + _record_required_capability_image_transport_denial( + request, + route="generations", + model=payload.model, + stream=bool(payload.stream), + ) + return capability_transport_denial return await _proxy_images_generation_request( request=request, payload=payload, @@ -2342,6 +2618,26 @@ def _record_images_edit_early_rejection( ) +def _record_required_capability_image_transport_denial( + request: Request, + *, + route: Literal["generations", "edits"], + model: str | None, + stream: bool, +) -> None: + started_at = getattr(request.state, IMAGE_ROUTE_STARTED_AT_STATE, None) + if not isinstance(started_at, float): + started_at = time.perf_counter() + record_images_route_observability( + route=route, + model=model, + stream=stream, + status=400, + outcome="invalid_request", + started_at=started_at, + ) + + def _images_edit_invalid_request_response( request: Request, *, @@ -2373,6 +2669,15 @@ async def v1_images_edits( context: ProxyContext = Depends(get_proxy_context), api_key: ApiKeyData | None = Security(validate_proxy_api_key), ) -> Response: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + _record_required_capability_image_transport_denial( + request, + route="edits", + model=None, + stream=False, + ) + return capability_transport_denial started_at = time.perf_counter() raise_for_unsupported_multipart_content_encoding(request) @@ -2542,6 +2847,15 @@ async def codex_images_edits( then delegate to the shared edit pipeline so validation and upstream behavior remain identical. """ + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + _record_required_capability_image_transport_denial( + request, + route="edits", + model=None, + stream=False, + ) + return capability_transport_denial started_at = time.perf_counter() try: raw_payload = await request.json() @@ -2920,6 +3234,8 @@ async def _stream_with_log_rewrite() -> AsyncIterator[bytes]: _output = captured.get("image_output_tokens") _cached = captured.get("image_cached_input_tokens") await _finalize_image_reservation( + context.service, + api_key, reservation, model=public_model, input_tokens=_input if isinstance(_input, int) else None, @@ -2973,6 +3289,8 @@ async def _stream_with_log_rewrite() -> AsyncIterator[bytes]: _output = captured.get("image_output_tokens") _cached = captured.get("image_cached_input_tokens") await _finalize_image_reservation( + context.service, + api_key, reservation, model=public_model, input_tokens=_input if isinstance(_input, int) else None, @@ -3215,6 +3533,8 @@ async def _stream_with_log_rewrite() -> AsyncIterator[bytes]: _output = captured.get("image_output_tokens") _cached = captured.get("image_cached_input_tokens") await _finalize_image_reservation( + context.service, + api_key, reservation, model=public_model, input_tokens=_input if isinstance(_input, int) else None, @@ -3268,6 +3588,8 @@ async def _stream_with_log_rewrite() -> AsyncIterator[bytes]: _output = captured.get("image_output_tokens") _cached = captured.get("image_cached_input_tokens") await _finalize_image_reservation( + context.service, + api_key, reservation, model=public_model, input_tokens=_input if isinstance(_input, int) else None, @@ -3546,16 +3868,17 @@ def _canonical_model_slug(model: str) -> str: def _to_model_list_item(slug: str, model: UpstreamModel, *, created: int) -> ModelListItem: + context_window = _resolved_context_window(model) return ModelListItem.model_validate( { "id": slug, "created": created, "owned_by": "codex-lb", - "metadata": _to_model_metadata(model), + "metadata": _to_model_metadata(model, context_window=context_window), "api_types": ["chat_completions"], - "capabilities": _v1_model_capabilities(model), - "context_length": _v1_input_context_window(model), - "contextLength": _v1_input_context_window(model), + "capabilities": _v1_model_capabilities(model, context_window=context_window), + "context_length": context_window, + "contextLength": context_window, "max_output_tokens": _v1_max_output_tokens(model), "maxOutputTokens": _v1_max_output_tokens(model), "supports_reasoning": _v1_supports_reasoning(model), @@ -3590,6 +3913,9 @@ def _is_codex_backend_catalog_model(model: UpstreamModel) -> bool: return model.raw.get("shell_type") == "shell_command" +_CODEX_WIRE_REASONING_EFFORTS = frozenset({"none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"}) + + def _codex_model_truncation_policy(model: UpstreamModel) -> CodexTruncationPolicy: if "truncation_policy" in model.raw: try: @@ -3607,8 +3933,24 @@ def _codex_model_experimental_supported_tools(model: UpstreamModel) -> list[str] return [tool for tool in tools if isinstance(tool, str)] +def _codex_wire_reasoning_levels(model: UpstreamModel) -> list[ReasoningLevelSchema]: + return [ + ReasoningLevelSchema(effort=level.effort, description=level.description) + for level in model.supported_reasoning_levels + if level.effort in _CODEX_WIRE_REASONING_EFFORTS + ] + + +def _codex_wire_default_reasoning_level(model: UpstreamModel) -> str | None: + default = model.default_reasoning_level + if default in _CODEX_WIRE_REASONING_EFFORTS: + return default + return None + + def _to_codex_model_entry(model: UpstreamModel, *, visibility: str | None = None) -> CodexModelEntry: raw = model.raw + reasoning_levels = _codex_wire_reasoning_levels(model) extra: dict[str, JsonValue] = {} skip_keys = { @@ -3638,7 +3980,7 @@ def _to_codex_model_entry(model: UpstreamModel, *, visibility: str | None = None extra[key] = value # If context_window is overridden, also override max_context_window to match - effective_cw = _effective_context_window(model) + effective_cw = _resolved_context_window(model) if effective_cw != model.context_window and "max_context_window" in extra: extra["max_context_window"] = effective_cw @@ -3647,11 +3989,8 @@ def _to_codex_model_entry(model: UpstreamModel, *, visibility: str | None = None display_name=model.display_name, description=model.description, base_instructions=model.base_instructions, - default_reasoning_level=model.default_reasoning_level, - supported_reasoning_levels=[ - ReasoningLevelSchema(effort=rl.effort, description=rl.description) - for rl in model.supported_reasoning_levels - ], + default_reasoning_level=_codex_wire_default_reasoning_level(model), + supported_reasoning_levels=reasoning_levels, supported_in_api=model.supported_in_api, priority=model.priority, minimal_client_version=model.minimal_client_version, @@ -3659,7 +3998,7 @@ def _to_codex_model_entry(model: UpstreamModel, *, visibility: str | None = None support_verbosity=model.support_verbosity, default_verbosity=model.default_verbosity, supports_parallel_tool_calls=model.supports_parallel_tool_calls, - context_window=_effective_context_window(model), + context_window=effective_cw, input_modalities=list(model.input_modalities), available_in_plans=sorted(model.available_in_plans), prefer_websockets=model.prefer_websockets, @@ -3673,18 +4012,39 @@ def _to_codex_model_entry(model: UpstreamModel, *, visibility: str | None = None ) -def _effective_context_window(model: UpstreamModel) -> int: - overrides = get_settings().model_context_window_overrides - return overrides.get(model.slug, model.context_window) - - -def _v1_full_context_window(model: UpstreamModel) -> int: +def _resolved_context_window(model: UpstreamModel) -> int: + # An explicit operator context-window override is an assertion about the usable + # input budget, so it must also reach the generic OpenAI-compatible fields + # (`context_length`, `contextLength`, `capabilities.context_length`, and + # `metadata.input_context_window`). Generic clients read those rather than + # `metadata.context_window` and would otherwise cap themselves at the + # un-overridden upstream budget while Codex-native clients use the wider window. + # The override is clamped to the upstream-declared `max_context_window` so it can + # never advertise more input than the backend sanctions — the same clamp the Codex + # client applies to `model_context_window` in config.toml. The clamp only applies + # when upstream declares a ceiling strictly above `context_window`: bootstrap + # subscription models (`_bootstrap_model`) and source-catalog models + # (`source_models_to_upstream_models`) synthesize `max_context_window == + # context_window` purely so Codex clients can parse the entry, and treating that + # parseability default as a real ceiling would silently disable every raise + # override for those models. + # + # This is the single resolution point for the reported window: the Codex-native + # `context_window`/`max_context_window` rewrite, `metadata.context_window`, and + # every input-budget field all share this one value, so an override above the + # backend ceiling can never split one model into two contradictory budgets. overrides = get_settings().model_context_window_overrides - return overrides.get(model.slug, model.context_window) - - -def _v1_input_context_window(model: UpstreamModel) -> int: - return model.context_window + override = overrides.get(model.slug) + if override is None: + return model.context_window + max_context_window = model.raw.get("max_context_window") + if ( + isinstance(max_context_window, int) + and not isinstance(max_context_window, bool) + and max_context_window > model.context_window + ): + return min(override, max_context_window) + return override def _v1_max_output_tokens(model: UpstreamModel) -> int | None: @@ -3694,11 +4054,11 @@ def _v1_max_output_tokens(model: UpstreamModel) -> int | None: return _V1_MAX_OUTPUT_TOKEN_OVERRIDES.get(model.slug) -def _v1_model_capabilities(model: UpstreamModel) -> dict[str, JsonValue]: +def _v1_model_capabilities(model: UpstreamModel, *, context_window: int) -> dict[str, JsonValue]: supports_streaming_raw = model.raw.get("supports_streaming") supports_streaming = supports_streaming_raw if isinstance(supports_streaming_raw, bool) else True return { - "context_length": _v1_input_context_window(model), + "context_length": context_window, "max_output_tokens": _v1_max_output_tokens(model), "supports_reasoning": _v1_supports_reasoning(model), "supports_images": _v1_supports_vision(model), @@ -3714,8 +4074,8 @@ def _v1_model_capabilities(model: UpstreamModel) -> dict[str, JsonValue]: def _v1_supports_reasoning(model: UpstreamModel) -> bool: if bool(model.supported_reasoning_levels) or model.supports_reasoning_summaries: return True - # OpenAI-compatible source models advertise no reasoning levels; their - # catalog entries opt in via raw metadata so /v1/models reflects reality. + # Source models whose operator declared no levels and no summary support + # opt in via raw metadata instead, so /v1/models reflects reality. return model.raw.get("supports_reasoning") is True @@ -3746,12 +4106,12 @@ def _effective_source_codex_visibility( return "list" -def _to_model_metadata(model: UpstreamModel) -> ModelMetadata: +def _to_model_metadata(model: UpstreamModel, *, context_window: int) -> ModelMetadata: return ModelMetadata( display_name=model.display_name, description=model.description, - context_window=_v1_full_context_window(model), - input_context_window=_v1_input_context_window(model), + context_window=context_window, + input_context_window=context_window, max_output_tokens=_v1_max_output_tokens(model), input_modalities=list(model.input_modalities), supported_reasoning_levels=[ @@ -3811,6 +4171,9 @@ async def v1_chat_completions( context: ProxyContext = Depends(get_proxy_context), api_key: ApiKeyData | None = Security(validate_proxy_api_key), ) -> Response: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial cursor_compat_client = _is_cursor_compat_client(request, api_key) effective_model = _effective_model_for_api_key(api_key, payload.model) @@ -3841,7 +4204,11 @@ async def v1_chat_completions( except ValidationError as exc: error = openai_validation_error(exc) return _logged_error_json_response(request, 400, error, headers=rate_limit_headers) - prohibit_fast_mode, service_tier_was_enforced = await _apply_api_key_enforcement_with_fast_mode_policy( + # The replaced effort is discarded: the enforced Responses payload built + # here is only ever forwarded to a subscription. This endpoint does + # source-route, but that branch forwards the untouched original chat + # payload, so there is nothing for a restore to undo. + prohibit_fast_mode, service_tier_was_enforced, _ = await _apply_api_key_enforcement_with_fast_mode_policy( responses_payload, api_key ) if prohibit_fast_mode and _is_fast_mode_model_alias(effective_model): @@ -3885,6 +4252,11 @@ async def v1_chat_completions( source=source, model=request_model, api_key=api_key, + allowed_reasoning_effort=( + responses_payload._codex_lb_client_reasoning_effort + if api_key is not None and api_key.allowed_reasoning_efforts is not None + else None + ), reservation=reservation, rate_limit_headers=rate_limit_headers, ) @@ -3953,22 +4325,25 @@ async def v1_chat_completions( ) try: - first = await stream.__anext__() - except StopAsyncIteration: - first = None - except ProxyResponseError as exc: - return _logged_error_json_response(request, exc.status_code, exc.payload, headers=rate_limit_headers) + try: + first = await stream.__anext__() + except StopAsyncIteration: + first = None + except ProxyResponseError as exc: + return _logged_error_json_response(request, exc.status_code, exc.payload, headers=rate_limit_headers) - stream_with_first = _prepend_first(first, stream) - result = await collect_chat_completion(stream_with_first, model=responses_payload.model) + result = await collect_chat_completion( + _prepend_first(first, stream), + model=responses_payload.model, + ) + finally: + await _aclose_stream(stream) if isinstance(result, OpenAIErrorEnvelopeModel): - error = result.error - code = error.code if error else None - status_code = 503 if code in _UNAVAILABLE_SELECTION_ERROR_CODES else 502 + status_code, envelope = _mask_previous_response_not_found_error(result) return _logged_error_json_response( request, status_code, - content=result.model_dump(mode="json", exclude_none=True), + content=envelope.model_dump(mode="json", exclude_none=True), headers=rate_limit_headers, ) if cursor_compat_client and isinstance(result, ChatCompletion): @@ -4025,35 +4400,28 @@ async def _select_responses_model_source( raw_model: str | None = None, require_streaming: bool = False, ) -> tuple[ModelSource, str] | None: + # Shared with the WebSocket path so both transports agree on which models + # belong to a model source. + return await select_responses_model_source( + model, + api_key, + raw_model=raw_model, + require_streaming=require_streaming, + ) + + +async def _select_embeddings_model_source(model: str, api_key: ApiKeyData | None) -> ModelSource | None: assigned_source_ids = _allowed_source_ids_for_api_key(api_key) - exact_allowed_models = set(api_key.allowed_models) if api_key and api_key.allowed_models else None - candidates = [candidate for candidate in (raw_model, model) if candidate] - if not candidates: + exact_allowed_models = _exact_source_allowed_models_for_api_key(api_key) + if exact_allowed_models is not None and model not in exact_allowed_models: return None - deduped_candidates = list(dict.fromkeys(candidates)) - registry_models = get_model_registry().get_models_with_fallback() async with get_background_session() as session: - repository = ModelSourcesRepository(session) - for candidate in deduped_candidates: - if exact_allowed_models is not None and candidate not in exact_allowed_models: - continue - subscription_model = registry_models.get(candidate) - if assigned_source_ids is None and subscription_model is not None: - continue - source = await repository.find_responses_source_for_model( - candidate, - allowed_source_ids=assigned_source_ids, - require_streaming=require_streaming, - ) - if source is not None: - break - else: - source = None - # ``close_session`` rolls back the read transaction, which would - # expire the loaded row; detach it so the forwarding path can read - # its attributes after this session boundary. + source = await ModelSourcesRepository(session).find_embeddings_source_for_model( + model, + allowed_source_ids=assigned_source_ids, + ) detach_session_objects(session) - return (source, candidate) if source is not None else None + return source async def _select_audio_transcriptions_model_source(model: str, api_key: ApiKeyData | None) -> ModelSource | None: @@ -4073,9 +4441,7 @@ async def _select_audio_transcriptions_model_source(model: str, api_key: ApiKeyD def _allowed_source_ids_for_api_key(api_key: ApiKeyData | None) -> set[str] | None: - if api_key is None or not api_key.source_assignment_scope_enabled: - return None - return set(api_key.assigned_source_ids) + return allowed_source_ids_for_api_key(api_key) async def _parse_transcription_multipart( @@ -4103,11 +4469,11 @@ async def _parse_transcription_multipart( ) -async def _source_audio_transcription_response( +async def _source_embeddings_response( *, request: Request, model: str, - multipart: _ParsedTranscriptionMultipart, + payload: "V1EmbeddingsRequest", source: ModelSource, api_key: ApiKeyData | None, rate_limit_headers: Mapping[str, str], @@ -4117,14 +4483,10 @@ async def _source_audio_transcription_response( request_model=model, request_service_tier=None, ) + outbound = payload.model_dump(exclude_none=True) + outbound["model"] = model try: - result = await forward_source_audio_transcription( - source, - audio_bytes=multipart.audio_bytes, - filename=multipart.filename, - content_type=multipart.content_type, - fields=list(multipart.ordered_text_fields), - ) + result = await forward_source_embeddings(source, outbound) except ModelSourceForwardingError as exc: await _release_reservation(reservation) await _log_source_chat_completion( @@ -4138,24 +4500,112 @@ async def _source_audio_transcription_response( upstream_status_code=exc.upstream_status_code, ) return _logged_error_json_response(request, exc.status_code, exc.payload, headers=rate_limit_headers) - - # ASR billing prefers audio duration: when the source model has a - # per-minute rate and the response carries a duration, settle cost from - # the duration with zero tokens. Only when there is no usable duration - # cost do we fall back to token usage (and fail closed for limited keys - # if neither is available). - audio_cost_usd = ( - source_model_audio_cost_usd(source, model, result.audio_seconds) if result.audio_seconds is not None else None + if result.usage is None and _reservation_requires_usage(reservation): + await _release_reservation(reservation) + error = openai_error( + "usage_unavailable", + "OpenAI-compatible model source embeddings response did not include token usage for a limited API key", + error_type="server_error", + ) + await _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="error", + error_code="usage_unavailable", + error_message="source embeddings response missing token usage", + upstream_status_code=result.upstream_status_code, + ) + return _logged_error_json_response(request, 502, error, headers=rate_limit_headers) + settled = await _settle_source_reservation( + reservation, + source=source, + model=model, + usage=result.usage, ) - if audio_cost_usd is not None: - settle_usage: SourceUsage | None = SourceUsage(input_tokens=0, output_tokens=0) - cost_override: float | None = audio_cost_usd - else: - settle_usage = result.usage - cost_override = None - if result.usage is None and _reservation_requires_usage(reservation): - await _release_reservation(reservation) - error = openai_error( + if not settled: + await _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="error", + error_code="usage_settlement_failed", + error_message="source usage settlement failed", + upstream_status_code=result.upstream_status_code, + ) + return _logged_error_json_response( + request, + 502, + _source_usage_settlement_failed_error(), + headers=rate_limit_headers, + ) + await _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="success", + usage=result.usage, + upstream_status_code=result.upstream_status_code, + ) + return JSONResponse(content=result.payload, headers=dict(rate_limit_headers)) + + +async def _source_audio_transcription_response( + *, + request: Request, + model: str, + multipart: _ParsedTranscriptionMultipart, + source: ModelSource, + api_key: ApiKeyData | None, + rate_limit_headers: Mapping[str, str], +) -> Response: + reservation = await _enforce_request_limits( + api_key, + request_model=model, + request_service_tier=None, + ) + try: + result = await forward_source_audio_transcription( + source, + audio_bytes=multipart.audio_bytes, + filename=multipart.filename, + content_type=multipart.content_type, + fields=list(multipart.ordered_text_fields), + ) + except ModelSourceForwardingError as exc: + await _release_reservation(reservation) + await _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="error", + error_code=_source_error_code(exc.payload), + error_message=_source_error_message(exc.payload), + upstream_status_code=exc.upstream_status_code, + ) + return _logged_error_json_response(request, exc.status_code, exc.payload, headers=rate_limit_headers) + + # ASR billing prefers audio duration: when the source model has a + # per-minute rate and the response carries a duration, settle cost from + # the duration with zero tokens. Only when there is no usable duration + # cost do we fall back to token usage (and fail closed for limited keys + # if neither is available). + audio_cost_usd = ( + source_model_audio_cost_usd(source, model, result.audio_seconds) if result.audio_seconds is not None else None + ) + if audio_cost_usd is not None: + settle_usage: SourceUsage | None = SourceUsage(input_tokens=0, output_tokens=0) + cost_override: float | None = audio_cost_usd + else: + settle_usage = result.usage + cost_override = None + if result.usage is None and _reservation_requires_usage(reservation): + await _release_reservation(reservation) + error = openai_error( "usage_unavailable", "OpenAI-compatible model source transcription response did not include token usage " "or a usable duration for a limited API key", @@ -4221,7 +4671,16 @@ async def _source_responses_response( source: ModelSource, api_key: ApiKeyData | None, rate_limit_headers: Mapping[str, str], + pre_normalization_effort: str | None, ) -> Response: + # This is the first point where the request is known to be served by a + # model source rather than a subscription account, so it is the only place + # the reasoning-effort workaround can be undone safely. + restore_source_reasoning_effort( + payload, + source, + pre_normalization_effort=pre_normalization_effort, + ) reservation = await _enforce_request_limits( api_key, request_model=payload.model, @@ -4229,6 +4688,37 @@ async def _source_responses_response( request_usage_budget=estimate_api_key_request_usage(payload), ) source_payload = payload.model_dump_for_forwarding() + preserve_materialized_provider_alias = payload._codex_lb_provider_reasoning_effort_materialized and ( + api_key is None or (api_key.enforced_reasoning_effort is None and api_key.allowed_reasoning_efforts is None) + ) + if preserve_materialized_provider_alias: + reasoning = source_payload.get("reasoning") + if isinstance(reasoning, dict): + reasoning = {key: value for key, value in reasoning.items() if key != "effort"} + if reasoning: + source_payload["reasoning"] = reasoning + else: + source_payload.pop("reasoning") + if api_key is not None and ( + api_key.enforced_reasoning_effort is not None + or (api_key.allowed_reasoning_efforts is not None and payload._codex_lb_client_reasoning_effort is not None) + ): + normalize_source_reasoning_aliases(source_payload) + source_reasoning_effort = ( + api_key.enforced_reasoning_effort + if api_key is not None and api_key.enforced_reasoning_effort is not None + else payload._codex_lb_client_reasoning_effort + ) + if source_reasoning_effort is not None and not preserve_materialized_provider_alias: + source_reasoning_effort = resolve_wire_reasoning_effort(source_reasoning_effort) + reasoning = source_payload.get("reasoning") + if isinstance(reasoning, dict): + source_payload["reasoning"] = { + **reasoning, + "effort": source_reasoning_effort, + } + else: + source_payload["reasoning"] = {"effort": source_reasoning_effort} strip_replayed_tool_call_namespaces_from_payload(source_payload) source_payload["stream"] = bool(payload.stream) _apply_source_response_request_overrides(source_payload, source_model_request_overrides(source, payload.model)) @@ -4531,13 +5021,19 @@ async def _source_chat_completion_response( source: ModelSource, model: str, api_key: ApiKeyData | None, + allowed_reasoning_effort: str | None = None, reservation: ApiKeyUsageReservationData | None, rate_limit_headers: Mapping[str, str], ) -> Response: source_payload = payload.model_dump(mode="json", exclude_none=True) source_payload["model"] = model source_payload["stream"] = bool(payload.stream) - apply_api_key_enforcement_to_chat_payload(source_payload, api_key) + apply_api_key_enforcement_to_chat_payload( + source_payload, + api_key, + allowed_reasoning_effort=allowed_reasoning_effort, + materialize_allowed_reasoning_effort=allowed_reasoning_effort is not None, + ) sanitize_source_chat_payload( source_payload, allow_reasoning=source_model_supports_reasoning(source, model), @@ -4564,6 +5060,36 @@ async def _source_chat_completion_response( upstream_status_code=exc.upstream_status_code, ) return _logged_error_json_response(request, exc.status_code, exc.payload, headers=rate_limit_headers) + except asyncio.CancelledError: + release_exc: BaseException | None = None + if reservation is not None: + try: + await _release_reservation_deferring_cancellation(reservation) + except BaseException as exc: + release_exc = exc + await _await_cleanup_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="cancelled", + error_code="client_disconnected", + error_message="client disconnected during source stream setup", + ) + ) + if release_exc is not None: + logger.warning( + "Failed to release source stream setup reservation after client disconnect source_id=%s model=%s", + source.id, + model, + exc_info=release_exc, + ) + raise + except BaseException: + if reservation is not None: + await _release_reservation_deferring_cancellation(reservation) + raise if _reservation_requires_usage(reservation): return await _buffered_limited_source_chat_stream_response( request, @@ -4605,6 +5131,36 @@ async def _source_chat_completion_response( upstream_status_code=exc.upstream_status_code, ) return _logged_error_json_response(request, exc.status_code, exc.payload, headers=rate_limit_headers) + except asyncio.CancelledError: + release_exc: BaseException | None = None + if reservation is not None: + try: + await _release_reservation_deferring_cancellation(reservation) + except BaseException as exc: + release_exc = exc + await _await_cleanup_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="cancelled", + error_code="client_disconnected", + error_message="client disconnected during source request setup", + ) + ) + if release_exc is not None: + logger.warning( + "Failed to release source request setup reservation after client disconnect source_id=%s model=%s", + source.id, + model, + exc_info=release_exc, + ) + raise + except BaseException: + if reservation is not None: + await _release_reservation_deferring_cancellation(reservation) + raise if result.usage is None and _reservation_requires_usage(reservation): await _release_reservation(reservation) @@ -4625,34 +5181,60 @@ async def _source_chat_completion_response( ) return _logged_error_json_response(request, 502, error, headers=rate_limit_headers) - settled = await _settle_source_reservation(reservation, source=source, model=model, usage=result.usage) + settled, settlement_deferred_cancellation = await _await_result_deferring_cancellation( + _settle_source_reservation(reservation, source=source, model=model, usage=result.usage) + ) + if settlement_deferred_cancellation: + await _await_cleanup_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="cancelled", + usage=result.usage, + timings=result.timings, + error_code="client_disconnected", + error_message="client disconnected during source usage settlement", + upstream_status_code=result.upstream_status_code, + ) + ) + raise asyncio.CancelledError if not settled: - await _log_source_chat_completion( - request, - source=source, - api_key=api_key, - model=model, - status="error", - error_code="usage_settlement_failed", - error_message="source usage settlement failed", - upstream_status_code=result.upstream_status_code, + _, log_deferred_cancellation = await _await_result_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="error", + error_code="usage_settlement_failed", + error_message="source usage settlement failed", + upstream_status_code=result.upstream_status_code, + ) ) + if log_deferred_cancellation: + raise asyncio.CancelledError return _logged_error_json_response( request, 502, _source_usage_settlement_failed_error(), headers=rate_limit_headers, ) - await _log_source_chat_completion( - request, - source=source, - api_key=api_key, - model=model, - status="success", - usage=result.usage, - timings=result.timings, - upstream_status_code=result.upstream_status_code, + _, log_deferred_cancellation = await _await_result_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="success", + usage=result.usage, + timings=result.timings, + upstream_status_code=result.upstream_status_code, + ) ) + if log_deferred_cancellation: + raise asyncio.CancelledError return JSONResponse(content=result.payload, status_code=200, headers=rate_limit_headers) @@ -4698,13 +5280,44 @@ async def _buffered_limited_source_chat_stream_response( error_message="source stream buffer limit exceeded", ) return _logged_error_json_response(request, 502, error, headers=rate_limit_headers) - except asyncio.CancelledError: + except asyncio.CancelledError as cancel_exc: # Starlette cancels this task when the downstream client disconnects; # CancelledError is a BaseException, so without this branch the # reservation would stay charged until stale-reservation cleanup. - await _aclose_stream(stream) - await _release_reservation(reservation) - raise + close_exc: BaseException | None = None + release_exc: BaseException | None = None + try: + await _await_cleanup_deferring_cancellation(_aclose_stream(stream)) + except BaseException as exc: + close_exc = exc + if reservation is not None: + try: + await _release_reservation_deferring_cancellation(reservation) + except BaseException as exc: + release_exc = exc + await _await_cleanup_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="cancelled", + usage=usage_holder.usage, + timings=usage_holder.timings, + error_code="client_disconnected", + error_message="client disconnected during source stream buffering", + ) + ) + if release_exc is not None: + logger.warning( + "Failed to release buffered source stream reservation after client disconnect source_id=%s model=%s", + source.id, + model, + exc_info=release_exc, + ) + if close_exc is not None: + raise close_exc + raise cancel_exc except ModelSourceForwardingError as exc: await _release_reservation(reservation) await _log_source_chat_completion( @@ -4754,32 +5367,57 @@ async def _buffered_limited_source_chat_stream_response( ) return _logged_error_json_response(request, 502, error, headers=rate_limit_headers) - settled = await _settle_source_reservation(reservation, source=source, model=model, usage=usage_holder.usage) + settled, settlement_deferred_cancellation = await _await_result_deferring_cancellation( + _settle_source_reservation(reservation, source=source, model=model, usage=usage_holder.usage) + ) + if settlement_deferred_cancellation: + await _await_cleanup_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="cancelled", + usage=usage_holder.usage, + timings=usage_holder.timings, + error_code="client_disconnected", + error_message="client disconnected during source stream usage settlement", + ) + ) + raise asyncio.CancelledError if not settled: - await _log_source_chat_completion( - request, - source=source, - api_key=api_key, - model=model, - status="error", - error_code="usage_settlement_failed", - error_message="source usage settlement failed", + _, log_deferred_cancellation = await _await_result_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="error", + error_code="usage_settlement_failed", + error_message="source usage settlement failed", + ) ) + if log_deferred_cancellation: + raise asyncio.CancelledError return _logged_error_json_response( request, 502, _source_usage_settlement_failed_error(), headers=rate_limit_headers, ) - await _log_source_chat_completion( - request, - source=source, - api_key=api_key, - model=model, - status="success", - usage=usage_holder.usage, - timings=usage_holder.timings, + _, log_deferred_cancellation = await _await_result_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="success", + usage=usage_holder.usage, + timings=usage_holder.timings, + ) ) + if log_deferred_cancellation: + raise asyncio.CancelledError async def body() -> AsyncIterator[bytes]: for chunk in chunks: @@ -4819,8 +5457,11 @@ async def _source_chat_stream_with_settlement( status = "cancelled" error_code = "client_disconnected" error_message = "client disconnected before stream completed" - await _aclose_stream(stream) - await _release_reservation(reservation) + try: + await _await_cleanup_deferring_cancellation(_aclose_stream(stream)) + finally: + if reservation is not None: + await _release_reservation_deferring_cancellation(reservation) raise except ModelSourceForwardingError as exc: status = "error" @@ -4835,7 +5476,14 @@ async def _source_chat_stream_with_settlement( await _release_reservation(reservation) raise else: - settled = await _settle_source_reservation(reservation, source=source, model=model, usage=usage_holder.usage) + settled, settlement_deferred_cancellation = await _await_result_deferring_cancellation( + _settle_source_reservation(reservation, source=source, model=model, usage=usage_holder.usage) + ) + if settlement_deferred_cancellation: + status = "cancelled" + error_code = "client_disconnected" + error_message = "client disconnected during source usage settlement" + raise asyncio.CancelledError if not settled: status = "error" error_code = "usage_settlement_failed" @@ -4851,17 +5499,19 @@ async def _source_chat_stream_with_settlement( model, ) finally: - await _log_source_chat_completion( - request, - source=source, - api_key=api_key, - model=model, - status=status, - usage=usage_holder.usage, - timings=usage_holder.timings, - error_code=error_code, - error_message=error_message, - upstream_status_code=None, + await _await_cleanup_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status=status, + usage=usage_holder.usage, + timings=usage_holder.timings, + error_code=error_code, + error_message=error_message, + upstream_status_code=None, + ) ) @@ -4889,6 +5539,7 @@ async def _stream_responses( forwarded_client_ip: str | None = None, enforce_openai_sdk_contract: bool = True, native_codex_heartbeat: bool = False, + api_key_policy_already_applied: bool = False, prohibit_fast_mode: bool = False, ) -> Response: # Owner-forwarded payloads have already passed API-key enforcement, @@ -4897,11 +5548,13 @@ async def _stream_responses( # signed effective tier: an owner with an older/staler model snapshot must # not re-add a tier that the origin authoritatively removed. forwarded_effective_service_tier = payload.service_tier if forwarded_request else None - service_tier_was_enforced = apply_api_key_enforcement( - payload, - api_key, - prohibit_fast_mode=prohibit_fast_mode, - ) + service_tier_was_enforced = False + if not api_key_policy_already_applied: + service_tier_was_enforced = apply_api_key_enforcement( + payload, + api_key, + prohibit_fast_mode=prohibit_fast_mode, + ).service_tier_was_enforced if forwarded_request: payload.service_tier = forwarded_effective_service_tier else: @@ -4931,7 +5584,14 @@ async def _stream_responses( prompt_cache_key_alias = payload.model_extra.get("promptCacheKey") if isinstance(prompt_cache_key_alias, str) and "prompt_cache_key" not in compact_payload_data: compact_payload_data["prompt_cache_key"] = prompt_cache_key_alias - compact_payload_data["input"] = compact_trigger_input + # The main /responses route trims the terminal trigger before + # compaction so the compact budget and image elision see only + # the history to summarize. The upstream /compact contract + # still requires exactly one terminal trigger on the wire. + compact_payload_data["input"] = [ + *compact_trigger_input, + {"type": "compaction_trigger"}, + ] if payload.previous_response_id is not None: compact_payload_data["previous_response_id"] = payload.previous_response_id if payload.conversation is not None: @@ -4958,18 +5618,33 @@ async def _stream_responses( request_usage_budget=estimate_api_key_request_usage(payload), ) ) + reservation_cleanup = _ResponsesReservationCleanup( + owns_reservation=owns_reservation, + reservation=reservation, + scheduler=_responses_cleanup_scheduler(context.service), + request_id=ensure_request_id(), + ) + responses_service_cleanup_ready_event = asyncio.Event() + responses_owner_forward_dispatched_event = asyncio.Event() + responses_owner_forward_rejected_event = asyncio.Event() rate_limit_headers = ( await _rate_limit_headers_with_reservation_cleanup( context, api_key, reservation if owns_reservation else None, + reservation_cleanup=reservation_cleanup if owns_reservation else None, ) if include_rate_limit_headers else {} ) bridge_active = prefer_http_bridge and proxy_service_module.get_settings().http_responses_session_bridge_enabled effective_headers = forwarded_headers or request.headers + bridge_recovery_eligible = _http_bridge_recovery_request_eligible( + payload, + bridge_active=bridge_active, + headers=effective_headers, + ) client_ip = forwarded_client_ip if forwarded_request else resolve_request_client_host(request) downstream_turn_state = ( forwarded_downstream_turn_state @@ -4984,6 +5659,9 @@ async def _stream_responses( else {} ) if compact_payload is not None: + responses_cleanup_ready_token = _bind_propagated_responses_service_cleanup_ready( + responses_service_cleanup_ready_event + ) try: try: compact_result = await context.service.compact_responses( @@ -4994,6 +5672,8 @@ async def _stream_responses( api_key=api_key, api_key_reservation=reservation, client_ip=client_ip, + forwarded_request=forwarded_request, + forwarded_file_owner_account_id=forwarded_file_owner_account_id, ) except NotImplementedError: error = OpenAIErrorEnvelopeModel( @@ -5010,6 +5690,31 @@ async def _stream_responses( headers=rate_limit_headers, ) except ProxyResponseError as exc: + if forwarded_request and responses_service_cleanup_ready_event.is_set(): + # Fallback settlement already transferred cleanup. A 502 + # would look like a definitive rejection and let origin + # replay a compact that already ran. + envelope = _parse_error_envelope(exc.payload) + error = envelope.error + stream = _synthetic_compaction_failure_stream( + response_id=get_request_id() or "unknown", + error_code=(error.code if error is not None and error.code else "upstream_error"), + error_message=( + error.message + if error is not None and error.message + else "Compact request failed after settlement" + ), + ) + return StreamingResponse( + stream, + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache, no-transform", + "X-Accel-Buffering": "no", + **turn_state_headers, + **rate_limit_headers, + }, + ) return _stream_startup_error_response( request, exc, @@ -5017,18 +5722,21 @@ async def _stream_responses( ) compact_item = _compact_response_output_item(compact_result) if compact_item is None: - error = openai_error( - "upstream_error", - "Compact response did not include a compaction output item", - error_type="server_error", + if forwarded_request and responses_service_cleanup_ready_event.is_set(): + stream = _synthetic_compaction_failure_stream(response_id=_compact_response_id(compact_result)) + else: + error = openai_error( + "upstream_error", + "Compact response did not include a compaction output item", + error_type="server_error", + ) + return _logged_error_json_response(request, 502, error, headers=rate_limit_headers) + else: + stream = _synthetic_compaction_response_stream( + compact_item, + response_id=_compact_response_id(compact_result), + usage=compact_result.usage, ) - return _logged_error_json_response(request, 502, error, headers=rate_limit_headers) - response_id = _compact_response_id(compact_result) - stream = _synthetic_compaction_response_stream( - compact_item, - response_id=response_id, - usage=compact_result.usage, - ) return StreamingResponse( stream, media_type="text/event-stream", @@ -5040,35 +5748,39 @@ async def _stream_responses( }, ) finally: - if owns_reservation: - await _release_reservation(reservation) + _reset_propagated_responses_service_cleanup_ready(responses_cleanup_ready_token) + if _responses_origin_may_release_reservation( + service_cleanup_ready_event=responses_service_cleanup_ready_event + ): + await reservation_cleanup.release(action="terminal compaction response") capacity_wait_event = asyncio.Event() capacity_ready_event = _CapacityStartupReadyEvent() payload.stream = True - if prefer_http_bridge: - stream = context.service.stream_http_responses( - payload, - effective_headers, - codex_session_affinity=codex_session_affinity, - propagate_http_errors=True, - openai_cache_affinity=openai_cache_affinity, - api_key=api_key, - api_key_reservation=reservation, - suppress_text_done_events=suppress_text_done_events, - downstream_turn_state=downstream_turn_state, - forwarded_request=forwarded_request, - forwarded_original_request_unanchored=forwarded_original_request_unanchored, - forwarded_legacy_signature=forwarded_legacy_signature, - forwarded_affinity_kind=forwarded_affinity_kind, - forwarded_affinity_key=forwarded_affinity_key, - forwarded_file_owner_account_id=forwarded_file_owner_account_id, - client_ip=client_ip, - enforce_openai_sdk_contract=enforce_openai_sdk_contract, - capacity_startup_wait_event=capacity_wait_event, - capacity_startup_ready_event=capacity_ready_event, - ) - else: - stream = context.service.stream_responses( + + def build_response_stream() -> AsyncIterator[str]: + if prefer_http_bridge: + return context.service.stream_http_responses( + payload, + effective_headers, + codex_session_affinity=codex_session_affinity, + propagate_http_errors=True, + openai_cache_affinity=openai_cache_affinity, + api_key=api_key, + api_key_reservation=reservation, + suppress_text_done_events=suppress_text_done_events, + downstream_turn_state=downstream_turn_state, + forwarded_request=forwarded_request, + forwarded_original_request_unanchored=forwarded_original_request_unanchored, + forwarded_legacy_signature=forwarded_legacy_signature, + forwarded_affinity_kind=forwarded_affinity_kind, + forwarded_affinity_key=forwarded_affinity_key, + forwarded_file_owner_account_id=forwarded_file_owner_account_id, + client_ip=client_ip, + enforce_openai_sdk_contract=enforce_openai_sdk_contract, + capacity_startup_wait_event=capacity_wait_event, + capacity_startup_ready_event=capacity_ready_event, + ) + return context.service.stream_responses( payload, request.headers, codex_session_affinity=codex_session_affinity, @@ -5080,37 +5792,169 @@ async def _stream_responses( client_ip=client_ip, enforce_openai_sdk_contract=enforce_openai_sdk_contract, ) + + def build_recovery_response_stream() -> AsyncIterator[str]: + """Build a server-owned retry with a fresh API-key reservation. + + The first bridge generator owns and settles the admission reservation + when it terminates. Indefinite recovery must not reuse that object: + each retry gets a new reservation and therefore remains accounted and + bounded even when the client connection stays open for a long time. + """ + + async def _retry() -> AsyncIterator[str]: + retry_reservation = reservation + if prefer_http_bridge and api_key is not None and reservation is not None: + retry_service_tier = dict(payload.to_payload()).get("service_tier") + retry_reservation = await _enforce_request_limits( + api_key, + request_model=payload.model, + request_service_tier=(retry_service_tier if isinstance(retry_service_tier, str) else None), + request_usage_budget=estimate_api_key_request_usage(payload), + ) + retry_stream = context.service.stream_http_responses( + payload, + effective_headers, + codex_session_affinity=codex_session_affinity, + propagate_http_errors=True, + openai_cache_affinity=openai_cache_affinity, + api_key=api_key, + api_key_reservation=retry_reservation, + suppress_text_done_events=suppress_text_done_events, + downstream_turn_state=downstream_turn_state, + forwarded_request=forwarded_request, + forwarded_original_request_unanchored=forwarded_original_request_unanchored, + forwarded_legacy_signature=forwarded_legacy_signature, + forwarded_affinity_kind=forwarded_affinity_kind, + forwarded_affinity_key=forwarded_affinity_key, + forwarded_file_owner_account_id=forwarded_file_owner_account_id, + client_ip=client_ip, + enforce_openai_sdk_contract=enforce_openai_sdk_contract, + capacity_startup_wait_event=capacity_wait_event, + capacity_startup_ready_event=capacity_ready_event, + ) + async for line in retry_stream: + yield line + + return _retry() + + stream = build_response_stream() + startup_handoff_tasks: list[asyncio.Task[str]] = [] capacity_wait_token = _bind_propagated_capacity_startup_wait(capacity_wait_event) capacity_ready_token = _bind_propagated_capacity_startup_ready(capacity_ready_event) + responses_owner_forward_dispatched_token = _bind_propagated_responses_owner_forward_dispatched( + responses_owner_forward_dispatched_event + ) + responses_owner_forward_rejected_token = _bind_propagated_responses_owner_forward_rejected( + responses_owner_forward_rejected_event + ) + responses_cleanup_ready_token = _bind_propagated_responses_service_cleanup_ready( + responses_service_cleanup_ready_event + ) try: - stream, startup_error = await _probe_stream_startup_error( - stream, - convert_event_errors=bridge_active and enforce_openai_sdk_contract, - timeout_seconds=( - _HTTP_BRIDGE_STARTUP_ERROR_PROBE_SECONDS if prefer_http_bridge else _STREAM_STARTUP_ERROR_PROBE_SECONDS - ), - capacity_wait_event=capacity_wait_event, - capacity_ready_event=capacity_ready_event, - ) - finally: - _reset_propagated_capacity_startup_ready(capacity_ready_token) - _reset_propagated_capacity_startup_wait(capacity_wait_token) + try: + stream, startup_error = await _probe_stream_startup_error( + stream, + convert_event_errors=bridge_active and enforce_openai_sdk_contract, + timeout_seconds=( + _HTTP_BRIDGE_STARTUP_ERROR_PROBE_SECONDS + if prefer_http_bridge + else _STREAM_STARTUP_ERROR_PROBE_SECONDS + ), + capacity_wait_event=capacity_wait_event, + capacity_ready_event=capacity_ready_event, + handoff_task_sink=startup_handoff_tasks, + service_cleanup_ready_event=( + responses_service_cleanup_ready_event if forwarded_request and reservation is not None else None + ), + ) + finally: + _reset_propagated_responses_service_cleanup_ready(responses_cleanup_ready_token) + _reset_propagated_responses_owner_forward_rejected(responses_owner_forward_rejected_token) + _reset_propagated_responses_owner_forward_dispatched(responses_owner_forward_dispatched_token) + _reset_propagated_capacity_startup_ready(capacity_ready_token) + _reset_propagated_capacity_startup_wait(capacity_wait_token) + except BaseException: + if _responses_origin_may_release_reservation( + service_cleanup_ready_event=responses_service_cleanup_ready_event, + owner_forward_dispatched_event=responses_owner_forward_dispatched_event, + owner_forward_rejected_event=responses_owner_forward_rejected_event, + ): + await reservation_cleanup.release(action="responses startup") + raise if startup_error is not None: - if owns_reservation: - await _release_reservation(reservation) - return _stream_startup_error_response( - request, - startup_error, - headers=rate_limit_headers, + startup_error_code = ( + _startup_error_details(startup_error)[0] if isinstance(startup_error, ProxyResponseError) else None + ) + startup_recovery_allowed = ( + isinstance(startup_error, ProxyResponseError) + and bridge_recovery_eligible + and get_settings().http_responses_session_bridge_ambiguous_continuation_recovery_mode + == "server_indefinite_recovery" + and getattr(startup_error, "http_bridge_durable_recovery_eligible", False) + and startup_error_code + in {"stream_incomplete", "stream_idle_timeout", "upstream_request_timeout", "upstream_unavailable"} + and _responses_origin_may_release_reservation( + service_cleanup_ready_event=responses_service_cleanup_ready_event, + owner_forward_dispatched_event=responses_owner_forward_dispatched_event, + owner_forward_rejected_event=responses_owner_forward_rejected_event, + ) + ) + if startup_recovery_allowed: + assert isinstance(startup_error, ProxyResponseError) + + # A durable bridge can fail before the startup probe observes the + # first response.created event. Feed that error through the same + # server-owned recovery loop used for failures after the probe; + # returning JSON here would hand a recoverable disconnect back to + # the client before recovery is even installed. + async def _raise_startup_error() -> AsyncIterator[str]: + raise startup_error + yield "" # pragma: no cover + + stream = _raise_startup_error() + else: + if _responses_origin_may_release_reservation( + service_cleanup_ready_event=responses_service_cleanup_ready_event, + owner_forward_dispatched_event=responses_owner_forward_dispatched_event, + owner_forward_rejected_event=responses_owner_forward_rejected_event, + ): + await reservation_cleanup.release(action="responses startup error") + return _stream_startup_error_response( + request, + startup_error, + headers=rate_limit_headers, + allow_client_full_history_once=bridge_recovery_eligible, + ) + # Server-indefinite recovery is only safe for an explicitly anchored + # continuation. Fresh first-turn requests have no durable parent + # operation to fence, so do not install the recovery loop for them. + recovery_stream_factory = ( + build_recovery_response_stream + if bridge_recovery_eligible + and _responses_origin_may_release_reservation( + service_cleanup_ready_event=responses_service_cleanup_ready_event, + owner_forward_dispatched_event=responses_owner_forward_dispatched_event, + owner_forward_rejected_event=responses_owner_forward_rejected_event, ) + else None + ) stream = _normalize_public_responses_stream( _stream_response_error_events( stream, owns_reservation=owns_reservation, reservation=reservation, + reservation_cleanup=reservation_cleanup, + responses_service_cleanup_ready_event=responses_service_cleanup_ready_event, + responses_owner_forward_dispatched_event=responses_owner_forward_dispatched_event, + responses_owner_forward_rejected_event=responses_owner_forward_rejected_event, + recovery_stream_factory=recovery_stream_factory, + allow_client_full_history_once=bridge_recovery_eligible, + require_durable_recovery_fence=bridge_recovery_eligible, ), enforce_openai_sdk_contract=enforce_openai_sdk_contract, ) + service_stream = stream use_codex_keepalive = native_codex_heartbeat or not enforce_openai_sdk_contract keepalive_frame = CODEX_KEEPALIVE_FRAME if use_codex_keepalive else SSE_KEEPALIVE_FRAME if use_codex_keepalive: @@ -5120,13 +5964,25 @@ async def _stream_responses( request_id=get_request_id(), route_family="responses", ) + stream = inject_sse_keepalives( + stream, + get_settings().sse_keepalive_interval_seconds, + keepalive_frame=keepalive_frame, + on_keepalive=lambda: _record_stream_keepalive("responses"), + ) + # Outermost so a client close after the initial heartbeat still closes + # the service stream, including when the startup probe already completed. + stream = _guard_responses_startup_handoff( + stream, + startup_task=startup_handoff_tasks[0] if startup_handoff_tasks else None, + streams_to_close=(service_stream,), + reservation_cleanup=reservation_cleanup, + responses_service_cleanup_ready_event=responses_service_cleanup_ready_event, + responses_owner_forward_dispatched_event=responses_owner_forward_dispatched_event, + responses_owner_forward_rejected_event=responses_owner_forward_rejected_event, + ) return StreamingResponse( - inject_sse_keepalives( - stream, - get_settings().sse_keepalive_interval_seconds, - keepalive_frame=keepalive_frame, - on_keepalive=lambda: _record_stream_keepalive("responses"), - ), + stream, media_type="text/event-stream", headers={ "Cache-Control": "no-cache, no-transform", @@ -5151,13 +6007,16 @@ async def _collect_responses( openai_cache_affinity: bool = False, suppress_text_done_events: bool = False, prefer_http_bridge: bool = False, + api_key_policy_already_applied: bool = False, prohibit_fast_mode: bool = False, ) -> Response: - service_tier_was_enforced = apply_api_key_enforcement( - payload, - api_key, - prohibit_fast_mode=prohibit_fast_mode, - ) + service_tier_was_enforced = False + if not api_key_policy_already_applied: + service_tier_was_enforced = apply_api_key_enforcement( + payload, + api_key, + prohibit_fast_mode=prohibit_fast_mode, + ).service_tier_was_enforced apply_enforced_service_tier_model_fallback( payload, service_tier_was_enforced=service_tier_was_enforced, @@ -5172,9 +6031,28 @@ async def _collect_responses( request_service_tier=payload.service_tier, request_usage_budget=estimate_api_key_request_usage(payload), ) + reservation_cleanup = _ResponsesReservationCleanup( + owns_reservation=True, + reservation=reservation, + scheduler=_responses_cleanup_scheduler(context.service), + request_id=ensure_request_id(), + ) + responses_service_cleanup_ready_event = asyncio.Event() + responses_owner_forward_dispatched_event = asyncio.Event() + responses_owner_forward_rejected_event = asyncio.Event() - rate_limit_headers = await _rate_limit_headers_with_reservation_cleanup(context, api_key, reservation) + rate_limit_headers = await _rate_limit_headers_with_reservation_cleanup( + context, + api_key, + reservation, + reservation_cleanup=reservation_cleanup, + ) bridge_active = prefer_http_bridge and proxy_service_module.get_settings().http_responses_session_bridge_enabled + bridge_recovery_eligible = _http_bridge_recovery_request_eligible( + payload, + bridge_active=bridge_active, + headers=request.headers, + ) downstream_turn_state = ( proxy_affinity_module.ensure_http_downstream_turn_state(request.headers) if bridge_active else None ) @@ -5211,25 +6089,68 @@ async def _collect_responses( client_ip=client_ip, ) captured_turn_state_headers: dict[str, str] = {} + responses_owner_forward_dispatched_token = _bind_propagated_responses_owner_forward_dispatched( + responses_owner_forward_dispatched_event + ) + responses_owner_forward_rejected_token = _bind_propagated_responses_owner_forward_rejected( + responses_owner_forward_rejected_event + ) + responses_cleanup_ready_token = _bind_propagated_responses_service_cleanup_ready( + responses_service_cleanup_ready_event + ) try: response_payload = await _collect_responses_payload( stream, captured_turn_state_headers=captured_turn_state_headers, ) + except asyncio.CancelledError: + if _responses_origin_may_release_reservation( + service_cleanup_ready_event=responses_service_cleanup_ready_event, + owner_forward_dispatched_event=responses_owner_forward_dispatched_event, + owner_forward_rejected_event=responses_owner_forward_rejected_event, + ): + await reservation_cleanup.release(action="responses collection cancellation") + raise except ProxyResponseError as exc: - await _release_reservation(reservation) + if _responses_origin_may_release_reservation( + service_cleanup_ready_event=responses_service_cleanup_ready_event, + owner_forward_dispatched_event=responses_owner_forward_dispatched_event, + owner_forward_rejected_event=responses_owner_forward_rejected_event, + ): + await reservation_cleanup.release(action="responses collection error") error = _parse_error_envelope(exc.payload) - status_code, error = _mask_previous_response_not_found_error(error, default_status=exc.status_code) + status_code, error = _mask_previous_response_not_found_error( + error, + default_status=exc.status_code, + allow_client_full_history_once=( + bridge_recovery_eligible and getattr(exc, "http_bridge_durable_recovery_eligible", False) + ), + ) return _logged_error_json_response( request, status_code, error.model_dump(mode="json", exclude_none=True), headers={**captured_turn_state_headers, **rate_limit_headers}, ) + except BaseException: + if _responses_origin_may_release_reservation( + service_cleanup_ready_event=responses_service_cleanup_ready_event, + owner_forward_dispatched_event=responses_owner_forward_dispatched_event, + owner_forward_rejected_event=responses_owner_forward_rejected_event, + ): + await reservation_cleanup.release(action="responses collection") + raise + finally: + _reset_propagated_responses_service_cleanup_ready(responses_cleanup_ready_token) + _reset_propagated_responses_owner_forward_rejected(responses_owner_forward_rejected_token) + _reset_propagated_responses_owner_forward_dispatched(responses_owner_forward_dispatched_token) if isinstance(response_payload, OpenAIResponsePayload): if response_payload.status == "failed": error_payload = _error_envelope_from_response(response_payload.error) - status_code, error_payload = _mask_previous_response_not_found_error(error_payload) + status_code, error_payload = _mask_previous_response_not_found_error( + error_payload, + allow_client_full_history_once=False, + ) return _logged_error_json_response( request, status_code, @@ -5240,7 +6161,10 @@ async def _collect_responses( content=response_payload.model_dump(mode="json", exclude_none=True), headers={**turn_state_headers, **captured_turn_state_headers, **rate_limit_headers}, ) - status_code, response_payload = _mask_previous_response_not_found_error(response_payload) + status_code, response_payload = _mask_previous_response_not_found_error( + response_payload, + allow_client_full_history_once=False, + ) return _logged_error_json_response( request, status_code, @@ -5249,13 +6173,23 @@ async def _collect_responses( ) -@router.post("/responses/compact", response_model=CompactResponseResult) +@router.post( + "/responses/compact", + response_model=CompactResponseResult, +) async def responses_compact( request: Request, payload: ResponsesCompactRequest = Body(...), + _raw_trigger_validation: None = Depends(_capture_raw_compaction_trigger_error), context: ProxyContext = Depends(get_proxy_context), api_key: ApiKeyData | None = Security(validate_proxy_api_key), ) -> JSONResponse: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial + raw_trigger_error = _raw_compaction_trigger_error(request) + if raw_trigger_error is not None: + return _logged_error_json_response(request, 400, openai_client_payload_error(raw_trigger_error)) return await _compact_responses( request, payload, @@ -5267,13 +6201,19 @@ async def responses_compact( ) -@v1_router.post("/responses/compact", response_model=CompactResponseResult) +@v1_router.post( + "/responses/compact", + response_model=CompactResponseResult, +) async def v1_responses_compact( request: Request, payload: V1ResponsesCompactRequest = Body(...), context: ProxyContext = Depends(get_proxy_context), api_key: ApiKeyData | None = Security(validate_proxy_api_key), ) -> JSONResponse: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial try: compact_payload = payload.to_compact_request() except ClientPayloadError as exc: @@ -5302,11 +6242,13 @@ async def _compact_responses( openai_cache_affinity: bool = False, prohibit_fast_mode: bool = False, ) -> JSONResponse: + # The replaced effort is discarded: this path is subscription-only, so the + # rewrite that works around the backend hang must stick. service_tier_was_enforced = apply_api_key_enforcement( payload, api_key, prohibit_fast_mode=prohibit_fast_mode, - ) + ).service_tier_was_enforced apply_enforced_service_tier_model_fallback( payload, service_tier_was_enforced=service_tier_was_enforced, @@ -5333,7 +6275,22 @@ async def _compact_responses( request_usage_budget=request_usage_budget, ) - rate_limit_headers = await _rate_limit_headers_with_reservation_cleanup(context, api_key, reservation) + reservation_cleanup = _ResponsesReservationCleanup( + owns_reservation=True, + reservation=reservation, + scheduler=_responses_cleanup_scheduler(context.service), + request_id=ensure_request_id(), + ) + responses_service_cleanup_ready_event = asyncio.Event() + rate_limit_headers = await _rate_limit_headers_with_reservation_cleanup( + context, + api_key, + reservation, + reservation_cleanup=reservation_cleanup, + ) + responses_cleanup_ready_token = _bind_propagated_responses_service_cleanup_ready( + responses_service_cleanup_ready_event + ) try: result = await context.service.compact_responses( payload, @@ -5368,10 +6325,12 @@ async def _compact_responses( headers=rate_limit_headers, ) finally: - await _release_reservation(reservation) + _reset_propagated_responses_service_cleanup_ready(responses_cleanup_ready_token) + if _responses_origin_may_release_reservation(service_cleanup_ready_event=responses_service_cleanup_ready_event): + await reservation_cleanup.release(action="compact response") result_payload = result.model_dump(mode="json", exclude_none=True) if codex_session_affinity: - result_payload = _normalize_codex_remote_compaction_v2_result(result, result_payload) + result_payload = _normalize_codex_remote_compaction_v2_result(result) return JSONResponse( content=result_payload, headers=rate_limit_headers, @@ -5379,14 +6338,64 @@ async def _compact_responses( def _normalize_codex_remote_compaction_v2_result( - payload: CompactResponsePayload, - result_payload: dict[str, JsonValue], + payload: CompactResponsePayload | OpenAIResponsePayload, ) -> dict[str, JsonValue]: + if isinstance(payload, OpenAIResponsePayload): + normalized: dict[str, JsonValue] = {} + if payload.id is not None: + normalized["id"] = payload.id + if payload.status is not None: + normalized["status"] = payload.status + model = (payload.model_extra or {}).get("model") + if isinstance(model, str) and model: + normalized["model"] = model + if payload.usage is not None: + normalized["usage"] = cast(JsonValue, payload.usage.model_dump(mode="json", exclude_none=True)) + if payload.error is not None: + normalized["error"] = cast(JsonValue, payload.error.model_dump(mode="json", exclude_none=True)) + extra = payload.model_extra or {} + output = extra.get("output") + if isinstance(output, list) and not output: + normalized["output"] = [] + return normalized + compaction_item = _compact_response_output_item(payload) - if compaction_item is None: - return result_payload - normalized = dict(result_payload) - normalized["output"] = [compaction_item] + normalized: dict[str, JsonValue] = {"object": payload.object} + if payload.id is not None: + normalized["id"] = payload.id + if payload.status is not None: + normalized["status"] = payload.status + model = (payload.model_extra or {}).get("model") + if isinstance(model, str) and model: + normalized["model"] = model + if payload.usage is not None: + normalized["usage"] = cast(JsonValue, payload.usage.model_dump(mode="json", exclude_none=True)) + if payload.error is not None: + normalized["error"] = cast(JsonValue, payload.error.model_dump(mode="json", exclude_none=True)) + if compaction_item is not None: + normalized["output"] = [compaction_item] + else: + extra = payload.model_extra or {} + output = extra.get("output") + if isinstance(output, list) and not output: + normalized["output"] = [] + retained_items = _normalize_compact_retained_items((payload.model_extra or {}).get("retained_items")) + if retained_items is not None: + normalized["retained_items"] = retained_items + return normalized + + +def _normalize_compact_retained_items(value: object) -> list[JsonValue] | None: + if not isinstance(value, list): + return None + normalized: list[JsonValue] = [] + for raw_item in value: + item = _json_mapping_from_model_or_mapping(raw_item) + if item is None or item.get("type") != "item_reference": + continue + item_id = item.get("id") + if isinstance(item_id, str) and item_id: + normalized.append({"type": "item_reference", "id": item_id}) return normalized @@ -5423,8 +6432,8 @@ def _normalize_compaction_output_item(item: Mapping[str, JsonValue]) -> dict[str "type": "compaction", "encrypted_content": encrypted_content, } - item_id = item.get("id") - if isinstance(item_id, str) and item_id.strip(): + item_id = normalize_compaction_item_id(item.get("id")) + if item_id is not None: normalized["id"] = item_id status = item.get("status") if isinstance(status, str) and status.strip(): @@ -5509,6 +6518,22 @@ async def _synthetic_compaction_response_stream( yield "data: [DONE]\n\n" +async def _synthetic_compaction_failure_stream( + *, + response_id: str, + error_code: str = "upstream_error", + error_message: str = "Compact response did not include a compaction output item", +) -> AsyncIterator[str]: + yield format_sse_event( + response_failed_event( + error_code, + error_message, + response_id=response_id, + ) + ) + yield "data: [DONE]\n\n" + + async def _transcribe_request( *, request: Request, @@ -5550,7 +6575,7 @@ async def _transcribe_request( async def codex_usage( request: Request, context: ProxyContext = Depends(get_proxy_context), - api_key: ApiKeyData | None = Depends(validate_codex_usage_identity), + api_key: ApiKeyData | None = Depends(validate_codex_provider_usage_identity), ) -> RateLimitStatusPayload: payload = ( await _build_codex_usage_payload_for_api_key(api_key) @@ -5572,8 +6597,11 @@ async def codex_usage( async def codex_consume_rate_limit_reset_credit( request: Request, payload: ConsumeRateLimitResetCreditRequest = Body(...), - api_key: ApiKeyData | None = Depends(validate_codex_usage_identity), + api_key: ApiKeyData | None = Depends(validate_codex_provider_usage_identity), ) -> ConsumeRateLimitResetCreditResponse | JSONResponse: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial if api_key is not None: raise ProxyAuthError("ChatGPT authentication required for usage limit reset credits") redeem_request_id = payload.redeem_request_id.strip() @@ -5800,8 +6828,9 @@ async def _wait_for_first_stream_probe( return_exceptions=True, ) except asyncio.CancelledError: - first_task.cancel() - await asyncio.gather(first_task, return_exceptions=True) + with anyio.CancelScope(shield=True): + first_task.cancel() + await asyncio.gather(first_task, return_exceptions=True) raise @@ -5812,6 +6841,8 @@ async def _probe_stream_startup_error( timeout_seconds: float | None = None, capacity_wait_event: asyncio.Event | None = None, capacity_ready_event: asyncio.Event | None = None, + handoff_task_sink: list[asyncio.Task[str]] | None = None, + service_cleanup_ready_event: asyncio.Event | None = None, ) -> tuple[AsyncIterator[str], ProxyResponseError | OpenAIErrorEnvelopeModel | None]: if timeout_seconds is None: timeout_seconds = _STREAM_STARTUP_ERROR_PROBE_SECONDS @@ -5822,12 +6853,94 @@ async def _probe_stream_startup_error( capacity_wait_event=capacity_wait_event, capacity_ready_event=capacity_ready_event, ) + if service_cleanup_ready_event is not None: + buffered_before_cleanup_ready: list[str] = [] + handoff_deadline = asyncio.get_running_loop().time() + timeout_seconds + while not service_cleanup_ready_event.is_set(): + remaining = handoff_deadline - asyncio.get_running_loop().time() + if remaining <= 0: + with anyio.CancelScope(shield=True): + if not first_task.done(): + first_task.cancel() + await asyncio.gather(first_task, return_exceptions=True) + return ( + _prepend_first(None, stream), + ProxyResponseError( + 503, + openai_error( + "upstream_unavailable", + "Reservation cleanup handoff timed out", + error_type="server_error", + ), + failure_phase="reservation_cleanup_handoff", + failure_detail="cleanup_handoff_timeout", + ), + ) + if not first_task.done(): + cleanup_ready_task = asyncio.create_task(service_cleanup_ready_event.wait()) + try: + await asyncio.wait( + {first_task, cleanup_ready_task}, + timeout=remaining, + return_when=asyncio.FIRST_COMPLETED, + ) + except asyncio.CancelledError: + with anyio.CancelScope(shield=True): + first_task.cancel() + cleanup_ready_task.cancel() + await asyncio.gather(first_task, cleanup_ready_task, return_exceptions=True) + raise + finally: + if not cleanup_ready_task.done(): + cleanup_ready_task.cancel() + await asyncio.gather(cleanup_ready_task, return_exceptions=True) + if service_cleanup_ready_event.is_set(): + break + if not first_task.done(): + continue + try: + first = first_task.result() + except StopAsyncIteration: + return ( + _prepend_first(None, stream), + ProxyResponseError( + 502, + openai_error( + "stream_incomplete", + "Upstream stream ended before reservation cleanup handoff", + error_type="server_error", + ), + ), + ) + except ProxyResponseError as exc: + return _prepend_first(None, stream), exc + if convert_event_errors: + first_error = _stream_event_error_envelope(first) + if first_error is not None: + aclose = getattr(stream, "aclose", None) + if callable(aclose): + await aclose() + return _prepend_first(None, stream), first_error + buffered_before_cleanup_ready.append(first) + first_task = _create_first_stream_probe_task(stream) + + if handoff_task_sink is not None: + handoff_task_sink.append(first_task) + return ( + _prepend_items( + buffered_before_cleanup_ready, + _prepend_first_task(first_task, stream), + ), + None, + ) if not probe_done: # Probe window elapsed before the first item arrived. Hand the still- # running task off to be consumed by the streamed response. asyncio.wait # (rather than wait_for + shield) never cancels the task on timeout, # avoiding the Python 3.14 "exception in shielded future" log when the # upstream later returns an error such as a 429 from the admission gate. + if handoff_task_sink is not None: + handoff_task_sink.append(first_task) return _prepend_first_task(first_task, stream), None try: first = first_task.result() @@ -6004,7 +7117,7 @@ async def _stream_with_cursor_usage_fallback( } logger.info( "cursor_usage_fallback source=stream model=%s prompt_tokens=%s completion_tokens=%s", - payload.model, + safe_log_field(payload.model), prompt_tokens, completion_tokens, ) @@ -6059,8 +7172,8 @@ def _apply_cursor_usage_fallback( ) logger.info( "cursor_usage_fallback source=%s model=%s prompt_tokens=%s completion_tokens=%s", - source, - payload.model, + safe_log_field(source), + safe_log_field(payload.model), prompt_tokens, completion_tokens, ) @@ -6204,9 +7317,12 @@ async def _prepend_initial_sse_heartbeat( request_id, route_family, ) - yield keepalive_frame - async for line in stream: - yield line + try: + yield keepalive_frame + async for line in stream: + yield line + finally: + await _close_responses_stream_best_effort(stream, action="initial heartbeat") def _record_stream_keepalive(surface: str) -> None: @@ -6214,6 +7330,63 @@ def _record_stream_keepalive(surface: str) -> None: stream_keepalive_sent_total.labels(surface=surface).inc() +async def _guard_responses_startup_handoff( + stream: AsyncIterator[str], + *, + startup_task: asyncio.Task[str] | None, + streams_to_close: tuple[AsyncIterator[str], ...], + reservation_cleanup: _ResponsesReservationCleanup, + responses_service_cleanup_ready_event: asyncio.Event, + responses_owner_forward_dispatched_event: asyncio.Event, + responses_owner_forward_rejected_event: asyncio.Event, +) -> AsyncIterator[str]: + try: + async for line in stream: + yield line + finally: + with anyio.CancelScope(shield=True): + release_candidate = startup_task is None + if startup_task is not None: + if startup_task.done(): + release_candidate = startup_task.cancelled() or startup_task.exception() is not None + else: + release_candidate = True + startup_task.cancel() + await asyncio.gather(startup_task, return_exceptions=True) + closed_stream_ids: set[int] = set() + for stream_index, stream_to_close in enumerate(reversed(streams_to_close)): + stream_id = id(stream_to_close) + if stream_id in closed_stream_ids: + continue + closed_stream_ids.add(stream_id) + await _close_responses_stream_best_effort( + stream_to_close, + action=f"startup wrapper {stream_index}", + ) + if release_candidate and _responses_origin_may_release_reservation( + service_cleanup_ready_event=responses_service_cleanup_ready_event, + owner_forward_dispatched_event=responses_owner_forward_dispatched_event, + owner_forward_rejected_event=responses_owner_forward_rejected_event, + ): + await reservation_cleanup.release(action="responses startup handoff") + + +async def _close_responses_stream_best_effort( + stream: AsyncIterator[str], + *, + action: str, +) -> None: + aclose = getattr(stream, "aclose", None) + if not callable(aclose): + return + try: + await aclose() + except asyncio.CancelledError: + logger.debug("Responses %s stream close was cancelled", action) + except Exception: + logger.warning("Failed to close Responses %s stream", action, exc_info=True) + + async def _stream_proxy_errors_as_response_failed(stream: AsyncIterator[str]) -> AsyncIterator[str]: async for line in _stream_response_error_events(stream, owns_reservation=False, reservation=None): yield line @@ -6224,18 +7397,128 @@ async def _stream_response_error_events( *, owns_reservation: bool, reservation: ApiKeyUsageReservationData | None, + reservation_cleanup: _ResponsesReservationCleanup | None = None, + responses_service_cleanup_ready_event: asyncio.Event | None = None, + responses_owner_forward_dispatched_event: asyncio.Event | None = None, + responses_owner_forward_rejected_event: asyncio.Event | None = None, + recovery_stream_factory: Callable[[], AsyncIterator[str]] | None = None, + allow_client_full_history_once: bool = False, + require_durable_recovery_fence: bool = False, ) -> AsyncIterator[str]: + cleanup = reservation_cleanup or _ResponsesReservationCleanup( + owns_reservation=owns_reservation, + reservation=reservation, + scheduler=None, + request_id=ensure_request_id(), + ) + + async def release_owned_reservation() -> None: + if responses_service_cleanup_ready_event is not None and not _responses_origin_may_release_reservation( + service_cleanup_ready_event=responses_service_cleanup_ready_event, + owner_forward_dispatched_event=responses_owner_forward_dispatched_event, + owner_forward_rejected_event=responses_owner_forward_rejected_event, + ): + return + await cleanup.release(action="responses stream cleanup") + + saw_downstream_event = False try: async for line in stream: + if line.startswith("data:") or line.startswith("event:"): + saw_downstream_event = True yield line except ProxyResponseError as exc: - if owns_reservation: - try: - await _release_reservation(reservation) - except Exception: - logger.warning("Failed to release stream reservation after upstream proxy error", exc_info=True) + error_code = exc.payload.get("error", {}).get("code") if isinstance(exc.payload, dict) else None + indefinite_recovery = ( + get_settings().http_responses_session_bridge_ambiguous_continuation_recovery_mode + == "server_indefinite_recovery" + ) + if ( + recovery_stream_factory is not None + and indefinite_recovery + and (not require_durable_recovery_fence or getattr(exc, "http_bridge_durable_recovery_eligible", False)) + and not saw_downstream_event + and error_code + in {"stream_incomplete", "stream_idle_timeout", "upstream_request_timeout", "upstream_unavailable"} + ): + # Keep the client stream alive while the server owns recovery. + # The operation remains serialized by the durable operation + # fingerprint; each new upstream attempt is still at-least-once. + retry_delay = max(1.0, min(30.0, float(exc.retry_after_seconds or 5.0))) + while True: + yield ": codex-lb recovery in progress\n\n" + await asyncio.sleep(retry_delay) + try: + retry_stream = recovery_stream_factory() + retry_saw_downstream_event = False + async for line in retry_stream: + if line.startswith("data:") or line.startswith("event:"): + retry_saw_downstream_event = True + saw_downstream_event = True + yield line + return + except ProxyResponseError as retry_exc: + retry_code = ( + retry_exc.payload.get("error", {}).get("code") if isinstance(retry_exc.payload, dict) else None + ) + if ( + retry_code + not in { + "stream_incomplete", + "stream_idle_timeout", + "upstream_request_timeout", + "upstream_unavailable", + } + or retry_saw_downstream_event + or ( + require_durable_recovery_fence + and not getattr(retry_exc, "http_bridge_durable_recovery_eligible", False) + ) + ): + exc = retry_exc + break + retry_delay = max(1.0, min(30.0, float(retry_exc.retry_after_seconds or retry_delay))) + except (ProxyRateLimitError, ProxyAuthError) as retry_limit_exc: + # A quota revocation or limit can happen between recovery + # attempts. Convert it into the same terminal SSE shape + # as other proxy failures instead of aborting an already + # started response stream without a response.failed event. + exc = ProxyResponseError( + retry_limit_exc.status_code, + openai_error( + retry_limit_exc.code, + retry_limit_exc.message, + error_type=getattr(retry_limit_exc, "error_type", "server_error"), + ), + ) + break + except Exception: + # Recovery admission can also fail before a replacement + # stream is created (for example, a transient database + # failure while reserving usage). Do not let that + # unexpected exception truncate an already-started SSE + # response; the outer cleanup still settles the original + # reservation and emits one terminal response.failed event. + logger.warning("HTTP bridge recovery admission failed", exc_info=True) + exc = ProxyResponseError( + 503, + openai_error( + "bridge_recovery_admission_failed", + "Recovery admission failed; retry shortly.", + error_type="server_error", + ), + retry_after_seconds=5, + ) + break + await release_owned_reservation() envelope = _parse_error_envelope(exc.payload) - _, envelope = _mask_previous_response_not_found_error(envelope, default_status=exc.status_code) + _, envelope = _mask_previous_response_not_found_error( + envelope, + default_status=exc.status_code, + allow_client_full_history_once=( + allow_client_full_history_once and getattr(exc, "http_bridge_durable_recovery_eligible", False) + ), + ) error = envelope.error retry_hint = "" if exc.retry_after_seconds is not None and exc.retry_after_seconds > 0: @@ -6260,10 +7543,17 @@ def _stream_startup_error_response( error: ProxyResponseError | OpenAIErrorEnvelopeModel, *, headers: Mapping[str, str], + allow_client_full_history_once: bool = False, ) -> JSONResponse: if isinstance(error, ProxyResponseError): envelope = _parse_error_envelope(error.payload) - status_code, envelope = _mask_previous_response_not_found_error(envelope, default_status=error.status_code) + status_code, envelope = _mask_previous_response_not_found_error( + envelope, + default_status=error.status_code, + allow_client_full_history_once=( + allow_client_full_history_once and getattr(error, "http_bridge_durable_recovery_eligible", False) + ), + ) startup_headers = dict(headers) if error.retry_after_seconds is not None and error.retry_after_seconds > 0: startup_headers.setdefault("Retry-After", str(error.retry_after_seconds)) @@ -6273,7 +7563,10 @@ def _stream_startup_error_response( envelope.model_dump(mode="json", exclude_none=True), headers=startup_headers, ) - status_code, envelope = _mask_previous_response_not_found_error(error) + status_code, envelope = _mask_previous_response_not_found_error( + error, + allow_client_full_history_once=False, + ) return _logged_error_json_response( request, status_code, @@ -6335,10 +7628,10 @@ def _logged_error_json_response( message, category="proxy_error_response", ) - # codeql[py/stack-trace-exposure] This is an OpenAI-compatible proxy boundary: - # upstream/provider error envelopes intentionally preserve diagnostics for - # clients, while internal exception handlers construct generic error + # Upstream/provider error envelopes intentionally preserve the public + # compatibility contract; internal exception handlers build generic # envelopes before reaching this response helper. + # lgtm [py/stack-trace-exposure] return JSONResponse(status_code=status_code, content=public_content, headers=effective_headers or None) @@ -6382,16 +7675,25 @@ def _is_legacy_proxy_auth_override_type_error(exc: TypeError) -> bool: return "unexpected keyword argument 'request'" in message +def _required_capability_values(headers: Mapping[str, str]) -> tuple[str, ...]: + if isinstance(headers, Headers): + return tuple(headers.getlist(CODEX_LB_REQUIRED_CAPABILITY_HEADER)) + normalized_name = CODEX_LB_REQUIRED_CAPABILITY_HEADER.lower() + return tuple(value for name, value in headers.items() if name.lower() == normalized_name) + + async def _validate_proxy_websocket_request( websocket: WebSocket, *, + allow_required_capability: bool = False, require_api_key: bool = False, ) -> tuple[ApiKeyData | None, JSONResponse | None]: denial = await _websocket_firewall_denial_response(websocket) if denial is not None: return None, denial + capability_header_values = _required_capability_values(websocket.headers) try: - if require_api_key: + if require_api_key or capability_header_values: api_key = await validate_required_proxy_api_key_authorization(websocket.headers.get("authorization")) else: api_key = await _validate_proxy_api_key_authorization_for_connection( @@ -6403,9 +7705,39 @@ async def _validate_proxy_websocket_request( status_code=exc.status_code, content=openai_error(exc.code, exc.message, error_type=exc.error_type), ) + if capability_header_values and not allow_required_capability: + return api_key, JSONResponse( + status_code=400, + content=openai_error( + "required_capability_transport_unsupported", + "Required capability routing is only supported over the Responses WebSocket transport.", + error_type="invalid_request_error", + ), + ) return api_key, None +async def _required_capability_http_transport_denial( + request: Request, + api_key: ApiKeyData | None, +) -> JSONResponse | None: + """Authenticate capability intent and reject unsupported HTTP routing.""" + + if not _required_capability_values(request.headers): + return None + if api_key is None: + await validate_required_proxy_api_key_authorization(request.headers.get("authorization")) + return _logged_error_json_response( + request, + 400, + openai_error( + "required_capability_transport_unsupported", + "Required capability routing is only supported over the Responses WebSocket transport.", + error_type="invalid_request_error", + ), + ) + + def _redact_realtime_live_websocket_scope(websocket: WebSocket, *, path: str) -> None: """Remove opaque live identifiers before Uvicorn emits handshake logs.""" @@ -6526,7 +7858,31 @@ async def _release_reservation(reservation: ApiKeyUsageReservationData | None) - await service.release_usage_reservation(reservation.reservation_id) +async def _release_reservation_best_effort( + reservation: ApiKeyUsageReservationData | None, + *, + action: str, + scheduler: _ResponsesCleanupScheduler | None, + request_id: str, +) -> None: + if reservation is None: + return + try: + await _release_reservation_deferring_cancellation(reservation) + except Exception: + logger.warning("Failed to release API key reservation during %s", action, exc_info=True) + if scheduler is None: + return + scheduler._schedule_cancel_safe_cleanup( + _release_reservation_deferring_cancellation(reservation), + action=f"{action.replace(' ', '_')}_retry", + request_id=request_id, + ) + + async def _finalize_image_reservation( + service: proxy_service_module.ProxyService, + api_key: ApiKeyData | None, reservation: ApiKeyUsageReservationData | None, *, model: str, @@ -6534,47 +7890,18 @@ async def _finalize_image_reservation( output_tokens: int | None, cached_input_tokens: int | None = None, ) -> None: - """Finalize the API-key usage reservation for a ``/v1/images/*`` call. - - The image adapter bypasses the standard stream settlement (``stream_responses`` - is invoked with ``api_key_reservation=None``) because the ``image_generation`` - tool path typically leaves ``response.usage`` empty; charging from - ``tool_usage.image_gen`` is the only source of truth. This helper - finalizes the reservation with the captured image tokens when present, - otherwise releases it. Calling this exactly once per request prevents - the double-billing scenario where both the standard settlement and - the post-hoc image record_usage path increment limits. - - Persistence errors are caught and logged so a transient DB/session - failure during the tail accounting cannot turn a successfully - generated image into a user-facing 500 (non-streaming) or an - abrupt stream termination (streaming). This mirrors the - best-effort accounting policy used by - ``ProxyService._settle_stream_api_key_usage``. - """ + """Transfer image-token settlement to tracked persistence ownership.""" if reservation is None: return - try: - if not input_tokens and not output_tokens: - await _release_reservation(reservation) - return - async with get_background_session() as session: - service = ApiKeysService(ApiKeysRepository(session)) - await service.finalize_usage_reservation( - reservation.reservation_id, - model=model, - input_tokens=int(input_tokens or 0), - output_tokens=int(output_tokens or 0), - cached_input_tokens=int(cached_input_tokens or 0), - service_tier=None, - ) - except Exception: - logger.warning( - "failed to finalize image reservation reservation_id=%s model=%s", - reservation.reservation_id, - model, - exc_info=True, - ) + await service.settle_image_api_key_usage( + api_key, + reservation, + model=model, + input_tokens=input_tokens, + output_tokens=output_tokens, + cached_input_tokens=cached_input_tokens, + request_id=get_request_id() or reservation.reservation_id, + ) async def _settle_source_reservation( @@ -6607,8 +7934,8 @@ async def _settle_source_reservation( except Exception: logger.warning( "failed to settle source reservation reservation_id=%s model=%s", - reservation.reservation_id, - model, + safe_log_field(reservation.reservation_id), + safe_log_field(model), exc_info=True, ) try: @@ -6681,14 +8008,14 @@ async def _log_source_chat_completion( except Exception: logger.warning( "failed to write source request log source_id=%s model=%s status=%s", - source.id, - model, + safe_log_field(source.id), + safe_log_field(model), status, exc_info=True, ) -async def _aclose_stream(stream: AsyncIterator[bytes]) -> None: +async def _aclose_stream(stream: AsyncIterator[object]) -> None: aclose = getattr(stream, "aclose", None) if aclose is not None: await aclose() @@ -6729,9 +8056,7 @@ def _effective_model_for_api_key(api_key: ApiKeyData | None, requested_model: st def _effective_optional_model_for_api_key(api_key: ApiKeyData | None, requested_model: str | None) -> str | None: - if api_key is None or api_key.enforced_model is None: - return requested_model - return api_key.enforced_model + return effective_model_for_api_key(api_key, requested_model) def _compact_request_service_tier(payload: ResponsesCompactRequest) -> str | None: @@ -6833,14 +8158,30 @@ def _collect_output_item_event( payload: dict[str, JsonValue], output_items: dict[int, dict[str, JsonValue]], ) -> None: - _collect_public_output_item_event_shared(payload, output_items) + event_type = payload.get("type") + if event_type not in ("response.output_item.added", "response.output_item.done"): + return + output_index = payload.get("output_index") + item = payload.get("item") + if not isinstance(output_index, int) or not isinstance(item, dict): + return + output_items[output_index] = dict(item) def _merge_collected_output_items( response: Mapping[str, JsonValue], output_items: dict[int, dict[str, JsonValue]], ) -> dict[str, JsonValue]: - return _merge_public_response_output_items(response, output_items) + merged = dict(response) + if not output_items: + return merged + + existing_output = response.get("output") + if isinstance(existing_output, list) and existing_output: + return merged + + merged["output"] = [item for _, item in sorted(output_items.items())] + return merged async def _normalize_public_responses_stream( @@ -7600,6 +8941,63 @@ def _normalize_public_response_mapping( return normalized, None +def _normalize_public_output_item(item: Mapping[str, JsonValue]) -> dict[str, JsonValue] | None: + item_type = item.get("type") + if item_type == "reasoning": + return _normalize_reasoning_output_item(item) + if isinstance(item_type, str) and _is_public_passthrough_output_item_type(item_type): + return dict(item) + text_value = _extract_public_output_item_text(item) + if text_value is None: + return None + normalized: dict[str, JsonValue] = { + "type": "message", + "role": "assistant", + "status": item.get("status") if isinstance(item.get("status"), str) else "completed", + "content": [{"type": "output_text", "text": text_value}], + } + item_id = item.get("id") + if isinstance(item_id, str) and item_id: + normalized["id"] = item_id + return normalized + + +def _normalize_reasoning_output_item(item: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + """Remove renderer-only blank HTML comments from reasoning summaries. + + Recent Codex reasoning summaries can include a standalone ```` + markdown placeholder after the visible summary heading. The Codex TUI renders + reasoning summary text directly, so proxying that inert marker verbatim makes + it visible between tool calls. Limit the cleanup to reasoning summary text so + assistant/user-visible content and non-empty HTML comments remain untouched. + """ + + normalized = dict(item) + summary = item.get("summary") + if not isinstance(summary, list): + return normalized + + normalized_summary: list[JsonValue] = [] + changed = False + for part in summary: + if not is_json_mapping(part): + normalized_summary.append(part) + continue + text = part.get("text") + if part.get("type") != "summary_text" or not isinstance(text, str): + normalized_summary.append(dict(part)) + continue + cleaned = _strip_blank_html_comment_lines(text) + normalized_part = dict(part) + normalized_part["text"] = cleaned + normalized_summary.append(normalized_part) + changed = changed or cleaned != text + + if changed: + normalized["summary"] = normalized_summary + return normalized + + async def _normalize_reasoning_summary_stream(stream: AsyncIterator[str]) -> AsyncIterator[str]: pending: dict[tuple[str | None, int | None, int | None], list[tuple[dict[str, JsonValue], str]]] = {} @@ -7678,6 +9076,42 @@ def flush(key: tuple[str | None, int | None, int | None]) -> list[str]: yield buffered +def _is_public_passthrough_output_item_type(item_type: str) -> bool: + if item_type in _PUBLIC_RESPONSE_OUTPUT_ITEM_TYPES: + return True + return item_type.endswith("_call") or item_type.endswith("_call_output") + + +def _extract_public_output_item_text(item: Mapping[str, JsonValue]) -> str | None: + direct_text = item.get("text") + if isinstance(direct_text, str) and direct_text: + return direct_text + content = item.get("content") + if is_json_mapping(content): + content_parts: list[Mapping[str, JsonValue]] = [content] + elif isinstance(content, list): + content_parts = [part for part in content if is_json_mapping(part)] + else: + content_parts = [] + parts: list[str] = [] + for part in content_parts: + part_type = part.get("type") + if isinstance(part_type, str) and part_type in _PUBLIC_RESPONSE_TEXT_PART_TYPES: + text = part.get("text") + if isinstance(text, str) and text: + parts.append(text) + continue + text = part.get("text") + if isinstance(text, str) and text: + parts.append(text) + if parts: + return "".join(parts) + summary = item.get("summary") + if isinstance(summary, str) and summary: + return summary + return None + + def _looks_like_sse_data_block(event_block: str) -> bool: return "data:" in event_block @@ -7778,13 +9212,48 @@ def _is_previous_response_not_found_public_error(error_value: OpenAIError | None ) +def _http_bridge_recovery_request_eligible( + payload: ResponsesRequest, + *, + bridge_active: bool, + headers: Mapping[str, str] | None = None, +) -> bool: + turn_state_anchor = proxy_affinity_module._sticky_key_from_turn_state_header(headers or {}) + if not bridge_active or (payload.previous_response_id is None and turn_state_anchor is None): + return False + settings = proxy_service_module.get_settings() + if not getattr(settings, "http_responses_session_bridge_operation_ledger_enabled", True): + return False + # Turn-state-only requests are admitted to the recovery-capable stream so + # the submit path can first prove a durable predecessor by advancing its + # operation anchor. The streaming layer marks an exception recovery-safe + # only after that proof; fresh first turns remain fail-closed there. + if proxy_service_module._responses_request_contains_input_image( + payload + ) or proxy_service_module._responses_request_uses_image_generation(payload): + return False + payload_bytes = len(json.dumps(payload.to_payload(), ensure_ascii=True, separators=(",", ":")).encode("utf-8")) + return payload_bytes <= proxy_service_module._ws_transport_payload_budget_bytes(settings) + + def _mask_previous_response_not_found_error( envelope: OpenAIErrorEnvelopeModel, *, default_status: int | None = None, + allow_client_full_history_once: bool = False, ) -> tuple[int, OpenAIErrorEnvelopeModel]: if not _is_previous_response_not_found_public_error(envelope.error): return default_status if default_status is not None else _status_for_error(envelope.error), envelope + # In recovery-first mode, preserve the upstream-shaped 400 so Codex can + # drop the ambiguous previous_response_id anchor and resend full local + # history. This is intentionally opt-in because the resend is at-least-once + # and may duplicate an upstream response that was accepted but not observed. + if ( + allow_client_full_history_once + and get_settings().http_responses_session_bridge_ambiguous_continuation_recovery_mode + == "client_full_history_once" + ): + return default_status if default_status is not None else 400, envelope return ( 502, OpenAIErrorEnvelopeModel( diff --git a/app/modules/proxy/continuity.py b/app/modules/proxy/continuity.py index 463497aeb2..fc92906e75 100644 --- a/app/modules/proxy/continuity.py +++ b/app/modules/proxy/continuity.py @@ -2,14 +2,24 @@ from __future__ import annotations +import logging from collections.abc import Mapping +from hashlib import sha256 +from typing import Protocol from app.core.clients.proxy import ProxyResponseError from app.core.errors import openai_error HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KIND = "internal_unanchored_parallel" HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KEY_PREFIX = "account-neutral-replay:v1:" -HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_REBINDABLE_KINDS = frozenset({"prompt_cache", "session_header", "turn_state_header"}) +# These are canonical lanes whose exact aliases may move only after the +# existing full-resend validator has proved the request account-neutral. A +# thread lane is hard during ordinary use, just like a session-header lane; +# omitting it here would accidentally remove safe owner-unavailable recovery +# merely because Codex now supplies a more precise canonical identity. +HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_REBINDABLE_KINDS = frozenset( + {"prompt_cache", "session_header", "thread_header", "turn_state_header"} +) _HTTP_BRIDGE_SESSION_AFFINITY_HEADERS = frozenset( { "session_id", @@ -20,6 +30,7 @@ "x-codex-turn-state", } ) +logger = logging.getLogger("app.modules.proxy.continuity") def make_http_bridge_account_neutral_replay_key(nonce: str) -> tuple[str, str]: @@ -51,6 +62,24 @@ def without_http_bridge_session_affinity_headers(headers: Mapping[str, str]) -> } +class _ReconnectPreferredOwner(Protocol): + preferred_account_id: str | None + file_required_preferred_account: bool + + +def resolve_reconnect_preferred_account_id( + request_state: _ReconnectPreferredOwner, + session_account_id: str, + require_preferred_account: bool, + account_neutral_recovery: bool, +) -> str | None: + if request_state.file_required_preferred_account: + return request_state.preferred_account_id or session_account_id + if require_preferred_account or account_neutral_recovery: + return request_state.preferred_account_id + return None + + def resolve_required_account_id(*owners: tuple[str, str | None]) -> str | None: """Return one proven owner or fail closed when hard sources disagree.""" resolved = [(source, account_id) for source, account_id in owners if account_id is not None] @@ -63,6 +92,15 @@ def resolve_required_account_id(*owners: tuple[str, str | None]) -> str | None: # side would silently abandon the other, so conflicts are never ordered # by caller precedence or softened into ordinary affinity fallback. sources = ", ".join(source for source, _account_id in resolved) + owner_hashes = ", ".join( + f"{source}={sha256(account_id.encode()).hexdigest()[:12]}" for source, account_id in resolved + ) + logger.warning( + "continuity_owner_conflict sources=%s conflicting_sources=%s owner_hashes=%s", + sources, + ", ".join(conflicting_sources), + owner_hashes, + ) raise ProxyResponseError( 502, openai_error( diff --git a/app/modules/proxy/durable_bridge_coordinator.py b/app/modules/proxy/durable_bridge_coordinator.py index 26b92190b6..ad3c7df0c3 100644 --- a/app/modules/proxy/durable_bridge_coordinator.py +++ b/app/modules/proxy/durable_bridge_coordinator.py @@ -16,14 +16,15 @@ from app.modules.proxy.durable_bridge_repository import ( DurableBridgeAliasRegistration, DurableBridgeAliasRegistrationReceipt, + DurableBridgeOperationEventInput, + DurableBridgeOperationSnapshot, DurableBridgeRecoveryAttemptSnapshot, DurableBridgeRepository, DurableBridgeRetryCircuitSnapshot, DurableBridgeSessionSnapshot, + DurableBridgeTranscriptTurn, durable_bridge_api_key_scope, - durable_bridge_hash, ) -from app.modules.proxy.response_transition_manifest import ResponseTransitionManifest _DURABLE_TURN_STATE_ALIAS = "turn_state" _DURABLE_PREVIOUS_RESPONSE_ALIAS = "previous_response_id" @@ -47,13 +48,7 @@ class DurableBridgeLookup: latest_input_full_fingerprint: str | None = None model: str | None = None latest_pending_tool_calls: dict[str, str] | None = None - latest_response_transition_manifest: ResponseTransitionManifest | None = None owner_process_epoch: str | None = None - recovery_required_anchor_hash: str | None = None - recovery_required_account_id: str | None = None - recovery_required_attempt_fingerprint: str | None = None - recovery_required_attempt_request_id: str | None = None - recovery_required_at: datetime | None = None def lease_is_active(self, *, now: datetime) -> bool: if self.owner_instance_id is None: @@ -66,14 +61,6 @@ def lease_is_active(self, *, now: datetime) -> bool: # on the anchored-lookup hot path. return to_utc_naive(self.lease_expires_at) > to_utc_naive(now) - def recovery_is_required_for_latest_anchor(self) -> bool: - return bool( - self.latest_response_id is not None - and self.account_id is not None - and self.recovery_required_anchor_hash == durable_bridge_hash(self.latest_response_id) - and self.recovery_required_account_id == self.account_id - ) - class DurableBridgeSessionCoordinator: def __init__(self, session_factory: Callable[[], AsyncSession]) -> None: @@ -320,8 +307,6 @@ async def claim_live_session( allow_takeover: bool, owner_process_epoch: str, force_owner_epoch_advance: bool = False, - expected_takeover_owner_instance_id: str | None = None, - expected_takeover_owner_process_epoch: str | None = None, ) -> DurableBridgeLookup: api_key_scope = durable_bridge_api_key_scope(api_key_id) async with self._session() as session: @@ -339,8 +324,6 @@ async def claim_live_session( allow_takeover=allow_takeover, owner_process_epoch=owner_process_epoch, force_owner_epoch_advance=force_owner_epoch_advance, - expected_takeover_owner_instance_id=expected_takeover_owner_instance_id, - expected_takeover_owner_process_epoch=expected_takeover_owner_process_epoch, ) return _to_lookup(snapshot) @@ -357,7 +340,6 @@ async def renew_live_session( latest_input_item_count: int | None = None, latest_input_full_fingerprint: str | None = None, latest_pending_tool_calls: Mapping[str, str] | None = None, - latest_response_transition_manifest: ResponseTransitionManifest | None = None, state: HttpBridgeSessionState | None = None, ) -> DurableBridgeLookup | None: del api_key_id @@ -372,7 +354,6 @@ async def renew_live_session( latest_input_item_count=latest_input_item_count, latest_input_full_fingerprint=latest_input_full_fingerprint, latest_pending_tool_calls=latest_pending_tool_calls, - latest_response_transition_manifest=latest_response_transition_manifest, state=state, ) if snapshot is None: @@ -435,205 +416,390 @@ async def clear_live_session_response_anchor( return None return _to_lookup(snapshot) - async def mark_live_session_recovery_required( + async def record_recovery_attempt( self, *, session_id: str, + api_key_id: str | None, instance_id: str, owner_epoch: int, - account_id: str, - rejected_response_id: str, - ) -> bool: + request_fingerprint: str, + request_id: str, + account_id: str | None, + model: str | None, + replay_safe: bool, + ) -> DurableBridgeRecoveryAttemptSnapshot | None: + del api_key_id async with self._session() as session: - return await DurableBridgeRepository(session).mark_recovery_required( + return await DurableBridgeRepository(session).record_recovery_attempt( session_id=session_id, instance_id=instance_id, owner_epoch=owner_epoch, + request_fingerprint=request_fingerprint, + request_id=request_id, account_id=account_id, - rejected_response_id=rejected_response_id, + model=model, + replay_safe=replay_safe, ) - async def claim_live_session_recovery_attempt( + async def lookup_recovery_attempt( self, *, session_id: str, + request_fingerprint: str, + ) -> DurableBridgeRecoveryAttemptSnapshot | None: + async with self._session() as session: + return await DurableBridgeRepository(session).lookup_recovery_attempt( + session_id=session_id, + request_fingerprint=request_fingerprint, + ) + + async def mark_recovery_attempt_replayed( + self, + *, + session_id: str, + api_key_id: str | None, instance_id: str, owner_epoch: int, - account_id: str, - rejected_response_id: str, - attempt_fingerprint: str, - request_id: str, + request_fingerprint: str, + response_id: str | None = None, ) -> bool: + del api_key_id async with self._session() as session: - return await DurableBridgeRepository(session).claim_recovery_required_attempt( + return await DurableBridgeRepository(session).mark_recovery_attempt_replayed( session_id=session_id, instance_id=instance_id, owner_epoch=owner_epoch, - account_id=account_id, - rejected_response_id=rejected_response_id, - attempt_fingerprint=attempt_fingerprint, - request_id=request_id, + request_fingerprint=request_fingerprint, + response_id=response_id, ) - async def rollback_live_session_recovery_attempt_before_dispatch( + async def rollback_recovery_attempt_replayed( self, *, session_id: str, api_key_id: str | None, instance_id: str, owner_epoch: int, - account_id: str, - attempt_fingerprint: str, - request_id: str, - journal_request_id: str, + request_fingerprint: str, ) -> bool: + del api_key_id async with self._session() as session: - return await DurableBridgeRepository(session).rollback_recovery_required_attempt_before_dispatch( + return await DurableBridgeRepository(session).rollback_recovery_attempt_replayed( session_id=session_id, - api_key_scope=durable_bridge_api_key_scope(api_key_id), instance_id=instance_id, owner_epoch=owner_epoch, - account_id=account_id, - attempt_fingerprint=attempt_fingerprint, - request_id=request_id, - journal_request_id=journal_request_id, + request_fingerprint=request_fingerprint, ) - async def claim_and_record_live_session_recovery_attempt( + async def rollback_recovery_attempt_before_dispatch( self, *, session_id: str, + api_key_id: str | None, instance_id: str, owner_epoch: int, - account_id: str, - rejected_response_id: str, - attempt_fingerprint: str, - claim_request_id: str, - journal_request_id: str, - model: str | None, - ) -> DurableBridgeRecoveryAttemptSnapshot | None: + request_fingerprint: str, + ) -> bool: + del api_key_id async with self._session() as session: - return await DurableBridgeRepository(session).claim_recovery_required_attempt_with_journal( + return await DurableBridgeRepository(session).rollback_recovery_attempt_before_dispatch( session_id=session_id, instance_id=instance_id, owner_epoch=owner_epoch, - account_id=account_id, - rejected_response_id=rejected_response_id, - attempt_fingerprint=attempt_fingerprint, - claim_request_id=claim_request_id, - journal_request_id=journal_request_id, - model=model, + request_fingerprint=request_fingerprint, ) - async def record_recovery_attempt( + async def record_operation( self, *, + operation_id: str, session_id: str, - api_key_id: str | None, instance_id: str, owner_epoch: int, request_fingerprint: str, - request_id: str, account_id: str | None, model: str | None, - replay_safe: bool, - ) -> DurableBridgeRecoveryAttemptSnapshot | None: - del api_key_id + parent_response_id: str | None, + api_key_scope: str | None = None, + request_text: str | None = None, + recovery_attempt_session_id: str | None = None, + recovery_attempt_owner_epoch: int | None = None, + recovery_attempt_fingerprint: str | None = None, + recovery_attempt_consumed: bool = False, + ) -> DurableBridgeOperationSnapshot | None: async with self._session() as session: - return await DurableBridgeRepository(session).record_recovery_attempt( + return await DurableBridgeRepository(session).record_operation( + operation_id=operation_id, session_id=session_id, instance_id=instance_id, owner_epoch=owner_epoch, request_fingerprint=request_fingerprint, - request_id=request_id, + api_key_scope=api_key_scope, account_id=account_id, model=model, - replay_safe=replay_safe, + parent_response_id=parent_response_id, + request_text=request_text, + recovery_attempt_session_id=recovery_attempt_session_id, + recovery_attempt_owner_epoch=recovery_attempt_owner_epoch, + recovery_attempt_fingerprint=recovery_attempt_fingerprint, + recovery_attempt_consumed=recovery_attempt_consumed, ) - async def lookup_recovery_attempt( + async def get_operation_events(self, *, operation_id: str) -> list[str]: + async with self._session() as session: + return await DurableBridgeRepository(session).get_operation_events(operation_id=operation_id) + + async def get_replayable_transcript( self, *, + response_id: str, + max_turns: int = 128, + max_bytes: int = 8 * 1024 * 1024, + ) -> list[DurableBridgeTranscriptTurn] | None: + async with self._session() as session: + return await DurableBridgeRepository(session).get_replayable_transcript( + response_id=response_id, + max_turns=max_turns, + max_bytes=max_bytes, + ) + + async def purge_operation_spool(self, *, cutoff: datetime, batch_size: int = 500) -> int: + async with self._session() as session: + return await DurableBridgeRepository(session).purge_operation_spool( + cutoff=cutoff, + batch_size=batch_size, + ) + + async def append_operation_event( + self, + *, + operation_id: str, session_id: str, - request_fingerprint: str, - ) -> DurableBridgeRecoveryAttemptSnapshot | None: + instance_id: str, + owner_epoch: int, + event_text: str, + max_bytes: int, + ) -> bool: async with self._session() as session: - return await DurableBridgeRepository(session).lookup_recovery_attempt( + return await DurableBridgeRepository(session).append_operation_event( + operation_id=operation_id, session_id=session_id, - request_fingerprint=request_fingerprint, + instance_id=instance_id, + owner_epoch=owner_epoch, + event_text=event_text, + max_bytes=max_bytes, ) - async def mark_recovery_attempt_replayed( + async def append_terminal_operation_event( self, *, + operation_id: str, session_id: str, - api_key_id: str | None, instance_id: str, owner_epoch: int, - request_fingerprint: str, + event_text: str, + max_bytes: int, + state: str, + expected_recovery_dispatch_count: int = 0, response_id: str | None = None, ) -> bool: - del api_key_id async with self._session() as session: - return await DurableBridgeRepository(session).mark_recovery_attempt_replayed( + return await DurableBridgeRepository(session).append_terminal_operation_event( + operation_id=operation_id, session_id=session_id, instance_id=instance_id, owner_epoch=owner_epoch, - request_fingerprint=request_fingerprint, + event_text=event_text, + max_bytes=max_bytes, + state=state, + expected_recovery_dispatch_count=expected_recovery_dispatch_count, response_id=response_id, ) - async def settle_marker_recovery_completed( + async def append_operation_events( + self, + *, + events: Sequence[DurableBridgeOperationEventInput], + max_bytes: int, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).append_operation_events( + events=events, + max_bytes=max_bytes, + ) + + async def finalize_operation_event_spool( self, *, + operation_id: str, session_id: str, - api_key_id: str | None, instance_id: str, owner_epoch: int, - account_id: str, - request_fingerprint: str, - claim_request_id: str, - request_id: str, - response_id: str, - input_item_count: int, - input_full_fingerprint: str, - pending_tool_calls: Mapping[str, str], - response_transition_manifest: ResponseTransitionManifest | None, - lease_ttl_seconds: float, ) -> bool: async with self._session() as session: - return await DurableBridgeRepository(session).settle_marker_recovery_completed( + return await DurableBridgeRepository(session).finalize_operation_event_spool( + operation_id=operation_id, session_id=session_id, - api_key_scope=durable_bridge_api_key_scope(api_key_id), instance_id=instance_id, owner_epoch=owner_epoch, - account_id=account_id, - request_fingerprint=request_fingerprint, - claim_request_id=claim_request_id, - request_id=request_id, + ) + + async def settle_terminal_append_failure( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + state: str, + expected_response_id: str | None, + expected_recovery_dispatch_count: int = 0, + alternate_expected_response_id: str | None = None, + response_id: str | None = None, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).settle_terminal_append_failure( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + state=state, + expected_response_id=expected_response_id, + expected_recovery_dispatch_count=expected_recovery_dispatch_count, + alternate_expected_response_id=alternate_expected_response_id, response_id=response_id, - input_item_count=input_item_count, - input_full_fingerprint=input_full_fingerprint, - pending_tool_calls=pending_tool_calls, - response_transition_manifest=response_transition_manifest, - lease_ttl_seconds=lease_ttl_seconds, ) - async def rollback_recovery_attempt_replayed( + async def update_operation( self, *, + operation_id: str, session_id: str, - api_key_id: str | None, instance_id: str, owner_epoch: int, - request_fingerprint: str, + state: str, + response_id: str | None = None, ) -> bool: - del api_key_id async with self._session() as session: - return await DurableBridgeRepository(session).rollback_recovery_attempt_replayed( + return await DurableBridgeRepository(session).update_operation( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + state=state, + response_id=response_id, + ) + + async def get_operation(self, *, operation_id: str) -> DurableBridgeOperationSnapshot | None: + async with self._session() as session: + return await DurableBridgeRepository(session).get_operation(operation_id=operation_id) + + async def reset_operation_event_spool( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).reset_operation_event_spool( + operation_id=operation_id, session_id=session_id, instance_id=instance_id, owner_epoch=owner_epoch, + ) + + async def claim_unknown_operation_for_recovery( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + max_recovery_dispatches: int | None = None, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + max_recovery_dispatches=max_recovery_dispatches, + ) + + async def mark_operation_unknown( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + restore_recovery_dispatch_claim: bool = False, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).mark_operation_unknown( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + restore_recovery_dispatch_claim=restore_recovery_dispatch_claim, + ) + + async def rollback_operation_before_dispatch( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).rollback_operation_before_dispatch( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + ) + + async def get_operation_by_fingerprint( + self, + *, + request_fingerprint: str, + api_key_scope: str | None = None, + ) -> DurableBridgeOperationSnapshot | None: + async with self._session() as session: + return await DurableBridgeRepository(session).get_operation_by_fingerprint( + request_fingerprint=request_fingerprint, + api_key_scope=api_key_scope, + ) + + async def get_latest_completed_operation( + self, + *, + session_id: str, + parent_response_id: str, + request_fingerprint: str | None = None, + ) -> DurableBridgeOperationSnapshot | None: + async with self._session() as session: + return await DurableBridgeRepository(session).get_latest_completed_operation( + session_id=session_id, + parent_response_id=parent_response_id, + request_fingerprint=request_fingerprint, + ) + + async def get_latest_completed_operation_any_session( + self, + *, + parent_response_id: str, + api_key_scope: str | None = None, + request_fingerprint: str | None = None, + ) -> DurableBridgeOperationSnapshot | None: + async with self._session() as session: + return await DurableBridgeRepository(session).get_latest_completed_operation_any_session( + parent_response_id=parent_response_id, + api_key_scope=api_key_scope, request_fingerprint=request_fingerprint, ) @@ -721,7 +887,6 @@ async def register_previous_response_id( input_item_count: int | None = None, input_full_fingerprint: str | None = None, pending_tool_calls: Mapping[str, str] | None = None, - response_transition_manifest: ResponseTransitionManifest | None = None, ) -> DurableBridgeAliasRegistration: api_key_scope = durable_bridge_api_key_scope(api_key_id) async with self._session() as session: @@ -737,7 +902,6 @@ async def register_previous_response_id( latest_input_item_count=input_item_count, latest_input_full_fingerprint=input_full_fingerprint, latest_pending_tool_calls=pending_tool_calls, - latest_response_transition_manifest=response_transition_manifest, ) async def register_session_header( @@ -783,10 +947,4 @@ def _to_lookup(snapshot: DurableBridgeSessionSnapshot) -> DurableBridgeLookup: latest_input_full_fingerprint=snapshot.latest_input_full_fingerprint, model=snapshot.model, latest_pending_tool_calls=snapshot.latest_pending_tool_calls, - latest_response_transition_manifest=snapshot.latest_response_transition_manifest, - recovery_required_anchor_hash=snapshot.recovery_required_anchor_hash, - recovery_required_account_id=snapshot.recovery_required_account_id, - recovery_required_attempt_fingerprint=snapshot.recovery_required_attempt_fingerprint, - recovery_required_attempt_request_id=snapshot.recovery_required_attempt_request_id, - recovery_required_at=snapshot.recovery_required_at, ) diff --git a/app/modules/proxy/durable_bridge_repository.py b/app/modules/proxy/durable_bridge_repository.py index 19ff694242..787b5a4b91 100644 --- a/app/modules/proxy/durable_bridge_repository.py +++ b/app/modules/proxy/durable_bridge_repository.py @@ -7,9 +7,9 @@ from datetime import datetime, timedelta from enum import StrEnum from hashlib import sha256 -from typing import Any +from typing import Any, cast -from sqlalchemy import Row, and_, case, delete, func, or_, select, text, true, update +from sqlalchemy import Row, and_, case, delete, exists, func, or_, select, text, true, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.exc import IntegrityError @@ -17,6 +17,8 @@ from app.core.utils.time import to_utc_naive, utcnow from app.db.models import ( + HttpBridgeOperationEvent, + HttpBridgeOperationRecord, HttpBridgeRecoveryAttemptRecord, HttpBridgeRecoveryAttemptState, HttpBridgeRetryCircuit, @@ -31,11 +33,6 @@ HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_REBINDABLE_KINDS, is_http_bridge_account_neutral_replay, ) -from app.modules.proxy.response_transition_manifest import ( - ResponseTransitionManifest, - decode_response_transition_manifest, - encode_response_transition_manifest, -) _ANONYMOUS_API_KEY_SCOPE = "__anonymous__" REQUIRED_DURABLE_BRIDGE_TABLES = ( @@ -43,21 +40,15 @@ "http_bridge_session_aliases", "http_bridge_retry_circuits", "http_bridge_recovery_attempts", - "http_bridge_rowless_recovery_authorities", + "http_bridge_operations", + "http_bridge_operation_events", ) -REQUIRED_DURABLE_BRIDGE_COLUMNS = { - "http_bridge_sessions": ( - "latest_response_transition_manifest_json", - "recovery_required_attempt_request_id", - ), - "http_bridge_rowless_recovery_authorities": ( - "origin_marker_session_id", - "authorization_mode", - "authorization_proof_sha256", - ), -} DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS = 3600.0 _PURGE_CLOSED_BATCH_SIZE = 500 +# Claim retry budget: insert races and epoch-CAS losses re-read and retry; +# each round has a winner, so a small budget converges under any realistic +# same-row claim contention. +_CLAIM_CAS_ATTEMPTS = 5 _SESSION_ID_LOOKUP_CHUNK_SIZE = 500 @@ -92,7 +83,20 @@ def durable_bridge_api_key_scope(api_key_id: str | None) -> str: def durable_bridge_hash(value: str) -> str: - return sha256(value.encode("utf-8")).hexdigest() + # These digests are deterministic storage/fingerprint keys, not password + # verifiers. Preserve the historical digest for database compatibility. + # lgtm [py/weak-sensitive-data-hashing] + return sha256(value.encode("utf-8"), usedforsecurity=False).hexdigest() + + +def durable_bridge_operation_fingerprint(*, api_key_scope: str, request_text: str) -> str: + """Hash the logical turn together with its authorization namespace.""" + return durable_bridge_hash(f"{api_key_scope}:{request_text}") + + +def durable_bridge_operation_id(session_id: str, request_fingerprint: str) -> str: + """Derive a stable, non-secret operation key for a continuity-bound turn.""" + return f"op_{durable_bridge_hash(f'{session_id}:{request_fingerprint}')[:64]}" def _encode_pending_tool_calls(response_id: str, value: Mapping[str, str] | None) -> str | None: @@ -147,21 +151,7 @@ class DurableBridgeSessionSnapshot: last_seen_at: datetime closed_at: datetime | None latest_pending_tool_calls: dict[str, str] | None = None - latest_response_transition_manifest: ResponseTransitionManifest | None = None owner_process_epoch: str | None = None - recovery_required_anchor_hash: str | None = None - recovery_required_account_id: str | None = None - recovery_required_attempt_fingerprint: str | None = None - recovery_required_attempt_request_id: str | None = None - recovery_required_at: datetime | None = None - - def recovery_is_required_for_latest_anchor(self) -> bool: - return bool( - self.latest_response_id is not None - and self.account_id is not None - and self.recovery_required_anchor_hash == durable_bridge_hash(self.latest_response_id) - and self.recovery_required_account_id == self.account_id - ) @dataclass(frozen=True, slots=True) @@ -187,6 +177,37 @@ class DurableBridgeRecoveryAttemptSnapshot: response_id: str | None +@dataclass(frozen=True, slots=True) +class DurableBridgeOperationSnapshot: + operation_id: str + session_id: str + request_fingerprint: str + account_id: str | None + model: str | None + parent_response_id: str | None + state: str + response_id: str | None + recovery_dispatch_count: int = 0 + request_text: str | None = None + event_spool_complete: bool = True + created: bool = False + + +@dataclass(frozen=True, slots=True) +class DurableBridgeTranscriptTurn: + operation: DurableBridgeOperationSnapshot + events: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class DurableBridgeOperationEventInput: + operation_id: str + session_id: str + instance_id: str + owner_epoch: int + event_text: str + + class DurableBridgeRepository: def __init__(self, session: AsyncSession) -> None: self._session = session @@ -580,11 +601,21 @@ async def claim_session( allow_takeover: bool, owner_process_epoch: str, force_owner_epoch_advance: bool = False, - expected_takeover_owner_instance_id: str | None = None, - expected_takeover_owner_process_epoch: str | None = None, ) -> DurableBridgeSessionSnapshot: session_key_hash = durable_bridge_hash(session_key_value) - for attempt in range(2): + # ``allow_takeover`` was decided by the caller against a pre-claim + # lookup. Once another claimant has demonstrably written this row under + # us (lost CAS, or lost insert race), that decision is stale: the row + # we re-read may now carry the winner's live lease, and reusing the + # permission would let the loser steal it. Revalidate from the fresh + # read instead — a live foreign owner then fails closed exactly like a + # non-takeover claim, which surfaces as the correct cross-replica + # "retry to reach the correct replica" response. + contended = False + # Bounded retry budget shared by the insert race (IntegrityError) and + # the epoch CAS: every round has a winner, so a loser converges after + # at most one fresh read per concurrent claimant. + for attempt in range(_CLAIM_CAS_ATTEMPTS): now = utcnow() lease_expires_at = now + timedelta(seconds=max(1.0, lease_ttl_seconds)) row = await self._session.execute( @@ -621,82 +652,153 @@ async def claim_session( await self._commit_writer_section() except IntegrityError: await self._session.rollback() - if attempt == 0: + if attempt < _CLAIM_CAS_ATTEMPTS - 1: + contended = True continue raise - await self._session.refresh(record) - return _to_snapshot_required(record) + # Same reason the CAS path builds its own snapshot: another + # same-instance claimant can advance this brand-new row before + # a refresh runs, and returning that epoch would hand two + # claimants the same fence. + inserted_id = record.id + return DurableBridgeSessionSnapshot( + id=inserted_id, + session_key_kind=session_key_kind, + session_key_value=session_key_value, + session_key_hash=session_key_hash, + api_key_scope=api_key_scope, + owner_instance_id=instance_id, + owner_process_epoch=owner_process_epoch, + owner_epoch=1, + lease_expires_at=lease_expires_at, + state=HttpBridgeSessionState.ACTIVE, + account_id=account_id, + model=model, + service_tier=service_tier, + latest_turn_state=latest_turn_state, + latest_response_id=latest_response_id, + latest_input_item_count=None, + latest_input_full_fingerprint=None, + latest_pending_tool_calls=None, + last_seen_at=now, + closed_at=None, + ) - state_allows_takeover = existing.state in { - HttpBridgeSessionState.DRAINING, - HttpBridgeSessionState.CLOSED, - } + state_closed = existing.state == HttpBridgeSessionState.CLOSED + owner_absent = existing.owner_instance_id is None account_changed = existing.account_id != account_id owner_changed = existing.owner_instance_id != instance_id - lease_expired = existing.lease_expires_at is None or to_utc_naive(existing.lease_expires_at) <= now - expected_owner_identity_changed = bool( - expected_takeover_owner_instance_id is not None - and existing.owner_instance_id is not None - and ( - existing.owner_instance_id != expected_takeover_owner_instance_id - or ( - expected_takeover_owner_process_epoch is not None - and existing.owner_process_epoch != expected_takeover_owner_process_epoch - ) - ) - ) - if expected_owner_identity_changed and not lease_expired and not state_allows_takeover: - return _to_snapshot_required(existing) if owner_changed: - if not allow_takeover and not lease_expired and not state_allows_takeover: + lease_expired = existing.lease_expires_at is None or to_utc_naive(existing.lease_expires_at) <= now + live_owned_draining = ( + existing.state == HttpBridgeSessionState.DRAINING and not lease_expired and not owner_absent + ) + takeover_permitted = allow_takeover and not contended + if live_owned_draining or ( + not takeover_permitted and not lease_expired and not owner_absent and not state_closed + ): return _to_snapshot_required(existing) - next_epoch = existing.owner_epoch + 1 - elif account_changed or force_owner_epoch_advance: - next_epoch = existing.owner_epoch + 1 + # Every claim advances the owner epoch, including a same-owner + # reclaim: claims come only from a successor in-memory session (a + # reused session renews instead of claiming), so a live same-owner + # row means the predecessor local session is retiring concurrently + # and its outstanding fenced release/renewals must no-op rather + # than race this claim into a closed, ownerless row (issue #1695). + next_epoch = existing.owner_epoch + 1 + + # Write through an explicit UPDATE that sets every ownership field + # unconditionally. Mutating ORM attributes lets SQLAlchemy omit + # fields whose values match this transaction's (possibly stale) + # read, so a release committing between the SELECT and this write + # survived the claim and the refresh below returned a closed, + # ownerless row to a claimant that believed it had succeeded + # (issue #1695; SQLite's with_for_update is a no-op). + values: dict[str, object] = { + "owner_instance_id": instance_id, + "owner_process_epoch": owner_process_epoch, + "owner_epoch": next_epoch, + "lease_expires_at": lease_expires_at, + "state": HttpBridgeSessionState.ACTIVE, + "account_id": account_id, + "model": model, + "service_tier": service_tier, + "last_seen_at": now, + "closed_at": None, + } + if account_changed: + values["latest_turn_state"] = latest_turn_state + values["latest_response_id"] = latest_response_id + values["latest_input_item_count"] = None + values["latest_input_full_fingerprint"] = None + values["latest_pending_tool_calls_json"] = None else: - next_epoch = existing.owner_epoch - + if latest_turn_state is not None: + values["latest_turn_state"] = latest_turn_state + if latest_response_id is not None: + values["latest_response_id"] = latest_response_id + values["latest_input_item_count"] = None + values["latest_input_full_fingerprint"] = None + values["latest_pending_tool_calls_json"] = None async with sqlite_writer_section(): - existing.owner_instance_id = instance_id - existing.owner_process_epoch = owner_process_epoch - existing.owner_epoch = next_epoch - existing.lease_expires_at = lease_expires_at - existing.state = HttpBridgeSessionState.ACTIVE + # Compare-and-set on the epoch read above: SQLite's + # with_for_update is a no-op, so two successor claims can both + # read epoch N; without the guard both would write N+1 and both + # believe they own the row with colliding fences. The loser's + # update matches zero rows and retries against fresh state. + result = await self._session.execute( + update(HttpBridgeSessionRecord) + .where( + HttpBridgeSessionRecord.id == existing.id, + HttpBridgeSessionRecord.owner_epoch == existing.owner_epoch, + ) + .values(**values) + ) + if not bool(getattr(result, "rowcount", 0)): + await self._session.rollback() + if attempt < _CLAIM_CAS_ATTEMPTS - 1: + contended = True + continue + raise RuntimeError("Failed to claim durable bridge session after retry") if account_changed: await self._clear_aliases_for_session(existing.id) - existing.account_id = account_id - existing.model = model - existing.service_tier = service_tier - if account_changed: - existing.latest_turn_state = latest_turn_state - existing.latest_response_id = latest_response_id - existing.latest_input_item_count = None - existing.latest_input_full_fingerprint = None - existing.latest_pending_tool_calls_json = None - existing.latest_response_transition_manifest_json = None - elif owner_changed: - if latest_turn_state is not None: - existing.latest_turn_state = latest_turn_state - if latest_response_id is not None: - existing.latest_response_id = latest_response_id - existing.latest_input_item_count = None - existing.latest_input_full_fingerprint = None - existing.latest_pending_tool_calls_json = None - existing.latest_response_transition_manifest_json = None - else: - if latest_turn_state is not None: - existing.latest_turn_state = latest_turn_state - if latest_response_id is not None: - existing.latest_response_id = latest_response_id - existing.latest_input_item_count = None - existing.latest_input_full_fingerprint = None - existing.latest_pending_tool_calls_json = None - existing.latest_response_transition_manifest_json = None - existing.last_seen_at = now - existing.closed_at = None await self._session.commit() - await self._session.refresh(existing) - return _to_snapshot_required(existing) + # Build the snapshot from the values THIS CAS wrote rather than a + # post-commit refresh: another successor can commit its own CAS + # between this commit and a refresh, and returning that later epoch + # would hand this claimant a fence that collides with the winner's. + written_turn_state = values.get("latest_turn_state", existing.latest_turn_state) + written_response_id = values.get("latest_response_id", existing.latest_response_id) + written_pending_json = values.get("latest_pending_tool_calls_json", existing.latest_pending_tool_calls_json) + return DurableBridgeSessionSnapshot( + id=existing.id, + session_key_kind=existing.session_key_kind, + session_key_value=existing.session_key_value, + session_key_hash=existing.session_key_hash, + api_key_scope=existing.api_key_scope, + owner_instance_id=instance_id, + owner_process_epoch=owner_process_epoch, + owner_epoch=next_epoch, + lease_expires_at=lease_expires_at, + state=HttpBridgeSessionState.ACTIVE, + account_id=account_id, + model=model, + service_tier=service_tier, + latest_turn_state=cast("str | None", written_turn_state), + latest_response_id=cast("str | None", written_response_id), + latest_input_item_count=cast( + "int | None", values.get("latest_input_item_count", existing.latest_input_item_count) + ), + latest_input_full_fingerprint=cast( + "str | None", + values.get("latest_input_full_fingerprint", existing.latest_input_full_fingerprint), + ), + latest_pending_tool_calls=_decode_pending_tool_calls( + cast("str | None", written_response_id), + cast("str | None", written_pending_json), + ), + last_seen_at=now, + closed_at=None, + ) raise RuntimeError("Failed to claim durable bridge session after retry") async def renew_session( @@ -711,7 +813,6 @@ async def renew_session( latest_input_item_count: int | None = None, latest_input_full_fingerprint: str | None = None, latest_pending_tool_calls: Mapping[str, str] | None = None, - latest_response_transition_manifest: ResponseTransitionManifest | None = None, state: HttpBridgeSessionState | None = None, ) -> DurableBridgeSessionSnapshot | None: """Renew the lease with a single fenced UPDATE. @@ -732,17 +833,9 @@ async def renew_session( latest_response_id, latest_pending_tool_calls, ) - values["latest_response_transition_manifest_json"] = encode_response_transition_manifest( - latest_response_transition_manifest - ) if latest_input_item_count is None or latest_input_full_fingerprint is None: values["latest_input_item_count"] = None values["latest_input_full_fingerprint"] = None - values["recovery_required_anchor_hash"] = None - values["recovery_required_account_id"] = None - values["recovery_required_attempt_fingerprint"] = None - values["recovery_required_attempt_request_id"] = None - values["recovery_required_at"] = None if latest_input_item_count is not None and latest_input_full_fingerprint is not None: values["latest_input_item_count"] = latest_input_item_count values["latest_input_full_fingerprint"] = latest_input_full_fingerprint @@ -775,7 +868,6 @@ async def rebind_session_account( latest_input_item_count=None, latest_input_full_fingerprint=None, latest_pending_tool_calls_json=None, - latest_response_transition_manifest_json=None, ) result = await self._session.execute( update(HttpBridgeSessionRecord) @@ -840,12 +932,6 @@ async def clear_latest_response_anchor( "latest_input_item_count": None, "latest_input_full_fingerprint": None, "latest_pending_tool_calls_json": None, - "latest_response_transition_manifest_json": None, - "recovery_required_anchor_hash": None, - "recovery_required_account_id": None, - "recovery_required_attempt_fingerprint": None, - "recovery_required_attempt_request_id": None, - "recovery_required_at": None, } return await self._execute_fenced_session_update( session_id=session_id, @@ -854,218 +940,1004 @@ async def clear_latest_response_anchor( values=values, ) - async def mark_recovery_required( + async def record_recovery_attempt( self, *, session_id: str, instance_id: str, owner_epoch: int, - account_id: str, - rejected_response_id: str, - ) -> bool: - """Fence and persist an owner/anchor-bound recovery requirement. - - The marker deliberately stores only the rejected anchor digest. The - durable row already owns the plaintext anchor needed for exact replay - verification; duplicating request or conversation content here would - create a second continuity authority. - """ - + request_fingerprint: str, + request_id: str, + account_id: str | None, + model: str | None, + replay_safe: bool, + ) -> DurableBridgeRecoveryAttemptSnapshot | None: + """Record a safe request before dispatch so an ambiguous outcome is recoverable.""" async with sqlite_writer_section(): - result = await self._session.execute( - update(HttpBridgeSessionRecord) + # Lock the owner row through the journal write so a takeover + # cannot advance the epoch after this check but before dispatch. + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) .where( HttpBridgeSessionRecord.id == session_id, HttpBridgeSessionRecord.owner_instance_id == instance_id, HttpBridgeSessionRecord.owner_epoch == owner_epoch, - HttpBridgeSessionRecord.account_id == account_id, - HttpBridgeSessionRecord.latest_response_id == rejected_response_id, - ) - .values( - recovery_required_anchor_hash=durable_bridge_hash(rejected_response_id), - recovery_required_account_id=account_id, - recovery_required_at=utcnow(), ) + .with_for_update() ) - await self._session.commit() - return bool(getattr(result, "rowcount", 0)) + if owner_exists is None: + await self._session.rollback() + return None + attempt = await self._session.scalar( + select(HttpBridgeRecoveryAttemptRecord) + .where(HttpBridgeRecoveryAttemptRecord.session_id == session_id) + .where(HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint) + .with_for_update() + ) + if attempt is None: + attempt = HttpBridgeRecoveryAttemptRecord( + session_id=session_id, + request_fingerprint=request_fingerprint, + request_id=request_id, + account_id=account_id, + model=model, + replay_safe=replay_safe, + state=HttpBridgeRecoveryAttemptState.UNKNOWN, + ) + self._session.add(attempt) + elif attempt.state == HttpBridgeRecoveryAttemptState.REPLAYED: + snapshot = _to_recovery_attempt_snapshot(attempt) + await self._session.rollback() + return snapshot + elif attempt.request_id != request_id: + # A different request already owns the UNKNOWN checkpoint. + # Do not overwrite it while that request may still be between + # admission and dispatch; the caller must fail closed rather + # than sharing a journal generation. + snapshot = _to_recovery_attempt_snapshot(attempt) + await self._session.rollback() + return snapshot + else: + attempt.request_id = request_id + attempt.account_id = account_id + attempt.model = model + attempt.replay_safe = replay_safe + attempt.state = HttpBridgeRecoveryAttemptState.UNKNOWN + attempt.response_id = None + try: + await self._session.commit() + except IntegrityError: + # A concurrent owner may have inserted the same fingerprint + # after our initial SELECT (the absent-row case cannot be + # locked by SQLite). Re-read the winner and use its durable + # state instead of surfacing a transient uniqueness failure. + await self._session.rollback() + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + if owner_exists is None: + await self._session.rollback() + return None + attempt = await self._session.scalar( + select(HttpBridgeRecoveryAttemptRecord) + .where(HttpBridgeRecoveryAttemptRecord.session_id == session_id) + .where(HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint) + ) + if attempt is None: + raise + if attempt.state == HttpBridgeRecoveryAttemptState.REPLAYED: + snapshot = _to_recovery_attempt_snapshot(attempt) + await self._session.rollback() + return snapshot + if attempt.request_id != request_id: + snapshot = _to_recovery_attempt_snapshot(attempt) + await self._session.rollback() + return snapshot + attempt.request_id = request_id + attempt.account_id = account_id + attempt.model = model + attempt.replay_safe = replay_safe + attempt.state = HttpBridgeRecoveryAttemptState.UNKNOWN + attempt.response_id = None + await self._session.commit() + await self._session.refresh(attempt) + return _to_recovery_attempt_snapshot(attempt) + + async def lookup_recovery_attempt( + self, + *, + session_id: str, + request_fingerprint: str, + ) -> DurableBridgeRecoveryAttemptSnapshot | None: + attempt = await self._session.scalar( + select(HttpBridgeRecoveryAttemptRecord) + .where(HttpBridgeRecoveryAttemptRecord.session_id == session_id) + .where(HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint) + .where(HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.UNKNOWN) + .where(HttpBridgeRecoveryAttemptRecord.replay_safe.is_(True)) + ) + return _to_recovery_attempt_snapshot(attempt) if attempt is not None else None - async def claim_recovery_required_attempt( + async def mark_recovery_attempt_replayed( self, *, session_id: str, instance_id: str, owner_epoch: int, - account_id: str, - rejected_response_id: str, - attempt_fingerprint: str, - request_id: str, + request_fingerprint: str, + response_id: str | None = None, ) -> bool: - """Bind one exact wire payload to the active marker generation.""" - async with sqlite_writer_section(): - marker = await self._session.scalar( - select(HttpBridgeSessionRecord) + # Keep the owner fence and journal transition in one transaction. + # PostgreSQL's row lock prevents a concurrent takeover from + # advancing the epoch between the check and the state update; + # sqlite_writer_section provides the equivalent writer + # serialization for SQLite. + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) .where( HttpBridgeSessionRecord.id == session_id, HttpBridgeSessionRecord.owner_instance_id == instance_id, HttpBridgeSessionRecord.owner_epoch == owner_epoch, - HttpBridgeSessionRecord.account_id == account_id, - HttpBridgeSessionRecord.latest_response_id == rejected_response_id, - HttpBridgeSessionRecord.recovery_required_anchor_hash == durable_bridge_hash(rejected_response_id), - HttpBridgeSessionRecord.recovery_required_account_id == account_id, ) .with_for_update() ) - if marker is None: - await self._session.rollback() - return False - existing = marker.recovery_required_attempt_fingerprint - existing_request_id = marker.recovery_required_attempt_request_id - if existing is not None and (existing != attempt_fingerprint or existing_request_id != request_id): + if owner_exists is None: await self._session.rollback() return False - if existing is None: - marker.recovery_required_attempt_fingerprint = attempt_fingerprint - marker.recovery_required_attempt_request_id = request_id - await self._session.commit() - else: - await self._session.rollback() - return True + values: dict[str, object] = {"state": HttpBridgeRecoveryAttemptState.REPLAYED} + if response_id is not None: + values["response_id"] = response_id + # A claim authorizes one replay and must only transition UNKNOWN + # rows. Settlement (which supplies response_id) remains idempotent + # for a REPLAYED row after the replay completes. + claimable_states = ( + (HttpBridgeRecoveryAttemptState.UNKNOWN,) + if response_id is None + else (HttpBridgeRecoveryAttemptState.UNKNOWN, HttpBridgeRecoveryAttemptState.REPLAYED) + ) + result = await self._session.execute( + update(HttpBridgeRecoveryAttemptRecord) + .where( + HttpBridgeRecoveryAttemptRecord.session_id == session_id, + HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint, + HttpBridgeRecoveryAttemptRecord.state.in_(claimable_states), + ) + .values(**values) + ) + await self._session.commit() + return bool(getattr(result, "rowcount", 0)) - async def claim_recovery_required_attempt_with_journal( + async def rollback_recovery_attempt_replayed( self, *, session_id: str, instance_id: str, owner_epoch: int, - account_id: str, - rejected_response_id: str, - attempt_fingerprint: str, - claim_request_id: str, - journal_request_id: str, - model: str | None, - ) -> DurableBridgeRecoveryAttemptSnapshot | None: - """Atomically bind a marker generation and create its send journal.""" - + request_fingerprint: str, + ) -> bool: + """Return a pre-dispatch replay claim to UNKNOWN under the owner fence.""" async with sqlite_writer_section(): - marker = await self._session.scalar( - select(HttpBridgeSessionRecord) + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) .where( HttpBridgeSessionRecord.id == session_id, HttpBridgeSessionRecord.owner_instance_id == instance_id, HttpBridgeSessionRecord.owner_epoch == owner_epoch, - HttpBridgeSessionRecord.account_id == account_id, - HttpBridgeSessionRecord.latest_response_id == rejected_response_id, - HttpBridgeSessionRecord.recovery_required_anchor_hash == durable_bridge_hash(rejected_response_id), - HttpBridgeSessionRecord.recovery_required_account_id == account_id, ) .with_for_update() ) - if marker is None or marker.recovery_required_attempt_fingerprint is not None: + if owner_exists is None: await self._session.rollback() - return None - existing_journal = await self._session.scalar( - select(HttpBridgeRecoveryAttemptRecord.id).where( + return False + result = await self._session.execute( + update(HttpBridgeRecoveryAttemptRecord) + .where( HttpBridgeRecoveryAttemptRecord.session_id == session_id, - HttpBridgeRecoveryAttemptRecord.request_fingerprint == attempt_fingerprint, + HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint, + HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.REPLAYED, + HttpBridgeRecoveryAttemptRecord.response_id.is_(None), ) + .values(state=HttpBridgeRecoveryAttemptState.UNKNOWN) ) - if existing_journal is not None: - await self._session.rollback() - return None + await self._session.commit() + return bool(getattr(result, "rowcount", 0)) - marker.recovery_required_attempt_fingerprint = attempt_fingerprint - marker.recovery_required_attempt_request_id = claim_request_id - attempt = HttpBridgeRecoveryAttemptRecord( + async def rollback_recovery_attempt_before_dispatch( + self, + *, + session_id: str, + instance_id: str, + owner_epoch: int, + request_fingerprint: str, + ) -> bool: + """Delete an UNKNOWN checkpoint proven not to have reached upstream.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + if owner_exists is None: + await self._session.rollback() + return False + result = await self._session.execute( + delete(HttpBridgeRecoveryAttemptRecord).where( + HttpBridgeRecoveryAttemptRecord.session_id == session_id, + HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint, + HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.UNKNOWN, + ) + ) + await self._session.commit() + return bool(getattr(result, "rowcount", 0)) + + async def record_operation( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + request_fingerprint: str, + account_id: str | None, + model: str | None, + parent_response_id: str | None, + api_key_scope: str | None = None, + request_text: str | None = None, + recovery_attempt_session_id: str | None = None, + recovery_attempt_owner_epoch: int | None = None, + recovery_attempt_fingerprint: str | None = None, + recovery_attempt_consumed: bool = False, + ) -> DurableBridgeOperationSnapshot | None: + """Create a fenced operation identity, or return the existing one.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + if owner_exists is None: + await self._session.rollback() + return None + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where(HttpBridgeOperationRecord.operation_id == operation_id) + .with_for_update() + ) + if operation is None: + fingerprint_statement = select(HttpBridgeOperationRecord).where( + HttpBridgeOperationRecord.request_fingerprint == request_fingerprint + ) + if api_key_scope is not None: + fingerprint_statement = fingerprint_statement.join( + HttpBridgeSessionRecord, + HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id, + ).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope) + operation = await self._session.scalar(fingerprint_statement.with_for_update()) + if operation is not None: + if recovery_attempt_consumed: + # A REPLAYED recovery checkpoint is immutable. Return the + # existing row for safe transcript replay or fail-closed + # handling; never rebind a failed row and clear its spool. + snapshot = _to_operation_snapshot(operation) + await self._session.rollback() + return snapshot + rebound = False + handoff_allowed = True + if operation.session_id != session_id and operation.state not in {"completed", "incomplete"}: + # A global fingerprint can outlive the durable session + # that first recorded it. Do not steal an operation from + # a still-live owner: its stream may still be dispatching + # the turn, and rebinding would fence its writes while a + # second owner sends a duplicate upstream request. + previous_session = await self._session.scalar( + select(HttpBridgeSessionRecord) + .where(HttpBridgeSessionRecord.id == operation.session_id) + .with_for_update() + ) + now = utcnow() + recovery_handoff_allowed = False + if ( + previous_session is not None + and recovery_attempt_session_id == operation.session_id + and recovery_attempt_owner_epoch is not None + and recovery_attempt_fingerprint is not None + and previous_session.owner_instance_id == instance_id + and previous_session.owner_epoch == recovery_attempt_owner_epoch + ): + # A fresh account-neutral replay has already fenced + # the one-shot journal on the origin session. That + # journal owner must remain fenced until settlement, + # but the operation itself must move to the + # replacement owner so its transcript and outcome + # writes are accepted there. This is the only + # cross-session handoff allowed while the origin + # lease is still active. + recovery_attempt = await self._session.scalar( + select(HttpBridgeRecoveryAttemptRecord) + .where( + HttpBridgeRecoveryAttemptRecord.session_id == recovery_attempt_session_id, + HttpBridgeRecoveryAttemptRecord.request_fingerprint == recovery_attempt_fingerprint, + HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.REPLAYED, + HttpBridgeRecoveryAttemptRecord.response_id.is_(None), + ) + .with_for_update() + ) + recovery_handoff_allowed = recovery_attempt is not None + handoff_allowed = ( + recovery_handoff_allowed + or previous_session is None + or not ( + previous_session.owner_instance_id is not None + and previous_session.lease_expires_at is not None + # PostgreSQL returns timestamptz values with an + # attached UTC offset, while ``utcnow`` is a + # naive UTC value used by the durable layer. + # Normalize before comparing so cross-session + # recovery remains database-backend agnostic. + and to_utc_naive(previous_session.lease_expires_at) > now + ) + ) + if handoff_allowed: + # Transfer only nonterminal operations to the currently + # fenced owner before the caller resets the attempt + # spool; completed transcripts remain attached to + # their original session for replay. + operation.session_id = session_id + operation.account_id = account_id + operation.model = model + operation.parent_response_id = parent_response_id + if request_text is not None and operation.request_text is None: + operation.request_text = request_text + operation.updated_at = now + if operation.state == "failed" and handoff_allowed: + # An explicit upstream failure is retryable. Rebind the + # durable operation to the current owner while preserving + # its global identity; concurrent reconnects will see the + # submitted state and remain fenced. + operation.session_id = session_id + operation.account_id = account_id + operation.model = model + operation.parent_response_id = parent_response_id + if request_text is not None and operation.request_text is None: + operation.request_text = request_text + operation.state = "submitted" + operation.response_id = None + # A failed attempt is a new replay attempt. Remove the + # previous attempt's SSE spool atomically so a later + # successful retry cannot replay a stale response.failed + # event before its fresh response.created sequence. + await self._session.execute( + delete(HttpBridgeOperationEvent).where( + HttpBridgeOperationEvent.operation_id == operation.operation_id + ) + ) + operation.event_bytes = 0 + operation.event_spool_complete = False + operation.updated_at = utcnow() + rebound = True + if request_text is not None and operation.request_text is None: + operation.request_text = request_text + operation.updated_at = utcnow() + snapshot = _to_operation_snapshot(operation, created=rebound) + await self._session.commit() + return snapshot + operation = HttpBridgeOperationRecord( + operation_id=operation_id, session_id=session_id, - request_fingerprint=attempt_fingerprint, - request_id=journal_request_id, + request_fingerprint=request_fingerprint, account_id=account_id, model=model, - replay_safe=True, - state=HttpBridgeRecoveryAttemptState.UNKNOWN, - ) - self._session.add(attempt) + parent_response_id=parent_response_id, + request_text=request_text, + state="submitted", + # A transcript is replayable only after the event batcher has + # drained and finalized it. Set this explicitly rather than + # relying on a backend-specific schema default (notably the + # pre-existing SQLite default on migrated databases). + event_spool_complete=False, + ) + self._session.add(operation) try: await self._session.commit() except IntegrityError: await self._session.rollback() - return None - return _to_recovery_attempt_snapshot(attempt) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord).where(HttpBridgeOperationRecord.operation_id == operation_id) + ) + if operation is None: + # A reconnect may derive a different session-scoped + # operation ID for the same anchored request. The global + # fingerprint fence makes that race resolve to the + # already-recorded operation instead of dispatching a + # duplicate. + fingerprint_statement = select(HttpBridgeOperationRecord).where( + HttpBridgeOperationRecord.request_fingerprint == request_fingerprint + ) + if api_key_scope is not None: + fingerprint_statement = fingerprint_statement.join( + HttpBridgeSessionRecord, + HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id, + ).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope) + operation = await self._session.scalar(fingerprint_statement) + if operation is None: + raise + return _to_operation_snapshot(operation) + await self._session.refresh(operation) + return _to_operation_snapshot(operation, created=True) - async def rollback_recovery_required_attempt_before_dispatch( + async def get_operation(self, *, operation_id: str) -> DurableBridgeOperationSnapshot | None: + operation = await self._session.scalar( + select(HttpBridgeOperationRecord).where(HttpBridgeOperationRecord.operation_id == operation_id) + ) + return _to_operation_snapshot(operation) if operation is not None else None + + async def reset_operation_event_spool( self, *, + operation_id: str, session_id: str, - api_key_scope: str, instance_id: str, owner_epoch: int, - account_id: str, - attempt_fingerprint: str, - request_id: str, - journal_request_id: str, ) -> bool: - """Release an exact marker claim that provably never reached dispatch.""" + """Start a fresh transcript for a server-owned ambiguous retry.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.state.not_in(("completed", "incomplete")), + ) + .with_for_update() + ) + if owner_exists is None or operation is None: + await self._session.rollback() + return False + await self._session.execute( + delete(HttpBridgeOperationEvent).where(HttpBridgeOperationEvent.operation_id == operation_id) + ) + operation.event_bytes = 0 + operation.event_spool_complete = False + operation.updated_at = utcnow() + await self._session.commit() + return True + + async def claim_unknown_operation_for_recovery( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + max_recovery_dispatches: int | None = None, + ) -> bool: + """Atomically claim an UNKNOWN operation for one recovery attempt. + Recovery admission can be reached by multiple reconnects at once. A + reset followed by a later state transition leaves a window where each + reconnect can observe UNKNOWN and submit the same operation. Keep the + owner fence, state transition, and transcript reset in one serialized + write so exactly one caller can move UNKNOWN back to SUBMITTED. + """ async with sqlite_writer_section(): - marker = await self._session.scalar( - select(HttpBridgeSessionRecord) + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.state == "unknown", + ) + .with_for_update() + ) + if owner_exists is None or operation is None: + await self._session.rollback() + return False + if max_recovery_dispatches is not None and operation.recovery_dispatch_count >= max_recovery_dispatches: + await self._session.rollback() + return False + await self._session.execute( + delete(HttpBridgeOperationEvent).where(HttpBridgeOperationEvent.operation_id == operation_id) + ) + operation.state = "submitted" + operation.response_id = None + operation.recovery_dispatch_count += 1 + operation.event_bytes = 0 + operation.event_spool_complete = False + operation.updated_at = utcnow() + await self._session.commit() + return True + + async def mark_operation_unknown( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + restore_recovery_dispatch_claim: bool = False, + ) -> bool: + """Fence an ambiguously dispatched SUBMITTED operation as UNKNOWN. + + The operation event reader can race the send-failure cleanup. Lock the + row before changing it and leave an already acknowledged or terminal + operation untouched; those states carry stronger evidence than the + transport exception and must never be downgraded to UNKNOWN. + """ + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + ) + .with_for_update() + ) + if owner_exists is None or operation is None: + await self._session.rollback() + return False + if operation.state == "submitted": + operation.state = "unknown" + if restore_recovery_dispatch_claim and operation.recovery_dispatch_count > 0: + operation.recovery_dispatch_count -= 1 + operation.updated_at = utcnow() + elif ( + restore_recovery_dispatch_claim + and operation.state == "unknown" + and operation.recovery_dispatch_count > 0 + ): + # A concurrent cleanup may have fenced the row first. The + # caller still owns a proven pre-dispatch recovery claim, so + # refund exactly that claim while retaining UNKNOWN. + operation.recovery_dispatch_count -= 1 + operation.updated_at = utcnow() + await self._session.commit() + return True + + async def rollback_operation_before_dispatch( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + ) -> bool: + """Remove a newly-created operation that never reached upstream.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) .where( HttpBridgeSessionRecord.id == session_id, - HttpBridgeSessionRecord.api_key_scope == api_key_scope, HttpBridgeSessionRecord.owner_instance_id == instance_id, HttpBridgeSessionRecord.owner_epoch == owner_epoch, - HttpBridgeSessionRecord.account_id == account_id, - HttpBridgeSessionRecord.recovery_required_account_id == account_id, - HttpBridgeSessionRecord.recovery_required_attempt_fingerprint == attempt_fingerprint, - HttpBridgeSessionRecord.recovery_required_attempt_request_id == request_id, ) .with_for_update() ) - if ( - marker is None - or marker.latest_response_id is None - or marker.recovery_required_anchor_hash != durable_bridge_hash(marker.latest_response_id) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.state == "submitted", + HttpBridgeOperationRecord.response_id.is_(None), + HttpBridgeOperationRecord.event_bytes == 0, + ) + .with_for_update() + ) + if owner_exists is None or operation is None: + await self._session.rollback() + return False + has_events = await self._session.scalar( + select(HttpBridgeOperationEvent.event_id) + .where(HttpBridgeOperationEvent.operation_id == operation_id) + .limit(1) + ) + if has_events is not None: + await self._session.rollback() + return False + await self._session.delete(operation) + await self._session.commit() + return True + + async def get_operation_by_fingerprint( + self, + *, + request_fingerprint: str, + api_key_scope: str | None = None, + ) -> DurableBridgeOperationSnapshot | None: + statement = select(HttpBridgeOperationRecord).where( + HttpBridgeOperationRecord.request_fingerprint == request_fingerprint + ) + if api_key_scope is not None: + statement = statement.join( + HttpBridgeSessionRecord, + HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id, + ).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope) + operation = await self._session.scalar(statement) + return _to_operation_snapshot(operation) if operation is not None else None + + async def get_operation_events(self, *, operation_id: str) -> list[str]: + result = await self._session.execute( + select(HttpBridgeOperationEvent.event_text) + .where(HttpBridgeOperationEvent.operation_id == operation_id) + .order_by(HttpBridgeOperationEvent.sequence_number.asc()) + ) + return [str(value) for value in result.scalars().all()] + + async def get_operation_by_response_id(self, *, response_id: str) -> DurableBridgeOperationSnapshot | None: + operation = await self._session.scalar( + select(HttpBridgeOperationRecord).where( + HttpBridgeOperationRecord.response_id == response_id, + HttpBridgeOperationRecord.state.in_(("completed", "incomplete")), + ) + ) + return _to_operation_snapshot(operation) if operation is not None else None + + async def get_replayable_transcript( + self, + *, + response_id: str, + max_turns: int = 128, + max_bytes: int = 8 * 1024 * 1024, + ) -> list[DurableBridgeTranscriptTurn] | None: + """Return a complete parent-response chain, newest turn last. + + Missing request bodies, truncated event spools, or a broken parent + chain make the transcript ineligible for reconstruction. + """ + turns: list[DurableBridgeTranscriptTurn] = [] + visited: set[str] = set() + total_bytes = 0 + current_response_id: str | None = response_id + while current_response_id is not None: + if current_response_id in visited or len(turns) >= max_turns: + return None + visited.add(current_response_id) + operation = await self.get_operation_by_response_id(response_id=current_response_id) + if operation is None or operation.request_text is None or not operation.event_spool_complete: + return None + events = await self.get_operation_events(operation_id=operation.operation_id) + if not events or not any( + "response.completed" in event or "response.incomplete" in event for event in events ): + return None + turn_bytes = len(operation.request_text.encode("utf-8")) + sum( + len(event.encode("utf-8")) for event in events + ) + total_bytes += turn_bytes + if total_bytes > max_bytes: + return None + turns.append(DurableBridgeTranscriptTurn(operation=operation, events=tuple(events))) + current_response_id = operation.parent_response_id + turns.reverse() + return turns + + async def purge_operation_spool(self, *, cutoff: datetime, batch_size: int = 500) -> int: + """Delete eligible transcript material past retention. + + Nonterminal rows are purgeable only after their owning session is + ownerless or its lease has expired. Recheck that predicate in the + delete transaction so an in-flight operation cannot lose its + duplicate-suppression ledger between selection and deletion. + """ + terminal_states = ("completed", "incomplete", "failed") + # UNKNOWN is an ambiguous, still-live operation while its owner lease + # is active. Treat it like the other nonterminal states so retention + # cannot delete the duplicate-suppression fence during a long-running + # server-indefinite recovery attempt. + nonterminal_states = ("submitted", "acknowledged", "unknown") + stale_owner = or_( + HttpBridgeSessionRecord.owner_instance_id.is_(None), + HttpBridgeSessionRecord.lease_expires_at.is_(None), + HttpBridgeSessionRecord.lease_expires_at < utcnow(), + ) + stale_nonterminal = and_( + HttpBridgeOperationRecord.state.in_(nonterminal_states), + exists( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id, + stale_owner, + ) + .correlate(HttpBridgeOperationRecord) + ), + ) + purgeable = or_(HttpBridgeOperationRecord.state.in_(terminal_states), stale_nonterminal) + async with sqlite_writer_section(): + selected = await self._session.execute( + select(HttpBridgeOperationRecord) + .join( + HttpBridgeSessionRecord, + HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id, + ) + .where(HttpBridgeOperationRecord.updated_at < cutoff, purgeable) + .order_by(HttpBridgeOperationRecord.updated_at.asc()) + .limit(batch_size) + .with_for_update() + ) + # The joined FOR UPDATE locks both the operation and owning + # session on PostgreSQL, serializing retention deletion with + # claim_session() on the same continuity row. + operation_ids = [str(operation.operation_id) for operation in selected.scalars().all()] + if not operation_ids: + await self._session.commit() + return 0 + deleted = await self._session.execute( + delete(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id.in_(operation_ids), + HttpBridgeOperationRecord.updated_at < cutoff, + purgeable, + ) + .returning(HttpBridgeOperationRecord.operation_id) + ) + deleted_ids = [str(value) for value in deleted.scalars().all()] + if deleted_ids: + await self._session.execute( + delete(HttpBridgeOperationEvent).where(HttpBridgeOperationEvent.operation_id.in_(deleted_ids)) + ) + await self._session.commit() + return len(deleted_ids) + + async def append_operation_event( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + event_text: str, + max_bytes: int, + ) -> bool: + """Append one replayable SSE block under the durable owner fence.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + ) + .with_for_update() + ) + if owner_exists is None or operation is None: await self._session.rollback() return False - journal = await self._session.scalar( - select(HttpBridgeRecoveryAttemptRecord) + event_size = len(event_text.encode("utf-8")) + if event_size > max_bytes or int(operation.event_bytes or 0) + event_size > max_bytes: + operation.event_spool_complete = False + await self._session.commit() + return False + next_sequence = await self._session.scalar( + select(func.coalesce(func.max(HttpBridgeOperationEvent.sequence_number), 0) + 1).where( + HttpBridgeOperationEvent.operation_id == operation_id, + ) + ) + sequence = int(next_sequence or 1) + self._session.add( + HttpBridgeOperationEvent( + operation_id=operation_id, + sequence_number=sequence, + # Include occurrence position so identical downstream + # blocks remain distinct in replay transcripts. + event_fingerprint=durable_bridge_hash(f"{sequence}:{event_text}"), + event_text=event_text, + ) + ) + operation.event_bytes = int(operation.event_bytes or 0) + event_size + await self._session.commit() + return True + + async def append_terminal_operation_event( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + event_text: str, + max_bytes: int, + state: str, + expected_recovery_dispatch_count: int = 0, + response_id: str | None = None, + ) -> bool: + """Append a terminal event and expose its operation state atomically.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.recovery_dispatch_count == expected_recovery_dispatch_count, + ) + .with_for_update() + ) + if owner_exists is None or operation is None: + await self._session.rollback() + return False + event_size = len(event_text.encode("utf-8")) + persisted = event_size <= max_bytes and int(operation.event_bytes or 0) + event_size <= max_bytes + if persisted: + next_sequence = await self._session.scalar( + select(func.coalesce(func.max(HttpBridgeOperationEvent.sequence_number), 0) + 1).where( + HttpBridgeOperationEvent.operation_id == operation_id, + ) + ) + sequence = int(next_sequence or 1) + self._session.add( + HttpBridgeOperationEvent( + operation_id=operation_id, + sequence_number=sequence, + event_fingerprint=durable_bridge_hash(f"{sequence}:{event_text}"), + event_text=event_text, + ) + ) + operation.event_bytes = int(operation.event_bytes or 0) + event_size + else: + operation.event_spool_complete = False + # The terminal outcome is still authoritative even when the + # transcript block cannot fit in the bounded spool. Expose + # the failed state so an identical retry does not remain + # fenced as an in-flight operation until retention expires. + operation.state = state + if response_id is not None: + operation.response_id = response_id + operation.updated_at = utcnow() + await self._session.commit() + return False + operation.state = state + if response_id is not None: + operation.response_id = response_id + operation.event_spool_complete = True + operation.updated_at = utcnow() + await self._session.commit() + return persisted + + async def append_operation_events( + self, + *, + events: Sequence[DurableBridgeOperationEventInput], + max_bytes: int, + ) -> bool: + """Append a batch of SSE blocks with one fenced transaction.""" + if not events: + return True + first = events[0] + if any( + event.operation_id != first.operation_id + or event.session_id != first.session_id + or event.instance_id != first.instance_id + or event.owner_epoch != first.owner_epoch + for event in events + ): + return False + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == first.session_id, + HttpBridgeSessionRecord.owner_instance_id == first.instance_id, + HttpBridgeSessionRecord.owner_epoch == first.owner_epoch, + ) + .with_for_update() + ) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) .where( - HttpBridgeRecoveryAttemptRecord.session_id == session_id, - HttpBridgeRecoveryAttemptRecord.request_fingerprint == attempt_fingerprint, + HttpBridgeOperationRecord.operation_id == first.operation_id, + HttpBridgeOperationRecord.session_id == first.session_id, ) .with_for_update() ) - if journal is not None and ( - journal.request_id != journal_request_id or journal.state != HttpBridgeRecoveryAttemptState.UNKNOWN - ): + if owner_exists is None or operation is None: await self._session.rollback() return False - if journal is not None: - await self._session.delete(journal) - marker.recovery_required_attempt_fingerprint = None - marker.recovery_required_attempt_request_id = None + next_sequence = await self._session.scalar( + select(func.coalesce(func.max(HttpBridgeOperationEvent.sequence_number), 0) + 1).where( + HttpBridgeOperationEvent.operation_id == first.operation_id, + ) + ) + sequence = int(next_sequence or 1) + pending: list[tuple[str, int, str, int]] = [] + total_bytes = int(operation.event_bytes or 0) + for event in events: + event_size = len(event.event_text.encode("utf-8")) + if total_bytes + event_size > max_bytes: + operation.event_spool_complete = False + await self._session.commit() + return False + total_bytes += event_size + pending.append( + ( + event.event_text, + sequence, + durable_bridge_hash(f"{sequence}:{event.event_text}"), + event_size, + ) + ) + sequence += 1 + if pending: + for event_text, sequence_number, fingerprint, event_size in pending: + self._session.add( + HttpBridgeOperationEvent( + operation_id=first.operation_id, + sequence_number=sequence_number, + event_fingerprint=fingerprint, + event_text=event_text, + ) + ) + operation.event_bytes = total_bytes await self._session.commit() - return True + return True - async def record_recovery_attempt( + async def finalize_operation_event_spool( self, *, + operation_id: str, session_id: str, instance_id: str, owner_epoch: int, - request_fingerprint: str, - request_id: str, - account_id: str | None, - model: str | None, - replay_safe: bool, - ) -> DurableBridgeRecoveryAttemptSnapshot | None: - """Record a safe request before dispatch so an ambiguous outcome is recoverable.""" + ) -> bool: + """Mark a terminal operation replay-complete after its queue drained.""" async with sqlite_writer_section(): - # Lock the owner row through the journal write so a takeover - # cannot advance the epoch after this check but before dispatch. owner_exists = await self._session.scalar( select(HttpBridgeSessionRecord.id) .where( @@ -1075,120 +1947,89 @@ async def record_recovery_attempt( ) .with_for_update() ) - if owner_exists is None: - await self._session.rollback() - return None - attempt = await self._session.scalar( - select(HttpBridgeRecoveryAttemptRecord) - .where(HttpBridgeRecoveryAttemptRecord.session_id == session_id) - .where(HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint) - .with_for_update() - ) - if attempt is None: - attempt = HttpBridgeRecoveryAttemptRecord( - session_id=session_id, - request_fingerprint=request_fingerprint, - request_id=request_id, - account_id=account_id, - model=model, - replay_safe=replay_safe, - state=HttpBridgeRecoveryAttemptState.UNKNOWN, + result = await self._session.execute( + update(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.state.in_(("completed", "incomplete")), + HttpBridgeOperationRecord.event_spool_complete.is_(False), ) - self._session.add(attempt) - elif attempt.state == HttpBridgeRecoveryAttemptState.REPLAYED: - snapshot = _to_recovery_attempt_snapshot(attempt) - await self._session.rollback() - return snapshot - elif attempt.request_id != request_id: - # A different request already owns the UNKNOWN checkpoint. - # Do not overwrite it while that request may still be between - # admission and dispatch; the caller must fail closed rather - # than sharing a journal generation. - snapshot = _to_recovery_attempt_snapshot(attempt) - await self._session.rollback() - return snapshot - else: - attempt.request_id = request_id - attempt.account_id = account_id - attempt.model = model - attempt.replay_safe = replay_safe - attempt.state = HttpBridgeRecoveryAttemptState.UNKNOWN - attempt.response_id = None - try: - await self._session.commit() - except IntegrityError: - # A concurrent owner may have inserted the same fingerprint - # after our initial SELECT (the absent-row case cannot be - # locked by SQLite). Re-read the winner and use its durable - # state instead of surfacing a transient uniqueness failure. + .values(event_spool_complete=True, updated_at=utcnow()) + ) + if owner_exists is None: await self._session.rollback() - owner_exists = await self._session.scalar( - select(HttpBridgeSessionRecord.id) - .where( - HttpBridgeSessionRecord.id == session_id, - HttpBridgeSessionRecord.owner_instance_id == instance_id, - HttpBridgeSessionRecord.owner_epoch == owner_epoch, - ) - .with_for_update() - ) - if owner_exists is None: - await self._session.rollback() - return None - attempt = await self._session.scalar( - select(HttpBridgeRecoveryAttemptRecord) - .where(HttpBridgeRecoveryAttemptRecord.session_id == session_id) - .where(HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint) - ) - if attempt is None: - raise - if attempt.state == HttpBridgeRecoveryAttemptState.REPLAYED: - snapshot = _to_recovery_attempt_snapshot(attempt) - await self._session.rollback() - return snapshot - if attempt.request_id != request_id: - snapshot = _to_recovery_attempt_snapshot(attempt) - await self._session.rollback() - return snapshot - attempt.request_id = request_id - attempt.account_id = account_id - attempt.model = model - attempt.replay_safe = replay_safe - attempt.state = HttpBridgeRecoveryAttemptState.UNKNOWN - attempt.response_id = None - await self._session.commit() - await self._session.refresh(attempt) - return _to_recovery_attempt_snapshot(attempt) + return False + await self._session.commit() + return bool(getattr(result, "rowcount", 0)) - async def lookup_recovery_attempt( + async def get_latest_completed_operation( self, *, session_id: str, - request_fingerprint: str, - ) -> DurableBridgeRecoveryAttemptSnapshot | None: - attempt = await self._session.scalar( - select(HttpBridgeRecoveryAttemptRecord) - .where(HttpBridgeRecoveryAttemptRecord.session_id == session_id) - .where(HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint) - .where(HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.UNKNOWN) - .where(HttpBridgeRecoveryAttemptRecord.replay_safe.is_(True)) + parent_response_id: str, + request_fingerprint: str | None = None, + ) -> DurableBridgeOperationSnapshot | None: + predicates = [ + HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.parent_response_id == parent_response_id, + HttpBridgeOperationRecord.state == "completed", + HttpBridgeOperationRecord.response_id.is_not(None), + ] + if request_fingerprint is not None: + predicates.append(HttpBridgeOperationRecord.request_fingerprint == request_fingerprint) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where(*predicates) + .order_by(HttpBridgeOperationRecord.updated_at.desc()) + .limit(1) ) - return _to_recovery_attempt_snapshot(attempt) if attempt is not None else None + return _to_operation_snapshot(operation) if operation is not None else None - async def mark_recovery_attempt_replayed( + async def get_latest_completed_operation_any_session( + self, + *, + parent_response_id: str, + api_key_scope: str | None = None, + request_fingerprint: str | None = None, + ) -> DurableBridgeOperationSnapshot | None: + statement = select(HttpBridgeOperationRecord) + if api_key_scope is not None: + statement = statement.join( + HttpBridgeSessionRecord, + HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id, + ).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope) + operation = await self._session.scalar( + statement.where( + HttpBridgeOperationRecord.parent_response_id == parent_response_id, + HttpBridgeOperationRecord.state == "completed", + HttpBridgeOperationRecord.response_id.is_not(None), + *( + [HttpBridgeOperationRecord.request_fingerprint == request_fingerprint] + if request_fingerprint is not None + else [] + ), + ) + .order_by(HttpBridgeOperationRecord.updated_at.desc()) + .limit(1) + ) + return _to_operation_snapshot(operation) if operation is not None else None + + async def settle_terminal_append_failure( self, *, + operation_id: str, session_id: str, instance_id: str, owner_epoch: int, - request_fingerprint: str, + state: str, + expected_response_id: str | None, + expected_recovery_dispatch_count: int = 0, + alternate_expected_response_id: str | None = None, response_id: str | None = None, ) -> bool: + """Settle only the terminal attempt whose append outcome was ambiguous.""" async with sqlite_writer_section(): - # Keep the owner fence and journal transition in one transaction. - # PostgreSQL's row lock prevents a concurrent takeover from - # advancing the epoch between the check and the state update; - # sqlite_writer_section provides the equivalent writer - # serialization for SQLite. owner_exists = await self._session.scalar( select(HttpBridgeSessionRecord.id) .where( @@ -1201,38 +2042,57 @@ async def mark_recovery_attempt_replayed( if owner_exists is None: await self._session.rollback() return False - values: dict[str, object] = {"state": HttpBridgeRecoveryAttemptState.REPLAYED} + acknowledged_response_matches = ( + HttpBridgeOperationRecord.response_id == expected_response_id + if expected_response_id is not None + else HttpBridgeOperationRecord.response_id.is_(None) + ) + if alternate_expected_response_id is not None: + acknowledged_response_matches = or_( + acknowledged_response_matches, + HttpBridgeOperationRecord.response_id == alternate_expected_response_id, + ) + terminal_response_matches = ( + HttpBridgeOperationRecord.response_id == response_id + if response_id is not None + else HttpBridgeOperationRecord.response_id.is_(None) + ) + values: dict[str, object] = { + "state": state, + "event_spool_complete": False, + "updated_at": utcnow(), + } if response_id is not None: values["response_id"] = response_id - # A claim authorizes one replay and must only transition UNKNOWN - # rows. Settlement (which supplies response_id) remains idempotent - # for a REPLAYED row after the replay completes. - claimable_states = ( - (HttpBridgeRecoveryAttemptState.UNKNOWN,) - if response_id is None - else (HttpBridgeRecoveryAttemptState.UNKNOWN, HttpBridgeRecoveryAttemptState.REPLAYED) - ) result = await self._session.execute( - update(HttpBridgeRecoveryAttemptRecord) + update(HttpBridgeOperationRecord) .where( - HttpBridgeRecoveryAttemptRecord.session_id == session_id, - HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint, - HttpBridgeRecoveryAttemptRecord.state.in_(claimable_states), + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.recovery_dispatch_count == expected_recovery_dispatch_count, + or_( + and_(HttpBridgeOperationRecord.state == "acknowledged", acknowledged_response_matches), + and_( + HttpBridgeOperationRecord.state == state, + or_(acknowledged_response_matches, terminal_response_matches), + ), + ), ) .values(**values) ) await self._session.commit() return bool(getattr(result, "rowcount", 0)) - async def rollback_recovery_attempt_replayed( + async def update_operation( self, *, + operation_id: str, session_id: str, instance_id: str, owner_epoch: int, - request_fingerprint: str, + state: str, + response_id: str | None = None, ) -> bool: - """Return a pre-dispatch replay claim to UNKNOWN under the owner fence.""" async with sqlite_writer_section(): owner_exists = await self._session.scalar( select(HttpBridgeSessionRecord.id) @@ -1246,15 +2106,16 @@ async def rollback_recovery_attempt_replayed( if owner_exists is None: await self._session.rollback() return False + values: dict[str, object] = {"state": state, "updated_at": utcnow()} + if response_id is not None: + values["response_id"] = response_id result = await self._session.execute( - update(HttpBridgeRecoveryAttemptRecord) + update(HttpBridgeOperationRecord) .where( - HttpBridgeRecoveryAttemptRecord.session_id == session_id, - HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint, - HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.REPLAYED, - HttpBridgeRecoveryAttemptRecord.response_id.is_(None), + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, ) - .values(state=HttpBridgeRecoveryAttemptState.UNKNOWN) + .values(**values) ) await self._session.commit() return bool(getattr(result, "rowcount", 0)) @@ -1363,10 +2224,7 @@ async def purge_owned_sessions_on_startup( HttpBridgeSessionRecord.last_seen_at < ownerless_cutoff, ) ) - startup_purge_filter = and_( - or_(*purge_predicates), - HttpBridgeSessionRecord.recovery_required_anchor_hash.is_(None), - ) + startup_purge_filter = or_(*purge_predicates) result = await self._session.execute( select( HttpBridgeSessionRecord.id, @@ -1384,25 +2242,90 @@ async def purge_owned_sessions_on_startup( session_ids = [candidate.id for candidate in candidates] if not session_ids: return deleted_count + # Operation rows are the durable recovery ledger. Never cascade + # delete a session that still owns a retained operation, including + # completed replayable transcripts; detach it so the next instance + # can inspect and take over without losing continuity history. + operation_session_ids = set( + await self._session.scalars( + select(HttpBridgeOperationRecord.session_id).where( + HttpBridgeOperationRecord.session_id.in_(session_ids), + ) + ) + ) retained_recovery_ids = { candidate.id for candidate in candidates - if candidate.owner_instance_id == instance_id - and getattr(candidate, "owner_process_epoch", None) == owner_process_epoch - and (ownerless_cutoff is None or to_utc_naive(candidate.last_seen_at) >= to_utc_naive(ownerless_cutoff)) - and is_http_bridge_account_neutral_replay( - kind=candidate.session_key_kind, - key=candidate.session_key_value, + if candidate.id in operation_session_ids + or ( + candidate.owner_instance_id == instance_id + and getattr(candidate, "owner_process_epoch", None) == owner_process_epoch + and ( + ownerless_cutoff is None + or to_utc_naive(candidate.last_seen_at) >= to_utc_naive(ownerless_cutoff) + ) + and is_http_bridge_account_neutral_replay( + kind=candidate.session_key_kind, + key=candidate.session_key_value, + ) ) } async with sqlite_writer_section(): + ownerless_operation_ids = { + candidate.id + for candidate in candidates + if candidate.id in retained_recovery_ids + and candidate.id in operation_session_ids + and candidate.owner_instance_id is None + } + if ownerless_operation_ids: + # The ownerless-cutoff predicate is part of the same + # startup query. Refresh retained rows so the bounded + # loop cannot select them forever while their operation + # transcript is awaiting normal retention cleanup. + await self._session.execute( + update(HttpBridgeSessionRecord) + .where( + HttpBridgeSessionRecord.id.in_(ownerless_operation_ids), + HttpBridgeSessionRecord.owner_instance_id.is_(None), + ) + .values(last_seen_at=now, lease_expires_at=now) + ) if retained_recovery_ids: + # A process can die after recording a submitted + # operation but before upstream acknowledges it. Once + # startup has fenced and detached that owner's session, + # classify those rows as UNKNOWN so the replacement can + # enter the normal proof-gated recovery path. + operation_retained_session_ids = retained_recovery_ids & operation_session_ids + if operation_retained_session_ids: + eligible_operation_sessions = set( + await self._session.scalars( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id.in_(operation_retained_session_ids), + startup_purge_filter, + ) + .with_for_update() + ) + ) + await self._session.execute( + update(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.session_id.in_(eligible_operation_sessions), + HttpBridgeOperationRecord.state == "submitted", + ) + .values(state="unknown", updated_at=now) + ) await self._session.execute( update(HttpBridgeSessionRecord) .where( HttpBridgeSessionRecord.id.in_(retained_recovery_ids), - HttpBridgeSessionRecord.owner_instance_id == instance_id, - HttpBridgeSessionRecord.owner_process_epoch == owner_process_epoch, + # Detach the rows selected as belonging to the + # previous process. With an explicit new epoch, + # matching the new epoch here would leave old + # retained rows selected forever on every loop. + startup_purge_filter, ) .values( owner_instance_id=None, @@ -1453,7 +2376,6 @@ async def purge_owned_sessions_on_startup( latest_input_item_count=None, latest_input_full_fingerprint=None, latest_pending_tool_calls_json=None, - latest_response_transition_manifest_json=None, ) .returning(HttpBridgeSessionRecord.id) ) @@ -1497,7 +2419,11 @@ async def purge_closed_before(self, cutoff: datetime, *, batch_size: int = _PURG .where( HttpBridgeSessionRecord.state == HttpBridgeSessionState.CLOSED, HttpBridgeSessionRecord.last_seen_at < cutoff, - HttpBridgeSessionRecord.recovery_required_anchor_hash.is_(None), + ~exists( + select(HttpBridgeOperationRecord.operation_id).where( + HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id, + ) + ), ) .order_by(HttpBridgeSessionRecord.last_seen_at.asc()) .limit(batch_size) @@ -1513,7 +2439,11 @@ async def purge_closed_before(self, cutoff: datetime, *, batch_size: int = _PURG HttpBridgeSessionRecord.id.in_(session_ids), HttpBridgeSessionRecord.state == HttpBridgeSessionState.CLOSED, HttpBridgeSessionRecord.last_seen_at < cutoff, - HttpBridgeSessionRecord.recovery_required_anchor_hash.is_(None), + ~exists( + select(HttpBridgeOperationRecord.operation_id).where( + HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id, + ) + ), ) ) ) @@ -1523,7 +2453,13 @@ async def purge_closed_before(self, cutoff: datetime, *, batch_size: int = _PURG .where(HttpBridgeSessionRecord.id.in_(session_ids)) .where(HttpBridgeSessionRecord.state == HttpBridgeSessionState.CLOSED) .where(HttpBridgeSessionRecord.last_seen_at < cutoff) - .where(HttpBridgeSessionRecord.recovery_required_anchor_hash.is_(None)) + .where( + ~exists( + select(HttpBridgeOperationRecord.operation_id).where( + HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id, + ) + ) + ) .returning(HttpBridgeSessionRecord.id) ) await self._session.commit() @@ -1542,7 +2478,11 @@ async def purge_abandoned_before(self, cutoff: datetime, *, batch_size: int = _P HttpBridgeSessionRecord.lease_expires_at < now, ), HttpBridgeSessionRecord.last_seen_at < cutoff, - HttpBridgeSessionRecord.recovery_required_anchor_hash.is_(None), + ~exists( + select(HttpBridgeOperationRecord.operation_id).where( + HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id, + ) + ), ) result = await self._session.execute( select(HttpBridgeSessionRecord.id) @@ -1642,7 +2582,6 @@ async def register_owned_alias( latest_input_item_count: int | None = None, latest_input_full_fingerprint: str | None = None, latest_pending_tool_calls: Mapping[str, str] | None = None, - latest_response_transition_manifest: ResponseTransitionManifest | None = None, ) -> DurableBridgeAliasRegistration: """Register continuity only while the caller still owns the durable row.""" @@ -1662,14 +2601,6 @@ async def register_owned_alias( latest_response_id, latest_pending_tool_calls, ) - session_values["latest_response_transition_manifest_json"] = encode_response_transition_manifest( - latest_response_transition_manifest - ) - session_values["recovery_required_anchor_hash"] = None - session_values["recovery_required_account_id"] = None - session_values["recovery_required_attempt_fingerprint"] = None - session_values["recovery_required_attempt_request_id"] = None - session_values["recovery_required_at"] = None elif latest_input_item_count is not None and latest_input_full_fingerprint is not None: session_values["latest_input_item_count"] = latest_input_item_count session_values["latest_input_full_fingerprint"] = latest_input_full_fingerprint @@ -1709,95 +2640,6 @@ async def register_owned_alias( await self._session.commit() return DurableBridgeAliasRegistration.REGISTERED - async def settle_marker_recovery_completed( - self, - *, - session_id: str, - api_key_scope: str, - instance_id: str, - owner_epoch: int, - account_id: str, - request_fingerprint: str, - claim_request_id: str, - request_id: str, - response_id: str, - input_item_count: int, - input_full_fingerprint: str, - pending_tool_calls: Mapping[str, str], - response_transition_manifest: ResponseTransitionManifest | None, - lease_ttl_seconds: float, - ) -> bool: - """Atomically publish a proof-gated durable-marker recovery checkpoint.""" - - async with sqlite_writer_section(): - marker = await self._session.scalar( - select(HttpBridgeSessionRecord) - .where( - HttpBridgeSessionRecord.id == session_id, - HttpBridgeSessionRecord.api_key_scope == api_key_scope, - HttpBridgeSessionRecord.owner_instance_id == instance_id, - HttpBridgeSessionRecord.owner_epoch == owner_epoch, - HttpBridgeSessionRecord.account_id == account_id, - ) - .with_for_update() - ) - journal = await self._session.scalar( - select(HttpBridgeRecoveryAttemptRecord) - .where( - HttpBridgeRecoveryAttemptRecord.session_id == session_id, - HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint, - HttpBridgeRecoveryAttemptRecord.request_id == request_id, - ) - .with_for_update() - ) - if ( - marker is None - or marker.latest_response_id is None - or marker.recovery_required_anchor_hash != durable_bridge_hash(marker.latest_response_id) - or marker.recovery_required_account_id != account_id - or marker.recovery_required_attempt_fingerprint != request_fingerprint - or marker.recovery_required_attempt_request_id != claim_request_id - or journal is None - or journal.account_id != account_id - or journal.state != HttpBridgeRecoveryAttemptState.UNKNOWN - or journal.response_id is not None - ): - await self._session.rollback() - return False - - marker.latest_response_id = response_id - marker.latest_input_item_count = input_item_count - marker.latest_input_full_fingerprint = input_full_fingerprint - marker.latest_pending_tool_calls_json = _encode_pending_tool_calls( - response_id, - pending_tool_calls, - ) - marker.latest_response_transition_manifest_json = encode_response_transition_manifest( - response_transition_manifest - ) - marker.recovery_required_anchor_hash = None - marker.recovery_required_account_id = None - marker.recovery_required_attempt_fingerprint = None - marker.recovery_required_attempt_request_id = None - marker.recovery_required_at = None - now = utcnow() - marker.last_seen_at = now - marker.lease_expires_at = now + timedelta(seconds=max(1.0, lease_ttl_seconds)) - registered = await self._execute_alias_upsert( - session_id=session_id, - alias_kind="previous_response_id", - alias_value=response_id, - api_key_scope=api_key_scope, - target_account_neutral_replay=False, - ) - if not registered: - await self._session.rollback() - return False - journal.state = HttpBridgeRecoveryAttemptState.REPLAYED - journal.response_id = response_id - await self._session.commit() - return True - async def register_reversible_turn_state_alias( self, *, @@ -2146,42 +2988,22 @@ async def missing_durable_bridge_tables(session: AsyncSession) -> tuple[str, ... "SELECT name FROM sqlite_master " "WHERE type = 'table' " "AND name IN ('http_bridge_sessions', 'http_bridge_session_aliases', 'http_bridge_retry_circuits', " - "'http_bridge_recovery_attempts', 'http_bridge_rowless_recovery_authorities')" + "'http_bridge_recovery_attempts', 'http_bridge_operations', 'http_bridge_operation_events')" ) ) else: result = await session.execute( text( "SELECT table_name FROM information_schema.tables " - "WHERE table_schema = ANY (current_schemas(false)) " + "WHERE table_schema = 'public' " "AND table_name IN (" "'http_bridge_sessions', 'http_bridge_session_aliases', 'http_bridge_retry_circuits', " - "'http_bridge_recovery_attempts', 'http_bridge_rowless_recovery_authorities'" + "'http_bridge_recovery_attempts', 'http_bridge_operations', 'http_bridge_operation_events'" ")" ) ) present = {str(row[0]) for row in result.fetchall()} - missing = set(expected - present) - for table_name, required_columns in REQUIRED_DURABLE_BRIDGE_COLUMNS.items(): - if table_name not in present: - continue - if dialect == "sqlite": - column_result = await session.execute(text(f"PRAGMA table_info({table_name})")) - present_columns = {str(row[1]) for row in column_result.fetchall()} - else: - column_result = await session.execute( - text( - "SELECT column_name FROM information_schema.columns " - "WHERE table_schema = ANY (current_schemas(false)) " - "AND table_name = :table_name" - ), - {"table_name": table_name}, - ) - present_columns = {str(row[0]) for row in column_result.fetchall()} - for column in required_columns: - if column not in present_columns: - missing.add(f"{table_name}.{column}") - return tuple(sorted(missing)) + return tuple(sorted(expected - present)) _SNAPSHOT_COLUMNS = ( @@ -2203,12 +3025,6 @@ async def missing_durable_bridge_tables(session: AsyncSession) -> tuple[str, ... HttpBridgeSessionRecord.latest_input_item_count, HttpBridgeSessionRecord.latest_input_full_fingerprint, HttpBridgeSessionRecord.latest_pending_tool_calls_json, - HttpBridgeSessionRecord.latest_response_transition_manifest_json, - HttpBridgeSessionRecord.recovery_required_anchor_hash, - HttpBridgeSessionRecord.recovery_required_account_id, - HttpBridgeSessionRecord.recovery_required_attempt_fingerprint, - HttpBridgeSessionRecord.recovery_required_attempt_request_id, - HttpBridgeSessionRecord.recovery_required_at, HttpBridgeSessionRecord.last_seen_at, HttpBridgeSessionRecord.closed_at, ) @@ -2238,14 +3054,6 @@ def _returned_row_to_snapshot(row: Row[tuple[object, ...]]) -> DurableBridgeSess mapping[HttpBridgeSessionRecord.latest_response_id], mapping[HttpBridgeSessionRecord.latest_pending_tool_calls_json], ), - latest_response_transition_manifest=decode_response_transition_manifest( - mapping[HttpBridgeSessionRecord.latest_response_transition_manifest_json] - ), - recovery_required_anchor_hash=mapping[HttpBridgeSessionRecord.recovery_required_anchor_hash], - recovery_required_account_id=mapping[HttpBridgeSessionRecord.recovery_required_account_id], - recovery_required_attempt_fingerprint=mapping[HttpBridgeSessionRecord.recovery_required_attempt_fingerprint], - recovery_required_attempt_request_id=mapping[HttpBridgeSessionRecord.recovery_required_attempt_request_id], - recovery_required_at=mapping[HttpBridgeSessionRecord.recovery_required_at], last_seen_at=mapping[HttpBridgeSessionRecord.last_seen_at], closed_at=mapping[HttpBridgeSessionRecord.closed_at], ) @@ -2276,14 +3084,6 @@ def _to_snapshot(row: HttpBridgeSessionRecord | None) -> DurableBridgeSessionSna row.latest_response_id, row.latest_pending_tool_calls_json, ), - latest_response_transition_manifest=decode_response_transition_manifest( - row.latest_response_transition_manifest_json - ), - recovery_required_anchor_hash=row.recovery_required_anchor_hash, - recovery_required_account_id=row.recovery_required_account_id, - recovery_required_attempt_fingerprint=row.recovery_required_attempt_fingerprint, - recovery_required_attempt_request_id=row.recovery_required_attempt_request_id, - recovery_required_at=row.recovery_required_at, last_seen_at=row.last_seen_at, closed_at=row.closed_at, ) @@ -2311,6 +3111,27 @@ def _to_recovery_attempt_snapshot( ) +def _to_operation_snapshot( + row: HttpBridgeOperationRecord, + *, + created: bool = False, +) -> DurableBridgeOperationSnapshot: + return DurableBridgeOperationSnapshot( + operation_id=row.operation_id, + session_id=row.session_id, + request_fingerprint=row.request_fingerprint, + account_id=row.account_id, + model=row.model, + parent_response_id=row.parent_response_id, + state=row.state, + response_id=row.response_id, + recovery_dispatch_count=row.recovery_dispatch_count, + request_text=row.request_text, + event_spool_complete=bool(row.event_spool_complete), + created=created, + ) + + def _to_retry_circuit_snapshot(row: HttpBridgeRetryCircuit | None) -> DurableBridgeRetryCircuitSnapshot | None: if row is None: return None diff --git a/app/modules/proxy/file_pin_repository.py b/app/modules/proxy/file_pin_repository.py new file mode 100644 index 0000000000..26750826ed --- /dev/null +++ b/app/modules/proxy/file_pin_repository.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +from collections.abc import Collection + +from sqlalchemy import Integer, bindparam, text +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.sql.elements import TextClause + +from app.db.session import sqlite_writer_section + +_TABLE = "file_account_pins" + +# Ownership TTLs stay entirely in the database clock domain. PostgreSQL's +# clock_timestamp() is evaluated when each clause executes. A successful claim +# is followed by a guarded refresh in the same transaction so even an INSERT +# that waited behind an ultimately rolled-back unique contender receives its +# full TTL after the wait. SQLite's padded strftime form matches SQLAlchemy +# DateTime's six-digit fractional width, preserving exact lexicographic expiry +# checks. +_POSTGRES_NOW = "clock_timestamp()" +_POSTGRES_NOW_PLUS_TTL = "clock_timestamp() + make_interval(secs => :ttl)" +_POSTGRES_STATEMENT_NOW = "statement_timestamp()" +_SQLITE_NOW = "(strftime('%Y-%m-%d %H:%M:%f', 'now') || '000')" +_SQLITE_NOW_PLUS_TTL = "(strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || :ttl || ' seconds') || '000')" + +_POSTGRES_CLAIM = text( + f""" + INSERT INTO {_TABLE} (file_id, account_id, expires_at) + VALUES (:file_id, :account_id, {_POSTGRES_NOW_PLUS_TTL}) + ON CONFLICT (file_id) DO UPDATE SET + account_id = excluded.account_id, + expires_at = {_POSTGRES_NOW_PLUS_TTL} + WHERE {_TABLE}.account_id = :account_id + OR {_TABLE}.expires_at <= {_POSTGRES_NOW} + RETURNING account_id + """ +).bindparams(bindparam("ttl", type_=Integer)) + +_SQLITE_CLAIM = text( + f""" + INSERT INTO {_TABLE} (file_id, account_id, expires_at) + VALUES (:file_id, :account_id, {_SQLITE_NOW_PLUS_TTL}) + ON CONFLICT (file_id) DO UPDATE SET + account_id = excluded.account_id, + expires_at = {_SQLITE_NOW_PLUS_TTL} + WHERE {_TABLE}.account_id = :account_id + OR {_TABLE}.expires_at <= {_SQLITE_NOW} + RETURNING account_id + """ +).bindparams(bindparam("ttl", type_=Integer)) + +_POSTGRES_CLEANUP = text(f"DELETE FROM {_TABLE} WHERE expires_at <= {_POSTGRES_STATEMENT_NOW}") +_SQLITE_CLEANUP = text(f"DELETE FROM {_TABLE} WHERE expires_at <= {_SQLITE_NOW}") + +_POSTGRES_REFRESH = text( + f""" + UPDATE {_TABLE} + SET expires_at = {_POSTGRES_NOW_PLUS_TTL} + WHERE file_id = :file_id + AND account_id = :account_id + RETURNING account_id + """ +).bindparams(bindparam("ttl", type_=Integer)) +_SQLITE_REFRESH = text( + f""" + UPDATE {_TABLE} + SET expires_at = {_SQLITE_NOW_PLUS_TTL} + WHERE file_id = :file_id + AND account_id = :account_id + RETURNING account_id + """ +).bindparams(bindparam("ttl", type_=Integer)) + +_POSTGRES_GET_LIVE = text(f"SELECT account_id FROM {_TABLE} WHERE file_id = :file_id AND expires_at > {_POSTGRES_NOW}") +_SQLITE_GET_LIVE = text(f"SELECT account_id FROM {_TABLE} WHERE file_id = :file_id AND expires_at > {_SQLITE_NOW}") + +_POSTGRES_GET_LIVE_MANY = text( + f""" + SELECT file_id, account_id + FROM {_TABLE} + WHERE file_id IN :file_ids + AND expires_at > {_POSTGRES_NOW} + """ +).bindparams(bindparam("file_ids", expanding=True)) +_SQLITE_GET_LIVE_MANY = text( + f""" + SELECT file_id, account_id + FROM {_TABLE} + WHERE file_id IN :file_ids + AND expires_at > {_SQLITE_NOW} + """ +).bindparams(bindparam("file_ids", expanding=True)) + +_GET_ACCOUNT = text(f"SELECT account_id FROM {_TABLE} WHERE file_id = :file_id") + + +class FileAccountPinOwnershipConflict(RuntimeError): + def __init__(self, file_id: str, persisted_account_id: str, requested_account_id: str) -> None: + super().__init__( + f"Live file ownership conflict for {file_id!r}: " + f"persisted={persisted_account_id!r} requested={requested_account_id!r}" + ) + self.file_id = file_id + self.persisted_account_id = persisted_account_id + self.requested_account_id = requested_account_id + + +def build_file_account_pin_claim(*, dialect_name: str) -> TextClause: + if dialect_name == "postgresql": + return _POSTGRES_CLAIM + if dialect_name == "sqlite": + return _SQLITE_CLAIM + raise RuntimeError(f"Unsupported database dialect for file account pins: {dialect_name}") + + +def build_file_account_pin_cleanup(*, dialect_name: str) -> TextClause: + if dialect_name == "postgresql": + return _POSTGRES_CLEANUP + if dialect_name == "sqlite": + return _SQLITE_CLEANUP + raise RuntimeError(f"Unsupported database dialect for file account pins: {dialect_name}") + + +def build_file_account_pin_refresh(*, dialect_name: str) -> TextClause: + if dialect_name == "postgresql": + return _POSTGRES_REFRESH + if dialect_name == "sqlite": + return _SQLITE_REFRESH + raise RuntimeError(f"Unsupported database dialect for file account pins: {dialect_name}") + + +def build_file_account_pin_live_lookup(*, dialect_name: str, many: bool = False) -> TextClause: + if dialect_name == "postgresql": + return _POSTGRES_GET_LIVE_MANY if many else _POSTGRES_GET_LIVE + if dialect_name == "sqlite": + return _SQLITE_GET_LIVE_MANY if many else _SQLITE_GET_LIVE + raise RuntimeError(f"Unsupported database dialect for file account pins: {dialect_name}") + + +class FileAccountPinRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def claim(self, file_id: str, account_id: str, *, ttl_seconds: int) -> None: + if ttl_seconds <= 0: + raise ValueError("File account pin TTL must be positive") + dialect_name = self._dialect_name() + params = { + "file_id": file_id, + "account_id": account_id, + "ttl": ttl_seconds, + } + async with sqlite_writer_section(): + await self._session.execute(build_file_account_pin_cleanup(dialect_name=dialect_name)) + persisted_account_id = ( + await self._session.execute( + build_file_account_pin_claim(dialect_name=dialect_name), + params, + ) + ).scalar_one_or_none() + if persisted_account_id is None: + persisted_account_id = await self._session.scalar( + _GET_ACCOUNT, + {"file_id": file_id}, + ) + if persisted_account_id != account_id: + await self._session.rollback() + raise FileAccountPinOwnershipConflict( + file_id, + persisted_account_id or "", + account_id, + ) + refreshed_account_id = ( + await self._session.execute( + build_file_account_pin_refresh(dialect_name=dialect_name), + params, + ) + ).scalar_one_or_none() + if refreshed_account_id != account_id: + await self._session.rollback() + raise RuntimeError(f"Failed to refresh file account pin after claim: {file_id!r}") + await self._session.commit() + + async def get_live_account_id(self, file_id: str) -> str | None: + return await self._session.scalar( + build_file_account_pin_live_lookup(dialect_name=self._dialect_name()), + {"file_id": file_id}, + ) + + async def get_live_account_ids(self, file_ids: Collection[str]) -> dict[str, str]: + unique_file_ids = tuple(dict.fromkeys(file_ids)) + if not unique_file_ids: + return {} + rows = ( + ( + await self._session.execute( + build_file_account_pin_live_lookup( + dialect_name=self._dialect_name(), + many=True, + ), + {"file_ids": unique_file_ids}, + ) + ) + .tuples() + .all() + ) + return dict(rows) + + def _dialect_name(self) -> str: + return self._session.get_bind().dialect.name diff --git a/app/modules/proxy/http_bridge_event_batcher.py b/app/modules/proxy/http_bridge_event_batcher.py new file mode 100644 index 0000000000..3ec3892ee2 --- /dev/null +++ b/app/modules/proxy/http_bridge_event_batcher.py @@ -0,0 +1,393 @@ +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass +from typing import Any + +from app.core.config.settings import get_settings +from app.modules.proxy.durable_bridge_repository import DurableBridgeOperationEventInput + +logger = logging.getLogger("app.modules.proxy.http_bridge_event_batcher") + + +@dataclass(frozen=True, slots=True) +class _PendingOperationEvent: + operation_id: str + session_id: str + instance_id: str + owner_epoch: int + event_text: str + + +@dataclass(frozen=True, slots=True) +class TerminalOperationEventAppendResult: + persisted: bool + settlement_required: bool = False + + def __bool__(self) -> bool: + return self.persisted + + +class HttpBridgeOperationEventBatcher: + """Best-effort in-memory event buffer for the HTTP bridge. + + Normal stream handling only appends to memory. A short-lived flusher + commits groups of events in one transaction. A terminal event drains its + operation synchronously once, so a completed operation is marked + replayable only after all queued events were persisted. A process crash or + queue overflow therefore loses optional transcript data, never upstream + work safety. + """ + + @classmethod + def from_settings(cls, durable_bridge: Any, settings: Any | None = None) -> "HttpBridgeOperationEventBatcher": + """Build the event spooler from the operator-facing settings surface.""" + settings = settings or get_settings() + return cls( + durable_bridge, + max_bytes=int( + getattr(settings, "http_responses_session_bridge_operation_event_spool_max_bytes", 2 * 1024 * 1024) + ), + batch_size=int(getattr(settings, "http_responses_session_bridge_operation_event_spool_batch_size", 32)), + flush_interval_seconds=float( + getattr(settings, "http_responses_session_bridge_operation_event_spool_flush_interval_seconds", 0.1) + ), + max_pending_events=int( + getattr(settings, "http_responses_session_bridge_operation_event_spool_max_pending_events", 2048) + ), + max_pending_bytes=int( + getattr( + settings, "http_responses_session_bridge_operation_event_spool_max_pending_bytes", 32 * 1024 * 1024 + ) + ), + ) + + def __init__( + self, + durable_bridge: Any, + *, + max_bytes: int, + batch_size: int = 32, + flush_interval_seconds: float = 0.1, + max_pending_events: int = 2048, + max_pending_bytes: int = 32 * 1024 * 1024, + ) -> None: + self._durable_bridge = durable_bridge + self._max_bytes = max_bytes + self._batch_size = batch_size + self._flush_interval_seconds = flush_interval_seconds + self._max_pending_events = max_pending_events + self._max_pending_bytes = max_pending_bytes + self._pending: dict[str, list[_PendingOperationEvent]] = {} + self._contexts: dict[str, _PendingOperationEvent] = {} + self._dropped_operations: set[str] = set() + self._closing_operations: set[str] = set() + self._pending_count = 0 + self._pending_bytes = 0 + self._lock = asyncio.Lock() + # SQLite already serializes writers; this also prevents a background + # flush racing a terminal drain and final marker for one operation. + self._flush_lock = asyncio.Lock() + self._wake = asyncio.Event() + self._task: asyncio.Task[None] | None = None + + async def enqueue( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + event_text: str, + terminal: bool = False, + ) -> None: + self._ensure_task() + pending = _PendingOperationEvent( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + event_text=event_text, + ) + async with self._lock: + self._contexts.setdefault(operation_id, pending) + if terminal: + self._closing_operations.add(operation_id) + if operation_id not in self._dropped_operations: + event_bytes = len(event_text.encode("utf-8")) + if ( + self._pending_count >= self._max_pending_events + or self._pending_bytes + event_bytes > self._max_pending_bytes + ): + self._dropped_operations.add(operation_id) + dropped = self._pending.pop(operation_id, []) + self._pending_count -= len(dropped) + self._pending_bytes -= sum(len(item.event_text.encode("utf-8")) for item in dropped) + logger.info( + "Dropping HTTP bridge transcript events after queue overflow operation_id=%s", + operation_id, + ) + else: + self._pending.setdefault(operation_id, []).append(pending) + self._pending_count += 1 + self._pending_bytes += event_bytes + self._wake.set() + if terminal: + await self.flush_operation(operation_id=operation_id) + + def _ensure_task(self) -> None: + if self._task is None or self._task.done(): + self._task = asyncio.create_task(self._run(), name="http-bridge-operation-event-flusher") + + async def _run(self) -> None: + while True: + try: + await asyncio.wait_for(self._wake.wait(), timeout=self._flush_interval_seconds) + except TimeoutError: + pass + self._wake.clear() + operation_ids = await self._operation_ids_to_flush() + for operation_id in operation_ids: + await self._flush_one(operation_id) + + async def _operation_ids_to_flush(self) -> list[str]: + async with self._lock: + return [operation_id for operation_id in self._pending if operation_id not in self._closing_operations] + + async def _take_batch(self, operation_id: str) -> list[_PendingOperationEvent]: + async with self._lock: + pending = self._pending.get(operation_id, []) + batch = pending[: self._batch_size] + if batch: + del pending[: len(batch)] + self._pending_count -= len(batch) + self._pending_bytes -= sum(len(item.event_text.encode("utf-8")) for item in batch) + if not pending: + self._pending.pop(operation_id, None) + return batch + + async def _flush_one(self, operation_id: str) -> None: + async with self._flush_lock: + batch = await self._take_batch(operation_id) + if not batch: + return + async with self._lock: + if operation_id in self._dropped_operations: + return + try: + persisted = await self._durable_bridge.append_operation_events( + events=[ + DurableBridgeOperationEventInput( + operation_id=item.operation_id, + session_id=item.session_id, + instance_id=item.instance_id, + owner_epoch=item.owner_epoch, + event_text=item.event_text, + ) + for item in batch + ], + max_bytes=self._max_bytes, + ) + if not persisted: + async with self._lock: + self._dropped_operations.add(operation_id) + dropped = self._pending.pop(operation_id, []) + self._pending_count -= len(dropped) + self._pending_bytes -= sum(len(item.event_text.encode("utf-8")) for item in dropped) + except Exception: + async with self._lock: + self._dropped_operations.add(operation_id) + dropped = self._pending.pop(operation_id, []) + self._pending_count -= len(dropped) + self._pending_bytes -= sum(len(item.event_text.encode("utf-8")) for item in dropped) + logger.debug( + "Dropping failed HTTP bridge transcript event batch operation_id=%s", + operation_id, + exc_info=True, + ) + + async def flush_operation(self, *, operation_id: str) -> None: + await self.flush_pending_operation(operation_id=operation_id) + async with self._lock: + dropped = operation_id in self._dropped_operations + context = self._contexts.get(operation_id) + self._closing_operations.discard(operation_id) + self._contexts.pop(operation_id, None) + self._dropped_operations.discard(operation_id) + if dropped or context is None: + return + # A single final marker is the only synchronous database operation on + # the terminal path. If it fails, the operation remains ineligible for + # transcript replay. + try: + finalized = await self._durable_bridge.finalize_operation_event_spool( + operation_id=context.operation_id, + session_id=context.session_id, + instance_id=context.instance_id, + owner_epoch=context.owner_epoch, + ) + if not finalized: + logger.debug( + "HTTP bridge operation spool finalization was fenced or ineligible operation_id=%s", + operation_id, + ) + except Exception: + logger.debug( + "Failed to finalize HTTP bridge operation event spool operation_id=%s", + operation_id, + exc_info=True, + ) + + async def append_terminal_event( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + event_text: str, + max_bytes: int, + state: str, + expected_recovery_dispatch_count: int = 0, + response_id: str | None = None, + ) -> TerminalOperationEventAppendResult: + """Drain queued events and atomically append the terminal outcome.""" + async with self._lock: + self._contexts.setdefault( + operation_id, + _PendingOperationEvent( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + event_text=event_text, + ), + ) + self._closing_operations.add(operation_id) + await self.flush_pending_operation(operation_id=operation_id) + async with self._lock: + context = self._contexts.get(operation_id) + dropped = operation_id in self._dropped_operations + if context is None: + return TerminalOperationEventAppendResult(persisted=False) + if dropped: + try: + await self._durable_bridge.update_operation( + operation_id=operation_id, + session_id=context.session_id, + instance_id=context.instance_id, + owner_epoch=context.owner_epoch, + state=state, + response_id=response_id, + ) + except Exception: + logger.debug( + "Failed to settle dropped terminal HTTP bridge operation_id=%s", + operation_id, + exc_info=True, + ) + finally: + async with self._lock: + self._closing_operations.discard(operation_id) + self._contexts.pop(operation_id, None) + self._dropped_operations.discard(operation_id) + return TerminalOperationEventAppendResult(persisted=False) + try: + persisted = await self._durable_bridge.append_terminal_operation_event( + operation_id=operation_id, + session_id=context.session_id, + instance_id=context.instance_id, + owner_epoch=context.owner_epoch, + event_text=event_text, + max_bytes=max_bytes, + state=state, + expected_recovery_dispatch_count=expected_recovery_dispatch_count, + response_id=response_id, + ) + return TerminalOperationEventAppendResult(persisted=bool(persisted and not dropped)) + except Exception: + logger.debug( + "Failed to append terminal HTTP bridge event operation_id=%s", + operation_id, + exc_info=True, + ) + return TerminalOperationEventAppendResult( + persisted=False, + settlement_required=True, + ) + finally: + async with self._lock: + self._closing_operations.discard(operation_id) + self._contexts.pop(operation_id, None) + self._dropped_operations.discard(operation_id) + + async def settle_terminal_event( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + state: str, + expected_response_id: str | None, + expected_recovery_dispatch_count: int = 0, + alternate_expected_response_id: str | None = None, + response_id: str | None = None, + ) -> None: + """Settle a failed terminal append after its SSE block was queued.""" + try: + settled = await self._durable_bridge.settle_terminal_append_failure( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + state=state, + expected_response_id=expected_response_id, + expected_recovery_dispatch_count=expected_recovery_dispatch_count, + alternate_expected_response_id=alternate_expected_response_id, + response_id=response_id, + ) + if not settled: + logger.warning( + "Terminal HTTP bridge operation fallback settlement was fenced operation_id=%s", + operation_id, + ) + except Exception: + logger.warning( + "Failed to settle terminal HTTP bridge operation after event append failure operation_id=%s", + operation_id, + exc_info=True, + ) + + async def flush_pending_operation(self, *, operation_id: str) -> bool: + """Drain queued events while retaining the operation context.""" + while True: + await self._flush_one(operation_id) + async with self._lock: + has_pending = bool(self._pending.get(operation_id)) + if not has_pending: + break + async with self._lock: + return operation_id not in self._dropped_operations + + async def discard_operation(self, *, operation_id: str) -> None: + """Drop an abandoned nonterminal context without finalizing its spool.""" + async with self._flush_lock: + async with self._lock: + pending = self._pending.pop(operation_id, []) + self._pending_count -= len(pending) + self._pending_bytes -= sum(len(item.event_text.encode("utf-8")) for item in pending) + self._contexts.pop(operation_id, None) + self._closing_operations.discard(operation_id) + self._dropped_operations.discard(operation_id) + + async def close(self) -> None: + task = self._task + self._task = None + if task is not None: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass diff --git a/app/modules/proxy/http_bridge_forwarding.py b/app/modules/proxy/http_bridge_forwarding.py index 7f8969da47..a69f82788d 100644 --- a/app/modules/proxy/http_bridge_forwarding.py +++ b/app/modules/proxy/http_bridge_forwarding.py @@ -123,6 +123,8 @@ async def stream_responses( headers: Mapping[str, str], context: HTTPBridgeForwardContext, request_started_at: float, + on_request_dispatched: Callable[[], None] | None = None, + on_response_rejected: Callable[[], None] | None = None, on_response_wait: Callable[[], None] | None = None, on_response_ready: Callable[[], None] | None = None, ) -> AsyncIterator[str]: @@ -134,51 +136,79 @@ async def stream_responses( if on_response_wait is not None: on_response_wait() async with aiohttp.ClientSession(timeout=timeout, trust_env=False) as session: - async with session.post( - f"{owner_endpoint}{HTTP_BRIDGE_INTERNAL_FORWARD_PATH}", - json=payload.model_dump_for_forwarding(), - headers=build_owner_forward_headers(headers=headers, payload=payload, context=context), + request_url = f"{owner_endpoint}{HTTP_BRIDGE_INTERNAL_FORWARD_PATH}" + request_payload = payload.model_dump_for_forwarding() + request_headers = build_owner_forward_headers(headers=headers, payload=payload, context=context) + request_context = session.post( + request_url, + json=request_payload, + headers=request_headers, skip_auto_headers=_OWNER_FORWARD_SKIP_AUTO_HEADERS, - ) as response: - if response.status != 200: - payload_text = await response.text() - raise ProxyResponseError( - response.status, - _owner_forward_error_payload(status_code=response.status, payload_text=payload_text), - failure_phase="owner_forward_status", - failure_detail="owner_forward_non_200", - upstream_status_code=response.status, - ) - if on_response_ready is not None: - on_response_ready() - yielded_event = False - try: - async for event_block in _iter_sse_event_blocks( - response, - request_started_at=request_started_at, - proxy_request_budget_seconds=_http_bridge_request_budget_seconds(settings), - stream_idle_timeout_seconds=settings.stream_idle_timeout_seconds, - ): - yielded_event = True - yield event_block - except _OwnerForwardStreamTimeoutError as exc: - raise OwnerForwardRelayFailure( - format_sse_event( + ) + # I/O begins when __aenter__ is awaited. Cancellation after that + # point can leave the receiver settling the reservation. + transport_started = False + observed_status = False + try: + transport_started = True + async with request_context as response: + observed_status = True + if response.status != 200: + if on_response_rejected is not None: + # The receiver contract never transfers cleanup on a + # non-200 response, so the origin may safely release. + on_response_rejected() + payload_text = await response.text() + raise ProxyResponseError( + response.status, + _owner_forward_error_payload(status_code=response.status, payload_text=payload_text), + failure_phase="owner_forward_status", + failure_detail="owner_forward_non_200", + upstream_status_code=response.status, + ) + if on_response_ready is not None: + on_response_ready() + yielded_event = False + try: + async for event_block in _iter_sse_event_blocks( + response, + request_started_at=request_started_at, + proxy_request_budget_seconds=_http_bridge_request_budget_seconds(settings), + stream_idle_timeout_seconds=settings.stream_idle_timeout_seconds, + ): + yielded_event = True + yield event_block + except _OwnerForwardStreamTimeoutError as exc: + raise OwnerForwardRelayFailure( + format_sse_event( + response_failed_event( + exc.error_code, + exc.error_message, + response_id=get_request_id(), + ) + ) + ) + if not yielded_event: + yield format_sse_event( response_failed_event( - exc.error_code, - exc.error_message, + "stream_incomplete", + "Upstream websocket closed before response.completed", response_id=get_request_id(), ) ) - ) - if not yielded_event: - yield format_sse_event( - response_failed_event( - "stream_incomplete", - "Upstream websocket closed before response.completed", - response_id=get_request_id(), - ) - ) + except aiohttp.ClientConnectorError: + # DNS/connect refusal never delivered the reservation. + raise + except asyncio.CancelledError: + if transport_started and not observed_status and on_request_dispatched is not None: + on_request_dispatched() + raise + except (aiohttp.ClientError, asyncio.TimeoutError): + if not observed_status and on_request_dispatched is not None: + # The request left local construction and may have reached + # the owner; origin must not release or replay. + on_request_dispatched() + raise def build_owner_forward_headers( diff --git a/app/modules/proxy/images_observability.py b/app/modules/proxy/images_observability.py index 8852b3481b..edef56e3b1 100644 --- a/app/modules/proxy/images_observability.py +++ b/app/modules/proxy/images_observability.py @@ -9,7 +9,7 @@ image_request_duration_seconds, image_requests_total, ) -from app.core.openai.images import is_supported_image_model +from app.core.runtime_logging import safe_log_field logger = logging.getLogger("app.modules.proxy.api") @@ -20,10 +20,19 @@ def _bounded_model_label(model: str | None) -> str: + # Keep the metric/log label bounded to a literal allowlist. Besides + # avoiding arbitrary-cardinality labels, this makes it explicit that a + # request-controlled model value is never copied into a log record. + if model == "gpt-image-2": + return "gpt-image-2" + if model == "gpt-image-1.5": + return "gpt-image-1.5" + if model == "gpt-image-1": + return "gpt-image-1" + if model == "gpt-image-1-mini": + return "gpt-image-1-mini" if model is None or not model: return "unknown" - if is_supported_image_model(model): - return model return "invalid" @@ -54,9 +63,10 @@ def record_images_route_observability( logging.INFO if status < 400 else logging.WARNING, "images_route_complete route=%s model=%s stream=%s status=%s outcome=%s duration_ms=%.2f", route, - model_label, + # lgtm [py/clear-text-logging-sensitive-data] + safe_log_field(model_label), stream_label, status, - outcome, + safe_log_field(outcome), duration_seconds * 1000.0, ) diff --git a/app/modules/proxy/load_balancer.py b/app/modules/proxy/load_balancer.py index ec6ca10917..112c11185d 100644 --- a/app/modules/proxy/load_balancer.py +++ b/app/modules/proxy/load_balancer.py @@ -135,7 +135,7 @@ if TYPE_CHECKING: from app.modules.accounts.repository import AccountsRepository - from app.modules.proxy.sticky_repository import StickySessionsRepository + from app.modules.proxy.sticky_repository import StickyOwnerLookup, StickySessionsRepository logger = logging.getLogger(__name__) @@ -228,6 +228,11 @@ class _SelectionInputs(SelectionInputsProtocol): # exclusion, runtime-health, budget, and account-cap filters. Keep that # stronger candidate pool alongside the effective routing pool. continuity_owner_candidates: list[Account] | None = None + # Sticky-row mutation is authorized by account assignment and security + # policy, before model/service-tier eligibility. Keep this separate from + # continuity ambiguity: a model-ineligible account can still own the raw + # row that this authenticated request is allowed to retire. + sticky_mutation_authority_account_ids: frozenset[str] | None = None quota_planner_settings: PlannerSettings = PlannerSettings() runtime_accounts: list[Account] | None = None error_message: str | None = None @@ -244,6 +249,12 @@ def effective_continuity_owner_candidates(self) -> list[Account]: return self.accounts return self.continuity_owner_candidates + @property + def effective_sticky_mutation_authority_account_ids(self) -> frozenset[str]: + if self.sticky_mutation_authority_account_ids is None: + return frozenset(account.id for account in self.effective_continuity_owner_candidates) + return self.sticky_mutation_authority_account_ids + def _required_continuity_owner_failure( selection_inputs: _SelectionInputs, @@ -445,7 +456,7 @@ def _api_key_stream_fair_share_denial_locked( logger.warning( "API key stream fair share denial api_key_id=%s key_inflight=%s fair_share=%s " "pool_inflight=%s pool_capacity=%s active_keys=%s", - "" if redact_sensitive_details else api_key_id, + "", decision.requester_inflight, decision.fair_share, decision.pool_inflight, @@ -521,7 +532,11 @@ async def select_account( reallocate_sticky: bool = False, sticky_source: _CodexSessionSource | None = None, legacy_sticky_key: str | None = None, + legacy_continuity_source: _CodexSessionSource | None = None, + sticky_seed_key: str | None = None, + sticky_seed_kind: StickySessionKind | None = None, spill_bare_session_on_account_cap: bool = False, + abandon_unavailable_legacy_owner: bool = False, require_unambiguous_account: bool = False, sticky_max_age_seconds: int | None = None, prefer_earlier_reset_accounts: bool = False, @@ -570,6 +585,20 @@ async def load_selection_inputs() -> _SelectionInputs: # Ownership scope and routing availability are separate. Even # an already-empty routing pool must have its owner candidates # security-filtered before conversation ambiguity is decided. + security_scope_accounts = ( + selection_inputs.runtime_accounts + if selection_inputs.runtime_accounts is not None + else [ + *selection_inputs.effective_continuity_owner_candidates, + *selection_inputs.accounts, + ] + ) + security_authorized_account_ids = frozenset( + account.id for account in security_scope_accounts if bool(account.security_work_authorized) + ) + authorized_mutation_account_ids = ( + selection_inputs.effective_sticky_mutation_authority_account_ids & security_authorized_account_ids + ) authorized_owner_candidates = [ account for account in selection_inputs.effective_continuity_owner_candidates @@ -585,6 +614,7 @@ async def load_selection_inputs() -> _SelectionInputs: latest_secondary={}, latest_monthly=selection_inputs.latest_monthly, continuity_owner_candidates=authorized_owner_candidates, + sticky_mutation_authority_account_ids=authorized_mutation_account_ids, quota_planner_settings=selection_inputs.quota_planner_settings, runtime_accounts=selection_inputs.runtime_accounts, error_message="No accounts marked as authorized for security work", @@ -596,6 +626,7 @@ async def load_selection_inputs() -> _SelectionInputs: latest_secondary=selection_inputs.latest_secondary, latest_monthly=selection_inputs.latest_monthly, continuity_owner_candidates=authorized_owner_candidates, + sticky_mutation_authority_account_ids=authorized_mutation_account_ids, quota_planner_settings=selection_inputs.quota_planner_settings, runtime_accounts=selection_inputs.runtime_accounts, error_message=selection_inputs.error_message, @@ -617,6 +648,9 @@ async def load_selection_inputs() -> _SelectionInputs: latest_secondary={}, latest_monthly=selection_inputs.latest_monthly, continuity_owner_candidates=selection_inputs.effective_continuity_owner_candidates, + sticky_mutation_authority_account_ids=( + selection_inputs.effective_sticky_mutation_authority_account_ids + ), quota_planner_settings=selection_inputs.quota_planner_settings, runtime_accounts=selection_inputs.runtime_accounts, error_message="No accounts marked as authorized for security work", @@ -628,6 +662,9 @@ async def load_selection_inputs() -> _SelectionInputs: latest_secondary=selection_inputs.latest_secondary, latest_monthly=selection_inputs.latest_monthly, continuity_owner_candidates=selection_inputs.effective_continuity_owner_candidates, + sticky_mutation_authority_account_ids=( + selection_inputs.effective_sticky_mutation_authority_account_ids + ), quota_planner_settings=selection_inputs.quota_planner_settings, runtime_accounts=selection_inputs.runtime_accounts, error_message=selection_inputs.error_message, @@ -686,24 +723,78 @@ async def load_selection_inputs() -> _SelectionInputs: selection_error_code: str | None = None selection_resets_at: int | None = None legacy_existing_account_id: str | None = None - if sticky_source == "session_header" and legacy_sticky_key is not None: + legacy_abandoned_account_id: str | None = None + sticky_seed_account_id: str | None = None + initial_sticky_owner_lookup: StickyOwnerLookup | None = None + needs_owner_lookups = ( + legacy_sticky_key is not None + or (sticky_seed_key is not None and sticky_seed_kind is not None) + or (sticky_key is not None and sticky_kind is not None) + ) + if needs_owner_lookups: + # One shared session serves the legacy/seed/first-sticky owner + # lookups. The SELECTs stay separate on purpose so the per-source + # predicate semantics of get_account_id_and_abandonment (tombstone + # visibility, max_age handling) are untouched; the saving is the + # 2-3 extra pool checkouts + session create/teardown lifecycles + # per request. Each later source still starts a fresh read + # transaction (release_read_snapshot): on SQLite/WAL the shared + # session would otherwise pin one snapshot at the first SELECT + # and hide a hard sticky or seed owner committed concurrently + # between the reads, letting selection overwrite that mapping. async with self._repo_factory() as repos: - legacy_existing_account_id = await repos.sticky_sessions.get_account_id( - legacy_sticky_key, - kind=StickySessionKind.CODEX_SESSION, - max_age_seconds=sticky_max_age_seconds, - ) - if required_account_id is not None and ( - legacy_existing_account_id is not None and legacy_existing_account_id != required_account_id - ): - # The required owner came from a file/response/bridge index, - # while the raw row may be legacy turn-state ownership. Neither - # source can be discarded or rewritten to resolve a conflict. - return AccountSelection( - account=None, - error_message="Account-owned continuity sources conflict; retry the logical turn", - error_code="continuity_owner_conflict", - ) + owner_snapshot_pinned = False + if legacy_sticky_key is not None: + legacy_owner_lookup = await repos.sticky_sessions.get_account_id_and_abandonment( + legacy_sticky_key, + kind=StickySessionKind.CODEX_SESSION, + # Raw rows may be historical turn-state ownership. The + # bounded thread TTL must never age out that hard evidence. + max_age_seconds=None, + # Process-session raw text is session_header even when + # request locality is thread_header. Thread-only raw keys + # keep thread_header so a session_header tombstone cannot + # hide a distinct thread owner. + continuity_source=legacy_continuity_source or "session_header", + ) + legacy_existing_account_id = legacy_owner_lookup.account_id + abandoned_account_id = legacy_owner_lookup.abandoned_account_id + if legacy_owner_lookup.continuity_abandoned is True and isinstance(abandoned_account_id, str): + legacy_abandoned_account_id = abandoned_account_id + if required_account_id is not None and ( + legacy_existing_account_id is not None and legacy_existing_account_id != required_account_id + ): + # The required owner came from a file/response/bridge index, + # while the raw row may be legacy turn-state ownership. Neither + # source can be discarded or rewritten to resolve a conflict. + return AccountSelection( + account=None, + error_message="Account-owned continuity sources conflict; retry the logical turn", + error_code="continuity_owner_conflict", + ) + owner_snapshot_pinned = True + if sticky_seed_key is not None and sticky_seed_kind is not None: + if owner_snapshot_pinned: + await repos.sticky_sessions.release_read_snapshot() + sticky_seed_account_id = await repos.sticky_sessions.get_account_id( + sticky_seed_key, + kind=sticky_seed_kind, + ) + owner_snapshot_pinned = True + if sticky_key is not None and sticky_kind is not None: + if owner_snapshot_pinned: + await repos.sticky_sessions.release_read_snapshot() + # First-iteration owner read for run_sticky_selection_path, + # hoisted here so it shares this session. The selection + # loop consumes it exactly once; every retry (including + # post-reset attempts) still re-reads fresh ownership + # evidence through its own repo bundle. + initial_sticky_owner_lookup = await repos.sticky_sessions.get_account_id_and_abandonment( + sticky_key, + kind=sticky_kind, + max_age_seconds=sticky_max_age_seconds, + continuity_source=sticky_source, + ) # Resolve uniqueness from the model/API-key/security-scoped pool before # runtime health, budget, or cap filtering. Transient pressure cannot # prove that another candidate does not own an upstream conversation. @@ -766,6 +857,27 @@ async def load_selection_inputs() -> _SelectionInputs: error_message=error_message, error_code=selection_error_code, ) + if ( + selected_snapshot is not None + and selected_lease is not None + and sticky_seed_key is not None + and sticky_seed_kind is not None + and sticky_seed_account_id is None + ): + # Required-owner selection bypasses the thread row, but a + # first-ever process preference still has to land so later + # unpinned siblings inherit that exact owner. + try: + async with self._repo_factory() as repos: + await repos.sticky_sessions.insert_if_absent( + sticky_seed_key, + selected_snapshot.id, + sticky_seed_kind, + ) + except BaseException: + await self.release_account_lease(selected_lease) + selected_lease = None + raise else: sticky_outcome = await run_sticky_selection_path( self, @@ -776,7 +888,12 @@ async def load_selection_inputs() -> _SelectionInputs: sticky_source=sticky_source, legacy_sticky_key=legacy_sticky_key, legacy_existing_account_id=legacy_existing_account_id, + legacy_abandoned_account_id=legacy_abandoned_account_id, + sticky_seed_key=sticky_seed_key, + sticky_seed_kind=sticky_seed_kind, + sticky_seed_account_id=sticky_seed_account_id, spill_bare_session_on_account_cap=spill_bare_session_on_account_cap, + abandon_unavailable_legacy_owner=abandon_unavailable_legacy_owner, require_unambiguous_account=require_unambiguous_account, sticky_max_age_seconds=sticky_max_age_seconds, prefer_earlier_reset_accounts=prefer_earlier_reset_accounts, @@ -800,6 +917,7 @@ async def load_selection_inputs() -> _SelectionInputs: reload_inputs=load_selection_inputs, record_account_cap_rejection=_record_account_cap_rejection, allow_usage_exhaustion_error=allow_usage_exhaustion_error, + initial_sticky_owner_lookup=initial_sticky_owner_lookup, ), ) selection_inputs = sticky_outcome.selection_inputs @@ -1039,6 +1157,7 @@ async def _load_selection_inputs( if account_ids is not None: allowed_account_ids = set(account_ids) scoped_accounts = [account for account in scoped_accounts if account.id in allowed_account_ids] + sticky_mutation_authority_account_ids = frozenset(account.id for account in scoped_accounts) accounts = _selectable_accounts(scoped_accounts) pre_model_filter_accounts = accounts model_catalog_omitted_account_ids: frozenset[str] = frozenset() @@ -1080,6 +1199,7 @@ async def _load_selection_inputs( continuity_owner_candidates=[ _clone_account(account) for account in continuity_owner_candidates ], + sticky_mutation_authority_account_ids=sticky_mutation_authority_account_ids, quota_planner_settings=quota_planner_settings, runtime_accounts=[_clone_account(account) for account in all_accounts], ) @@ -1094,6 +1214,7 @@ async def _load_selection_inputs( latest_secondary={}, latest_monthly={}, continuity_owner_candidates=[], + sticky_mutation_authority_account_ids=sticky_mutation_authority_account_ids, quota_planner_settings=quota_planner_settings, runtime_accounts=[_clone_account(account) for account in all_accounts], ) @@ -1110,6 +1231,7 @@ async def _load_selection_inputs( continuity_owner_candidates=[ _clone_account(account) for account in continuity_owner_candidates ], + sticky_mutation_authority_account_ids=sticky_mutation_authority_account_ids, quota_planner_settings=quota_planner_settings, runtime_accounts=[_clone_account(account) for account in all_accounts], ) @@ -1123,6 +1245,7 @@ async def _load_selection_inputs( latest_secondary={}, latest_monthly={}, continuity_owner_candidates=[_clone_account(account) for account in continuity_owner_candidates], + sticky_mutation_authority_account_ids=sticky_mutation_authority_account_ids, quota_planner_settings=quota_planner_settings, runtime_accounts=[_clone_account(account) for account in all_accounts], error_message=( @@ -1156,6 +1279,7 @@ async def _load_selection_inputs( continuity_owner_candidates=[ _clone_account(account) for account in continuity_owner_candidates ], + sticky_mutation_authority_account_ids=sticky_mutation_authority_account_ids, quota_planner_settings=quota_planner_settings, runtime_accounts=[_clone_account(account) for account in all_accounts], error_message=additional_filter.error_message, @@ -1172,6 +1296,7 @@ async def _load_selection_inputs( latest_secondary={}, latest_monthly={}, continuity_owner_candidates=[_clone_account(account) for account in continuity_owner_candidates], + sticky_mutation_authority_account_ids=sticky_mutation_authority_account_ids, quota_planner_settings=quota_planner_settings, runtime_accounts=[_clone_account(account) for account in all_accounts], ) @@ -1232,6 +1357,7 @@ async def _load_selection_inputs( account_id: _clone_standard_usage_history(entry) for account_id, entry in latest_monthly.items() }, continuity_owner_candidates=[_clone_account(account) for account in continuity_owner_candidates], + sticky_mutation_authority_account_ids=sticky_mutation_authority_account_ids, quota_planner_settings=quota_planner_settings, runtime_accounts=[_clone_account(account) for account in all_accounts], ignore_standard_quota_account_ids=ignore_standard_quota_account_ids, @@ -1539,11 +1665,13 @@ async def _select_with_stickiness( sticky_repo: StickySessionsRepository | None, routing_costs_by_account_id: RoutingCostsByAccount | None = None, sticky_existing_account_id: str | None | object = _STICKY_EXISTING_UNSET, + initial_preferred_account_id: str | None = None, preserve_existing_mapping_on_fallback: bool = False, traffic_class: TrafficClass = TRAFFIC_CLASS_FOREGROUND, ignore_standard_quota: bool = False, allow_usage_exhaustion_error: bool = True, usage_exhaustion_states: Iterable[AccountState] | None = None, + sticky_refresh_skip_deadline: datetime | None = None, ) -> _StickySelectionOutcome: return await _run_select_with_stickiness( states=states, @@ -1562,11 +1690,13 @@ async def _select_with_stickiness( sticky_repo=sticky_repo, routing_costs_by_account_id=routing_costs_by_account_id, sticky_existing_account_id=sticky_existing_account_id, + initial_preferred_account_id=initial_preferred_account_id, preserve_existing_mapping_on_fallback=preserve_existing_mapping_on_fallback, traffic_class=traffic_class, ignore_standard_quota=ignore_standard_quota, allow_usage_exhaustion_error=allow_usage_exhaustion_error, usage_exhaustion_states=usage_exhaustion_states, + sticky_refresh_skip_deadline=sticky_refresh_skip_deadline, ) _persist_sticky_mutation = staticmethod(_persist_sticky_mutation) @@ -2934,6 +3064,11 @@ def _clone_selection_inputs(selection_inputs: SelectionInputs) -> SelectionInput if selection_inputs.continuity_owner_candidates is None else [_clone_account(account) for account in selection_inputs.continuity_owner_candidates] ), + sticky_mutation_authority_account_ids=( + None + if selection_inputs.sticky_mutation_authority_account_ids is None + else frozenset(selection_inputs.sticky_mutation_authority_account_ids) + ), quota_planner_settings=selection_inputs.quota_planner_settings, runtime_accounts=( None diff --git a/app/modules/proxy/replay_safety.py b/app/modules/proxy/replay_safety.py index e7d9db9967..fd44be4fbb 100644 --- a/app/modules/proxy/replay_safety.py +++ b/app/modules/proxy/replay_safety.py @@ -2,24 +2,14 @@ from __future__ import annotations -import json -import math -import re from collections import deque from collections.abc import Mapping from dataclasses import dataclass -from hashlib import sha256 -from typing import Literal, cast +from typing import cast from urllib.parse import urlsplit -from uuid import UUID from app.core.openai.requests import extract_input_file_ids from app.core.types import JsonValue -from app.modules.proxy.response_transition_manifest import ( - ResponseTransitionManifest, - match_response_transition_manifest_prefix, - response_transition_manifest_matches_context, -) _TOOL_CALL_TYPE_BY_OUTPUT_TYPE = { "function_call_output": "function_call", @@ -32,14 +22,10 @@ ) _INTERNAL_CHAT_MESSAGE_METADATA_FIELD = "internal_chat_message_metadata_passthrough" _ACCOUNT_NEUTRAL_INTERNAL_CHAT_MESSAGE_METADATA_FIELDS = frozenset({"turn_id"}) -_ACCOUNT_NEUTRAL_TOOL_TYPES = frozenset( - {"custom", "function", "namespace", "tool_search", "web_search", "web_search_preview"} -) +_ACCOUNT_NEUTRAL_TOOL_TYPES = frozenset({"custom", "function", "web_search", "web_search_preview"}) _ACCOUNT_NEUTRAL_TOOL_DECLARATION_FIELDS = { - "custom": frozenset({"defer_loading", "description", "format", "name", "type"}), - "function": frozenset({"defer_loading", "description", "name", "parameters", "strict", "type"}), - "namespace": frozenset({"description", "name", "tools", "type"}), - "tool_search": frozenset({"description", "execution", "parameters", "type"}), + "custom": frozenset({"description", "format", "name", "type"}), + "function": frozenset({"description", "name", "parameters", "strict", "type"}), "web_search": frozenset({"filters", "search_context_size", "type", "user_location"}), "web_search_preview": frozenset({"filters", "search_context_size", "type", "user_location"}), } @@ -48,35 +34,6 @@ _ACCOUNT_NEUTRAL_WEB_SEARCH_FILTER_FIELDS = frozenset({"allowed_domains"}) _ACCOUNT_NEUTRAL_WEB_SEARCH_LOCATION_FIELDS = frozenset({"city", "country", "region", "timezone", "type"}) _ACCOUNT_NEUTRAL_MESSAGE_ROLES = frozenset({"assistant", "developer", "system", "user"}) -_RESPONSE_OWNED_AGENT_MESSAGE_FIELDS = frozenset( - {"author", "content", "id", _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, "recipient", "type"} -) -_TRANSPORT_RESPONSE_OWNED_AGENT_MESSAGE_FIELDS = frozenset({"author", "content", "id", "recipient", "type"}) -_RESPONSE_OWNED_AGENT_MESSAGE_METADATA_FIELDS = frozenset({"create_time", "turn_id"}) -_RESPONSE_OWNED_USER_MESSAGE_FIELDS = frozenset( - {"content", "id", _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, "role", "type"} -) -_TRANSPORT_RESPONSE_OWNED_USER_MESSAGE_FIELDS = frozenset({"content", "id", "role", "type"}) -# Agent paths are produced by the collaboration runtime, whose root is -# literally ``/root`` and whose task-name segments are restricted to lowercase -# letters, digits, and underscores. This is replay authority, so accepting a -# merely path-shaped client string would be too broad. -_AGENT_PATH_PATTERN = re.compile(r"^/root(?:/[a-z0-9_]+)*$") -AbandonedPendingBoundaryRejectionReason = Literal[ - "stored_prefix_invalid", - "pending_call_manifest_missing", - "boundary_reasoning_shape_invalid", - "boundary_agent_message_shape_invalid", - "followup_missing", - "followup_shape_invalid", - "developer_message_shape_invalid", - "developer_message_sequence_invalid", - "pending_call_conflict", - "projection_failed", - "direct_call_prefix_state_invalid", - "projected_boundary_invalid", - "projected_followup_invalid", -] _ACCOUNT_NEUTRAL_INPUT_ITEM_TYPES = frozenset( { "additional_tools", @@ -131,17 +88,7 @@ {"call_id", "caller", "id", _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, "output", "status", "type"} ), "function_call": frozenset( - { - "arguments", - "call_id", - "caller", - "id", - _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, - "name", - "namespace", - "status", - "type", - } + {"arguments", "call_id", "caller", "id", _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, "name", "status", "type"} ), "function_call_output": frozenset( {"call_id", "caller", "id", _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, "output", "status", "type"} @@ -153,14 +100,10 @@ "delete_file": frozenset({"path", "type"}), "update_file": frozenset({"diff", "path", "type"}), } -_ACCOUNT_NEUTRAL_REASONING_CONFIG_FIELDS = frozenset({"context", "effort", "summary"}) +_ACCOUNT_NEUTRAL_REASONING_CONFIG_FIELDS = frozenset({"effort", "summary"}) _ACCOUNT_NEUTRAL_CLIENT_METADATA_FIELDS = frozenset( { "ws_request_header_x_openai_internal_codex_responses_lite", - "root_turn_id", - "session_id", - "thread_id", - "turn_id", "x-codex-installation-id", "x-codex-parent-thread-id", "x-codex-turn-metadata", @@ -168,38 +111,6 @@ "x-openai-subagent", } ) -# Codex deliberately omits the potentially large tool namespace inventory -# from the compatibility header while retaining it in the request body. Keep -# the header within conventional proxy limits, but permit a bounded body -# carrier that has already passed the request-size gate and the closed schema -# validation below. -_ACCOUNT_NEUTRAL_TURN_METADATA_DIRECT_MAX_BYTES = 16 * 1024 -_ACCOUNT_NEUTRAL_TURN_METADATA_BODY_MAX_BYTES = 1024 * 1024 -_ACCOUNT_NEUTRAL_WORKSPACE_KIND_MAX_BYTES = 128 -_ACCOUNT_NEUTRAL_TURN_METADATA_FIELDS = frozenset( - { - "agent_name", - "auto_review_enabled", - "forked_from_thread_id", - "installation_id", - "node_repl_auto_review_required", - "node_repl_disabled", - "request_kind", - "root_turn_id", - "sandbox", - "sandbox_mode", - "session_id", - "thread_id", - "thread_source", - "tool_namespaces_info", - "turn_id", - "turn_started_at_unix_ms", - "window_id", - "workspace_kind", - "workspaces", - } -) -_ACCOUNT_NEUTRAL_TURN_METADATA_LINEAGE_FIELDS = frozenset({"parent_thread_id", "parent_turn_id", "subagent_kind"}) _ACCOUNT_SCOPED_HOSTED_INPUT_TYPES = frozenset( { "code_interpreter_call", @@ -250,87 +161,17 @@ class AccountNeutralReplayProjection: """ -@dataclass(frozen=True, slots=True) -class AccountNeutralCodexTurnMetadataEvidence: - session_identity: str - task_identity: str - turn_identity: str - root_turn_identity: str | None - installation_identity: str | None - window_identity: str | None - workspace_kind: str | None - forked_from_thread_identity: str | None - shared_projection_fingerprint: str - - -@dataclass(frozen=True, slots=True) -class DirectCallLedgerSummary: - digest: str - unresolved_count: int - - -def responses_direct_call_ledger_summary( - input_items: list[JsonValue], -) -> DirectCallLedgerSummary | None: - """Hash the ordered direct-call lifecycle without retaining call content. - - The digest includes only call/output identity, type, and status. Invalid, - duplicate, orphaned, or type-mismatched entries are not a settlement - ledger and fail closed. - """ - - pending: dict[str, str] = {} - seen: set[str] = set() - ledger: list[dict[str, str | None]] = [] - for item in input_items: - if not isinstance(item, dict): - continue - item_type_value = item.get("type") - item_type = item_type_value if isinstance(item_type_value, str) else None - if item_type not in _TOOL_CALL_TYPES and item_type not in _TOOL_CALL_TYPE_BY_OUTPUT_TYPE: - continue - call_id = item.get("call_id") - status_value = item.get("status") - status = status_value if isinstance(status_value, str) else None - if not isinstance(call_id, str) or not call_id or status not in (None, "completed", "failed"): - return None - if item_type in _TOOL_CALL_TYPES: - if call_id in seen: - return None - seen.add(call_id) - pending[call_id] = item_type - ledger.append({"call_id": call_id, "kind": "call", "status": status, "type": item_type}) - continue - expected_call_type = _TOOL_CALL_TYPE_BY_OUTPUT_TYPE[item_type] - if pending.get(call_id) != expected_call_type: - return None - pending.pop(call_id) - ledger.append({"call_id": call_id, "kind": "output", "status": status, "type": item_type}) - canonical = json.dumps(ledger, ensure_ascii=True, separators=(",", ":"), sort_keys=True) - return DirectCallLedgerSummary( - digest=sha256(canonical.encode("utf-8")).hexdigest(), - unresolved_count=len(pending), - ) - - def project_responses_input_for_account_neutral_fresh_replay( input_items: list[JsonValue], *, stored_count: int, preserve_developer_message_ids: bool = False, - preserve_response_owned_agent_message_ids: bool = False, - omit_response_owned_agent_messages_from_stored_prefix: bool = False, - project_response_owned_developer_messages_from_stored_prefix: bool = False, - project_response_owned_developer_messages_from_suffix: bool = False, ) -> AccountNeutralReplayProjection | None: """Remove known response-owned bookkeeping after durable prefix proof. - The two ``preserve_*_ids`` options are classification-only evidence for - response-owned items. A projection created with either option must not be - serialized as an account-neutral replay payload. The stored-prefix - developer option is reserved for the separately fingerprint-bound - abandoned-pending recovery path; it strips response ownership without - changing developer content or admitting a new developer item. + ``preserve_developer_message_ids`` is classification-only evidence for + inline Responses-Lite messages. A projection created with that option must + not be serialized as an account-neutral replay payload. """ if stored_count <= 0 or stored_count > len(input_items): @@ -341,45 +182,10 @@ def project_responses_input_for_account_neutral_fresh_replay( canonical_lite_developer_index: int | None = None prefix_begins_with_lite_tool_bundle = stored_count >= 2 and _is_canonical_lite_tool_bundle(input_items[0]) for index, item in enumerate(input_items): - if ( - omit_response_owned_agent_messages_from_stored_prefix - and index < stored_count - and isinstance(item, dict) - and item.get("type") == "agent_message" - and _is_retained_agent_message(item) - ): - projected_item = None - elif ( - project_response_owned_developer_messages_from_stored_prefix - and index < stored_count - and isinstance(item, dict) - and _is_response_owned_developer_message(item) - ): - projected_item = dict(item) - projected_item.pop("id") - metadata = projected_item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD) - if isinstance(metadata, dict): - projected_item[_INTERNAL_CHAT_MESSAGE_METADATA_FIELD] = {"turn_id": metadata["turn_id"]} - elif ( - project_response_owned_developer_messages_from_suffix - and index >= stored_count - and isinstance(item, dict) - and _is_response_owned_developer_message(item) - ): - projected_item = dict(item) - projected_item.pop("id") - metadata = projected_item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD) - if isinstance(metadata, dict): - projected_item[_INTERNAL_CHAT_MESSAGE_METADATA_FIELD] = {"turn_id": metadata["turn_id"]} - else: - projected_item = _project_account_neutral_replay_item( - item, - preserve_developer_message_ids=preserve_developer_message_ids, - preserve_response_owned_agent_message_ids=( - preserve_response_owned_agent_message_ids - and (not omit_response_owned_agent_messages_from_stored_prefix or index >= stored_count) - ), - ) + projected_item = _project_account_neutral_replay_item( + item, + preserve_developer_message_ids=preserve_developer_message_ids, + ) if projected_item is not None: projected_items.append(projected_item) # The canonical position is the bundle's original immediate @@ -403,106 +209,6 @@ def project_responses_input_for_account_neutral_fresh_replay( ) -def project_responses_input_for_abandoned_pending_fresh_replay( - input_items: list[JsonValue], - *, - stored_count: int, - pending_tool_calls: Mapping[str, str], -) -> AccountNeutralReplayProjection | None: - """Project one exact stale-anchor recovery after an abandoned agent call. - - Codex may compact a retained input window so that it begins with a tool - output whose matching response-owned call is outside the retained window. - Such an orphan is valid only while the old ``previous_response_id`` still - supplies that call; forwarding it on an unanchored recovery request is - invalid. For the narrowly sealed abandoned-pending recovery path, omit a - leading run of those clipped outputs only when all of the following are - physically proven by the caller's exact durable-prefix binding: - - * every omitted item is a canonical, account-neutral tool output; - * none names the abandoned pending call or reuses an id later in context; - * a later retained assistant output closes over the clipped history; and - * the remaining stored prefix has a complete direct call/output manifest. - - No call is synthesized or executed. Non-leading or otherwise ambiguous - orphan outputs remain fail closed. - """ - - projection = project_responses_input_for_account_neutral_fresh_replay( - input_items, - stored_count=stored_count, - preserve_developer_message_ids=True, - preserve_response_owned_agent_message_ids=True, - omit_response_owned_agent_messages_from_stored_prefix=True, - project_response_owned_developer_messages_from_stored_prefix=True, - project_response_owned_developer_messages_from_suffix=True, - ) - if projection is None: - return None - - prefix = projection.input_items[: projection.stored_prefix_count] - leading_output_count = 0 - leading_call_ids: set[str] = set() - for item in prefix: - if not isinstance(item, dict): - break - item_type_value = item.get("type") - item_type = item_type_value if isinstance(item_type_value, str) else None - call_type = _TOOL_CALL_TYPE_BY_OUTPUT_TYPE.get(item_type or "") - if call_type is None: - break - call_id = item.get("call_id") - if ( - not isinstance(call_id, str) - or not call_id - or call_id in leading_call_ids - or call_id in pending_tool_calls - or item.get("status") not in (None, "completed", "failed") - or not _internal_chat_message_metadata_is_account_neutral(item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD)) - or not _input_item_has_only_known_fields(item, item_type) - or not _caller_is_self_contained(item) - or not _tool_output_is_self_contained(item_type or "", item) - ): - return None - leading_call_ids.add(call_id) - leading_output_count += 1 - - if leading_output_count == 0: - return projection - - retained_prefix = prefix[leading_output_count:] - if not retained_prefix or not any( - isinstance(item, dict) and _is_retained_response_message(item) for item in retained_prefix - ): - return None - if any( - isinstance(item, dict) and item.get("call_id") in leading_call_ids - for item in projection.input_items[leading_output_count:] - ): - return None - - projected_canonical_index = ( - None - if projection.canonical_lite_developer_index is None - else projection.canonical_lite_developer_index - leading_output_count - ) - if ( - _direct_tool_call_prefix_state( - retained_prefix, - allow_exact_stored_developer_items=True, - canonical_lite_developer_index=projected_canonical_index, - ) - is None - ): - return None - - return AccountNeutralReplayProjection( - input_items=[*retained_prefix, *projection.input_items[projection.stored_prefix_count :]], - stored_prefix_count=len(retained_prefix), - canonical_lite_developer_index=projected_canonical_index, - ) - - def _is_canonical_lite_tool_bundle(item: JsonValue) -> bool: return ( isinstance(item, dict) @@ -521,7 +227,6 @@ def _project_account_neutral_replay_item( item: JsonValue, *, preserve_developer_message_ids: bool, - preserve_response_owned_agent_message_ids: bool, ) -> JsonValue | None: if not isinstance(item, dict): return item @@ -533,15 +238,6 @@ def _project_account_neutral_replay_item( # additional_tools bundle is a distinct Responses-Lite input item, # not an inline developer message. return item - if preserve_response_owned_agent_message_ids and item_type == "agent_message": - return item - if _is_response_owned_user_message(item): - projected_item = dict(item) - projected_item.pop("id") - metadata = projected_item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD) - if isinstance(metadata, dict): - projected_item[_INTERNAL_CHAT_MESSAGE_METADATA_FIELD] = {"turn_id": metadata["turn_id"]} - return projected_item if item_type is not None and not isinstance(item_type, str): return item if item_type == "reasoning" or ( @@ -553,19 +249,6 @@ def _project_account_neutral_replay_item( return item projected_item = dict(item) projected_item.pop("id") - metadata = projected_item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD) - if ( - isinstance(metadata, dict) - and set(metadata) == _RESPONSE_OWNED_AGENT_MESSAGE_METADATA_FIELDS - and _is_nonblank_string(metadata.get("turn_id")) - and _is_finite_nonnegative_number(metadata.get("create_time")) - ): - # Codex persists response-owned calls, outputs, and assistant messages - # with the same creation timestamp bookkeeping as user messages. The - # timestamp belongs to the old response and is not replay authority; - # retain only the request-scoped turn id after the response-owned id - # has been removed. - projected_item[_INTERNAL_CHAT_MESSAGE_METADATA_FIELD] = {"turn_id": metadata["turn_id"]} return projected_item @@ -614,154 +297,6 @@ def responses_input_items_are_self_contained_fresh_replay(input_items: list[Json return all(not call_ids for call_ids in unsettled_call_ids_by_type.values()) -def responses_input_items_are_self_contained_rowless_replay( - original_items: list[JsonValue], - projected_items: list[JsonValue], -) -> bool: - """Admit only canonical, settled Codex agent deliveries for rowless replay.""" - - pending_calls: set[str] = set() - agent_indexes: list[int] = [] - for index, item in enumerate(original_items): - if not isinstance(item, dict): - return False - item_type = item.get("type") - call_id = item.get("call_id") - if item_type in _TOOL_CALL_TYPES and isinstance(call_id, str): - pending_calls.add(call_id) - elif item_type in _TOOL_CALL_TYPE_BY_OUTPUT_TYPE and isinstance(call_id, str): - pending_calls.discard(call_id) - if item_type != "agent_message": - continue - normalized_agent = _normalized_rowless_agent_message(item) - if pending_calls or normalized_agent is None or not _is_retained_agent_message(normalized_agent): - return False - agent_indexes.append(index) - if not any(isinstance(later, dict) and later.get("role") == "user" for later in original_items[index + 1 :]): - return False - if not agent_indexes: - return responses_input_items_are_self_contained_fresh_replay(projected_items) - - projected_without_agents: list[JsonValue] = [] - for item in projected_items: - if not isinstance(item, dict) or item.get("type") != "agent_message": - projected_without_agents.append(item) - continue - metadata = item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD) - content = item.get("content") - if ( - set(item) - not in { - frozenset({"author", "content", "recipient", "type"}), - frozenset( - { - "author", - "content", - _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, - "recipient", - "type", - } - ), - } - or not isinstance(item.get("author"), str) - or not _AGENT_PATH_PATTERN.fullmatch(cast(str, item["author"])) - or not isinstance(item.get("recipient"), str) - or not _AGENT_PATH_PATTERN.fullmatch(cast(str, item["recipient"])) - or item["author"] == item["recipient"] - or not _internal_chat_message_metadata_is_account_neutral(metadata) - or not isinstance(content, list) - or len(content) != 1 - or not isinstance(content[0], dict) - or content[0].get("type") != "input_text" - or not _input_content_part_is_self_contained( - cast(dict[str, JsonValue], content[0]), - allow_output=False, - ) - ): - return False - return responses_input_items_are_self_contained_fresh_replay(projected_without_agents) - - -def normalize_responses_input_for_rowless_replay( - projected_items: list[JsonValue], -) -> list[JsonValue] | None: - """Drop only canonical, semantics-free response transport artifacts. - - Codex can persist an empty ``input_text`` tail in a direct tool output and - an opaque ``encrypted_content`` sibling beside the delivered text of an - inter-agent message. Neither item carries replayable conversation - semantics. Keep the original request fingerprint unchanged, but remove - those exact shapes from the separately fingerprinted rowless projection. - Any drift in fields, ordering, multiplicity, or non-empty text remains - fail closed. - """ - - normalized_items: list[JsonValue] = [] - for item in projected_items: - if not isinstance(item, dict): - normalized_items.append(item) - continue - - if item.get("type") == "agent_message": - normalized = _normalized_rowless_agent_message(item) - if normalized is None: - return None - normalized_items.append(normalized) - continue - - item_type = item.get("type") - if item_type in _TOOL_CALL_TYPE_BY_OUTPUT_TYPE and isinstance(item.get("output"), list): - output = cast(list[JsonValue], item["output"]) - filtered_output = [part for part in output if not _is_exact_empty_input_text_part(part)] - if len(filtered_output) != len(output): - if not filtered_output: - return None - normalized_item = dict(item) - normalized_item["output"] = filtered_output - normalized_items.append(normalized_item) - continue - - normalized_items.append(item) - return normalized_items - - -def _is_exact_empty_input_text_part(part: JsonValue) -> bool: - return ( - isinstance(part, dict) - and set(part) == {"text", "type"} - and part.get("type") == "input_text" - and part.get("text") == "" - ) - - -def _normalized_rowless_agent_message(item: Mapping[str, JsonValue]) -> dict[str, JsonValue] | None: - content = item.get("content") - if not isinstance(content, list): - return None - input_parts = [ - part - for part in content - if isinstance(part, dict) - and part.get("type") == "input_text" - and _input_content_part_is_self_contained(part, allow_output=False) - ] - encrypted_parts = [ - part - for part in content - if isinstance(part, dict) - and set(part) == {"encrypted_content", "type"} - and part.get("type") == "encrypted_content" - and _is_nonblank_string(part.get("encrypted_content")) - ] - if len(input_parts) != 1 or len(encrypted_parts) > 1 or len(input_parts) + len(encrypted_parts) != len(content): - return None - if encrypted_parts and content != [input_parts[0], encrypted_parts[0]]: - return None - normalized = dict(item) - normalized["content"] = [input_parts[0]] - return normalized - - def _internal_chat_message_metadata_is_account_neutral(value: JsonValue | None) -> bool: if value is None: return True @@ -777,33 +312,15 @@ def responses_input_suffix_retains_prior_output( *, stored_count: int, canonical_lite_developer_index: int | None = None, - exact_stored_prefix_without_pending_manifest: bool = False, - allow_response_owned_agent_message: bool = True, - allow_empty_stored_prefix: bool = False, ) -> bool: """Prove that a stored input prefix is followed by prior output and new input.""" - if stored_count < 0 or (stored_count == 0 and not allow_empty_stored_prefix) or len(input_items) <= stored_count: + if stored_count <= 0 or len(input_items) <= stored_count: return False - stored_prefix = input_items[:stored_count] - if exact_stored_prefix_without_pending_manifest: - # A store-context proof binds this prefix byte-for-byte to the input - # already completed by the same live/durable session and separately - # identifies the exact historical input boundary. It therefore need - # not reinterpret valid account-neutral developer items inside that - # sealed prefix as new cross-account authority. Still parse every - # direct call/output pair so an output crossing the stored boundary is - # retained and appended call-id reuse remains fail-closed. - prefix_state = _direct_tool_call_prefix_state( - stored_prefix, - allow_exact_stored_developer_items=True, - canonical_lite_developer_index=canonical_lite_developer_index, - ) - else: - prefix_state = _direct_tool_call_prefix_state( - stored_prefix, - canonical_lite_developer_index=canonical_lite_developer_index, - ) + prefix_state = _direct_tool_call_prefix_state( + input_items[:stored_count], + canonical_lite_developer_index=canonical_lite_developer_index, + ) if prefix_state is None: return False pending_suffix_calls, seen_suffix_call_ids = prefix_state @@ -857,21 +374,6 @@ def responses_input_suffix_retains_prior_output( fresh_followup_count = 0 fresh_followup_is_user_message = False continue - if item_type == "agent_message": - if ( - not allow_response_owned_agent_message - or pending_suffix_calls - or retained_output_seen - or fresh_followup_seen - or not _is_retained_agent_message(item) - ): - return False - retained_output_seen = True - # An inter-agent delivery closes the prior task but is not an - # assistant final answer. Keep the stricter developer-followup - # rule while still permitting one or more later user messages. - retained_output_is_final_answer = False - continue if _is_fresh_followup_input(item): if not retained_output_seen or pending_suffix_calls: return False @@ -894,29 +396,6 @@ def responses_input_suffix_retains_prior_output( return retained_output_seen and fresh_followup_seen and not pending_suffix_calls -def responses_input_retains_prior_output_and_fresh_followup( - input_items: list[JsonValue], -) -> bool: - """Prove a full resend retains completed output before its new user turn.""" - - for index in range(len(input_items) - 2, -1, -1): - item = input_items[index] - if not isinstance(item, dict): - continue - if not ( - (item.get("type") in (None, "message") and item.get("role") == "assistant") - or item.get("type") == "agent_message" - ): - continue - return responses_input_suffix_retains_prior_output( - input_items, - stored_count=index, - exact_stored_prefix_without_pending_manifest=True, - allow_empty_stored_prefix=index == 0, - ) - return False - - def responses_input_suffix_matches_pending_tool_calls( input_items: list[JsonValue], *, @@ -924,14 +403,7 @@ def responses_input_suffix_matches_pending_tool_calls( pending_tool_calls: Mapping[str, str], canonical_lite_developer_index: int | None = None, ) -> bool: - """Prove the suffix exactly settles the durable prior-response call manifest. - - A completed call/output manifest is the physical client-side settlement - proof. Codex can retain bounded later user/inter-agent inputs after that - settlement in the same complete-context resend. Those later inputs do - not weaken the proof, but another tool loop does: the latter could belong - to a different response and must never stand in for the durable manifest. - """ + """Prove the suffix exactly settles the durable prior-response call manifest.""" if stored_count <= 0 or len(input_items) <= stored_count or not pending_tool_calls: return False @@ -943,390 +415,39 @@ def responses_input_suffix_matches_pending_tool_calls( if prefix_state is None or prefix_state[0] or prefix_state[1] & pending_tool_calls.keys(): return False suffix = input_items[stored_count:] - settlement_end = _exact_pending_tool_call_settlement_prefix_length( - suffix, - pending_tool_calls=pending_tool_calls, - ) - if settlement_end is None: - return False - followups = suffix[settlement_end:] - if not followups: - return True - if any(isinstance(item, dict) and item.get("role") == "developer" for item in suffix[:settlement_end]): - # The historical one-call developer interleave exception is sealed to - # that exact three-item window. It is not authority for accepting a - # later turn boundary or user follow-up. - return False - first = followups[0] - if isinstance(first, dict) and first.get("type") == "agent_message": - if not _is_retained_agent_message(first): - return False - followups = followups[1:] - return _abandoned_pending_followup_sequence_is_bounded( - followups, - allow_response_owned_messages=False, - ) - - -def responses_input_suffix_matches_transition_manifest( - input_items: list[JsonValue], - *, - stored_count: int, - response_id: str, - pending_tool_calls: Mapping[str, str], - transition_manifest: ResponseTransitionManifest, -) -> bool: - """Prove one gateway-recorded output transition and its fresh retry turns.""" - - if not response_transition_manifest_matches_context( - transition_manifest, - response_id=response_id, - pending_tool_calls=pending_tool_calls, + if ( + len(suffix) == 3 + and isinstance(suffix[1], dict) + and _fresh_developer_message_is_transparent(suffix[1]) + and _fresh_developer_interleave_is_bounded(suffix, index=1) + ): + suffix = [suffix[0], suffix[2]] + if not all( + isinstance(item, dict) + and isinstance(item.get("type"), str) + and item.get("type") in (_TOOL_CALL_TYPES | _TOOL_CALL_TYPE_BY_OUTPUT_TYPE.keys()) + for item in suffix ): return False - manifest_end = match_response_transition_manifest_prefix( - input_items, - stored_count=stored_count, - manifest=transition_manifest, - ) - if manifest_end is None: - return False - - unsettled_calls = dict(pending_tool_calls) - seen_item_ids: set[str] = set() - for item in input_items[stored_count:manifest_end]: - if not isinstance(item, dict): - return False - item_id = item.get("id") - if item_id is None: - continue - if not isinstance(item_id, str) or not item_id or item_id in seen_item_ids: - return False - seen_item_ids.add(item_id) - index = manifest_end - while index < len(input_items): - item = input_items[index] - if not isinstance(item, dict): - return False - item_type_value = item.get("type") - item_type = item_type_value if isinstance(item_type_value, str) else "" - call_type = _TOOL_CALL_TYPE_BY_OUTPUT_TYPE.get(item_type) - if call_type is None: - break - call_id = item.get("call_id") - if ( - not isinstance(call_id, str) - or unsettled_calls.get(call_id) != call_type - or not _input_item_has_only_known_fields(item, item_type) - or not _caller_is_self_contained(item) - or not _response_owned_tool_metadata_is_account_neutral(item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD)) - or not _tool_output_is_self_contained(item_type, item) - ): - return False - item_id = item.get("id") - if item_id is not None: - if not isinstance(item_id, str) or not item_id or item_id in seen_item_ids: - return False - seen_item_ids.add(item_id) - del unsettled_calls[call_id] - index += 1 - if unsettled_calls: + if not responses_input_items_are_self_contained_fresh_replay(suffix): return False - - retry_items = input_items[index:] - if ( - len(retry_items) == 1 - and isinstance(retry_items[0], dict) - and retry_items[0].get("role") == "user" - and retry_items[0].get("id") in (None, "") - and responses_input_items_are_self_contained_fresh_replay(retry_items) - ): - # Non-Codex Responses clients do not carry turn metadata. Admit only - # one self-contained user follow-up; multi-item retries require the - # response-owned turn identities validated below. - return True - - retry_turn_roles: dict[str, set[str]] = {} - user_count = 0 - for item in retry_items: - if not isinstance(item, dict): - return False - if _is_response_owned_user_message(item): - role = "user" - user_count += 1 - elif _is_response_owned_developer_message(item): - role = "developer" + suffix_calls: dict[str, str] = {} + suffix_outputs: dict[str, str] = {} + for item in cast(list[dict[str, JsonValue]], suffix): + item_type = cast(str, item["type"]) + call_id = cast(str, item["call_id"]) + if item_type in _TOOL_CALL_TYPES: + suffix_calls[call_id] = item_type else: - return False - message_id = item.get("id") - metadata = item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD) - turn_id = metadata.get("turn_id") if isinstance(metadata, dict) else None - if not isinstance(message_id, str) or message_id in seen_item_ids or not _is_uuid(turn_id): - return False - seen_item_ids.add(message_id) - roles = retry_turn_roles.setdefault(cast(str, turn_id), set()) - if role in roles: - return False - roles.add(role) - return user_count > 0 and all("user" in roles for roles in retry_turn_roles.values()) - - -def _exact_pending_tool_call_settlement_prefix_length( - input_items: list[JsonValue], - *, - pending_tool_calls: Mapping[str, str], -) -> int | None: - """Return the shortest prefix that exactly settles one durable manifest.""" - + suffix_outputs[call_id] = _TOOL_CALL_TYPE_BY_OUTPUT_TYPE[item_type] expected = dict(pending_tool_calls) - for end in range(1, len(input_items) + 1): - candidate = input_items[:end] - normalized = candidate - if ( - len(candidate) == 3 - and isinstance(candidate[1], dict) - and _fresh_developer_message_is_transparent(candidate[1]) - and _fresh_developer_interleave_is_bounded(candidate, index=1) - ): - normalized = [candidate[0], candidate[2]] - if not all( - isinstance(item, dict) - and isinstance(item.get("type"), str) - and item.get("type") in (_TOOL_CALL_TYPES | _TOOL_CALL_TYPE_BY_OUTPUT_TYPE.keys()) - for item in normalized - ): - # A bounded call/developer/output window becomes recognizable - # only when its output arrives. Keep scanning possible prefixes; - # an unrelated non-settlement item will remain in every later - # candidate and therefore can never satisfy this predicate. - continue - if not responses_input_items_are_self_contained_fresh_replay(normalized): - continue - suffix_calls: dict[str, str] = {} - suffix_outputs: dict[str, str] = {} - for item in cast(list[dict[str, JsonValue]], normalized): - item_type = cast(str, item["type"]) - call_id = cast(str, item["call_id"]) - if item_type in _TOOL_CALL_TYPES: - suffix_calls[call_id] = item_type - else: - suffix_outputs[call_id] = _TOOL_CALL_TYPE_BY_OUTPUT_TYPE[item_type] - if suffix_calls == expected and suffix_outputs == expected: - return end - return None - - -def responses_input_suffix_proves_abandoned_pending_agent_boundary( - input_items: list[JsonValue], - *, - stored_count: int, - pending_tool_calls: Mapping[str, str], -) -> bool: - """Prove a later inter-agent boundary excludes an undelivered pending call. - - This is deliberately narrower than ordinary fresh-replay eligibility. It - exists for one stale-anchor recovery case: the durable response manifest - records a pending client-side tool call, the exact client resend contains - none of that call's ids, and a canonical response-owned ``agent_message`` - followed by fresh user input proves that the client advanced without ever - accepting or executing the pending call. The caller must additionally - prove that upstream rejected the exact response anchor before emitting any - response event; this predicate alone never authorizes proactive replay. - """ - - return ( - abandoned_pending_agent_boundary_rejection_reason( - input_items, - stored_count=stored_count, - pending_tool_calls=pending_tool_calls, - ) - is None - ) - - -def abandoned_pending_agent_boundary_rejection_reason( - input_items: list[JsonValue], - *, - stored_count: int, - pending_tool_calls: Mapping[str, str], -) -> AbandonedPendingBoundaryRejectionReason | None: - """Return the first content-free failure branch for boundary proof.""" - - if stored_count <= 0 or len(input_items) <= stored_count: - return "stored_prefix_invalid" - if not pending_tool_calls: - return "pending_call_manifest_missing" - raw_suffix = input_items[stored_count:] - boundary_index = 0 - while boundary_index < len(raw_suffix) and isinstance(raw_suffix[boundary_index], dict): - item = cast(dict[str, JsonValue], raw_suffix[boundary_index]) - if item.get("type") != "reasoning": - break - if not _is_response_owned_reasoning_boundary_item(item): - return "boundary_reasoning_shape_invalid" - boundary_index += 1 - if boundary_index >= len(raw_suffix): - return "boundary_agent_message_shape_invalid" - boundary = raw_suffix[boundary_index] - if not isinstance(boundary, dict) or not _is_retained_agent_message(boundary): - return "boundary_agent_message_shape_invalid" - followups = raw_suffix[boundary_index + 1 :] - followup_rejection = _abandoned_pending_followup_sequence_rejection_reason( - followups, - allow_response_owned_messages=True, - ) - if followup_rejection is not None: - return followup_rejection - for item in input_items: - if isinstance(item, dict) and item.get("call_id") in pending_tool_calls: - return "pending_call_conflict" - replay_projection = project_responses_input_for_abandoned_pending_fresh_replay( - input_items, - stored_count=stored_count, - pending_tool_calls=pending_tool_calls, - ) - if replay_projection is None: - return "projection_failed" - prefix_state = _direct_tool_call_prefix_state( - replay_projection.input_items[: replay_projection.stored_prefix_count], - allow_exact_stored_developer_items=True, - canonical_lite_developer_index=replay_projection.canonical_lite_developer_index, - ) - if prefix_state is None or prefix_state[0]: - return "direct_call_prefix_state_invalid" - if prefix_state[1] & pending_tool_calls.keys(): - return "pending_call_conflict" - suffix = replay_projection.input_items[replay_projection.stored_prefix_count :] - if len(suffix) < 2: - return "followup_missing" - first = suffix[0] - if not isinstance(first, dict) or first.get("type") != "agent_message" or not _is_retained_agent_message(first): - return "projected_boundary_invalid" - if not _abandoned_pending_followup_sequence_is_bounded( - suffix[1:], - allow_response_owned_messages=False, - ): - return "projected_followup_invalid" - return None - - -def _abandoned_pending_followup_sequence_is_bounded( - input_items: list[JsonValue], - *, - allow_response_owned_messages: bool, -) -> bool: - """Require one bounded developer refresh between proven user followups.""" - - return ( - _abandoned_pending_followup_sequence_rejection_reason( - input_items, - allow_response_owned_messages=allow_response_owned_messages, - ) - is None - ) - - -def _abandoned_pending_followup_sequence_rejection_reason( - input_items: list[JsonValue], - *, - allow_response_owned_messages: bool, -) -> AbandonedPendingBoundaryRejectionReason | None: - """Classify one bounded follow-up sequence without exposing content.""" - - if not input_items: - return "followup_missing" - user_seen = False - developer_seen = False - user_after_developer_seen = False - for item in input_items: - if not isinstance(item, dict): - return "followup_shape_invalid" - is_user = _is_fresh_followup_input(item) or ( - allow_response_owned_messages and _is_response_owned_user_message(item) - ) - item_type_value = item.get("type") - item_type = item_type_value if isinstance(item_type_value, str) else None - is_developer = ( - _is_response_owned_developer_message(item) - if allow_response_owned_messages - else _historical_pending_developer_message_is_transparent(item, item_type=item_type) - ) - if is_user: - user_seen = True - if developer_seen: - user_after_developer_seen = True - continue - if is_developer: - if developer_seen or not user_seen: - return "developer_message_sequence_invalid" - developer_seen = True - continue - if item.get("role") == "developer": - return "developer_message_shape_invalid" - return "followup_shape_invalid" - if not user_seen: - return "followup_missing" - if developer_seen and not user_after_developer_seen: - return "developer_message_sequence_invalid" - return None - - -def _is_response_owned_reasoning_boundary_item(item: Mapping[str, JsonValue]) -> bool: - """Recognize the exact Codex response bookkeeping allowed before a boundary.""" - - allowed_fields = { - "content", - "encrypted_content", - "id", - _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, - "status", - "summary", - "type", - } - metadata = item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD) - summary = item.get("summary") - status = item.get("status") - return ( - set(item) <= allowed_fields - and {"encrypted_content", "id", "summary", "type"} <= set(item) - and item.get("type") == "reasoning" - and isinstance(item.get("id"), str) - and cast(str, item["id"]).startswith("rs_") - and _is_nonblank_string(item.get("encrypted_content")) - and isinstance(summary, list) - and all( - isinstance(part, dict) - and set(part) == {"text", "type"} - and part.get("type") == "summary_text" - and isinstance(part.get("text"), str) - for part in summary - ) - and ( - ( - metadata is None - # Codex sends ``content: null`` on the HTTP transport. The - # ResponsesRequest model intentionally drops that null field - # before the recovery predicate sees the item, so both exact - # representations describe the same response-owned boundary. - # No non-null content or additional field is admitted. - and ("content" not in item or item.get("content") is None) - ) - or ( - isinstance(metadata, dict) - and "content" not in item - and set(metadata) == _ACCOUNT_NEUTRAL_INTERNAL_CHAT_MESSAGE_METADATA_FIELDS - and _is_uuid(metadata.get("turn_id")) - ) - ) - and status in (None, "completed") - ) + return suffix_calls == expected and suffix_outputs == expected def _direct_tool_call_prefix_state( input_items: list[JsonValue], *, allow_historical_developer_interleave: bool = False, - allow_exact_stored_developer_items: bool = False, canonical_lite_developer_index: int | None = None, ) -> tuple[deque[tuple[str, str]], set[str]] | None: pending_calls: deque[tuple[str, str]] = deque() @@ -1355,8 +476,6 @@ def _direct_tool_call_prefix_state( ) if developer_message_is_transparent and occupies_canonical_lite_position: continue - if developer_message_is_transparent and allow_exact_stored_developer_items: - continue historical_interleave_is_bounded = ( allow_historical_developer_interleave and len(pending_calls) == 1 @@ -1491,175 +610,13 @@ def _is_retained_response_message(item: Mapping[str, JsonValue]) -> bool: return _message_has_valid_account_neutral_content(item) -def _response_owned_tool_metadata_is_account_neutral(value: JsonValue | None) -> bool: - return _internal_chat_message_metadata_is_account_neutral(value) or ( - isinstance(value, dict) - and set(value) == _RESPONSE_OWNED_AGENT_MESSAGE_METADATA_FIELDS - and _is_nonblank_string(value.get("turn_id")) - and _is_finite_nonnegative_number(value.get("create_time")) - ) - - -def _is_retained_agent_message(item: Mapping[str, JsonValue]) -> bool: - """Validate the exact response-owned Codex inter-agent delivery shape.""" - - item_id = item.get("id") - author = item.get("author") - recipient = item.get("recipient") - metadata = item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD) - content = item.get("content") - if ( - set(item) - not in { - _RESPONSE_OWNED_AGENT_MESSAGE_FIELDS, - _TRANSPORT_RESPONSE_OWNED_AGENT_MESSAGE_FIELDS, - } - or item.get("type") != "agent_message" - or not isinstance(item_id, str) - or not item_id.startswith("amsg_") - or not _is_uuid(item_id.removeprefix("amsg_")) - or not isinstance(author, str) - or not _AGENT_PATH_PATTERN.fullmatch(author) - or not isinstance(recipient, str) - or not _AGENT_PATH_PATTERN.fullmatch(recipient) - or author == recipient - or not ( - metadata is None - or ( - isinstance(metadata, dict) - and set(metadata) == _RESPONSE_OWNED_AGENT_MESSAGE_METADATA_FIELDS - and _is_uuid(metadata.get("turn_id")) - and _is_finite_nonnegative_number(metadata.get("create_time")) - ) - ) - or not isinstance(content, list) - or len(content) != 1 - or not isinstance(content[0], dict) - or content[0].get("type") != "input_text" - ): - return False - return _input_content_part_is_self_contained( - cast(dict[str, JsonValue], content[0]), - allow_output=False, - ) - - -def _is_response_owned_user_message(item: Mapping[str, JsonValue]) -> bool: - """Validate Codex's persisted user-message bookkeeping before stripping it.""" - - item_id = item.get("id") - metadata = item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD) - content = item.get("content") - if ( - set(item) - not in { - _RESPONSE_OWNED_USER_MESSAGE_FIELDS, - _TRANSPORT_RESPONSE_OWNED_USER_MESSAGE_FIELDS, - } - or item.get("type") != "message" - or item.get("role") != "user" - or not isinstance(item_id, str) - or not item_id.startswith("msg_") - or not _is_uuid(item_id.removeprefix("msg_")) - or not ( - metadata is None - or ( - isinstance(metadata, dict) - and set(metadata) == _RESPONSE_OWNED_AGENT_MESSAGE_METADATA_FIELDS - and _is_uuid(metadata.get("turn_id")) - and _is_finite_nonnegative_number(metadata.get("create_time")) - ) - ) - or not isinstance(content, list) - or len(content) != 1 - or not isinstance(content[0], dict) - or content[0].get("type") != "input_text" - ): - return False - return _input_content_part_is_self_contained( - cast(dict[str, JsonValue], content[0]), - allow_output=False, - ) - - -def _is_response_owned_developer_message(item: Mapping[str, JsonValue]) -> bool: - """Validate a persisted developer message inside an exact stored prefix.""" - - item_id = item.get("id") - metadata = item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD) - content = item.get("content") - return ( - set(item) - in { - _RESPONSE_OWNED_USER_MESSAGE_FIELDS, - _TRANSPORT_RESPONSE_OWNED_USER_MESSAGE_FIELDS, - } - and item.get("type") == "message" - and item.get("role") == "developer" - and isinstance(item_id, str) - and item_id.startswith("msg_") - and _is_uuid(item_id.removeprefix("msg_")) - and ( - metadata is None - or ( - isinstance(metadata, dict) - and set(metadata) == _ACCOUNT_NEUTRAL_INTERNAL_CHAT_MESSAGE_METADATA_FIELDS - and _is_uuid(metadata.get("turn_id")) - ) - or ( - isinstance(metadata, dict) - and set(metadata) == _RESPONSE_OWNED_AGENT_MESSAGE_METADATA_FIELDS - and _is_uuid(metadata.get("turn_id")) - and _is_finite_nonnegative_number(metadata.get("create_time")) - ) - ) - and isinstance(content, list) - and bool(content) - and all( - isinstance(part, dict) - and part.get("type") == "input_text" - and _input_content_part_is_self_contained(part, allow_output=False) - for part in content - ) - ) - - -def _is_uuid(value: JsonValue | None) -> bool: - if not isinstance(value, str) or not value: - return False - try: - return str(UUID(value)) == value.lower() - except ValueError: - return False - - -def _is_finite_nonnegative_number(value: JsonValue | None) -> bool: - if not isinstance(value, (int, float)) or isinstance(value, bool) or value < 0: - return False - try: - return math.isfinite(value) - except OverflowError: - # Python integers are unbounded, while JSON numbers accepted by the - # upstream timestamp contract must still be representable as finite - # numeric metadata. Oversized integers therefore fail closed. - return False - - def _is_fresh_followup_input(item: Mapping[str, JsonValue]) -> bool: item_type = item.get("type") if item_type in {"input_file", "input_image", "input_text"}: - return _input_item_has_only_known_fields(item, cast(str, item_type)) and _input_content_part_is_self_contained( - item, - allow_output=False, - ) + return _input_content_part_is_self_contained(item, allow_output=False) return ( item_type in (None, "message") and item.get("role") == "user" - and item.get("id") in (None, "") - and item.get("phase") is None - and item.get("status") in (None, "completed") - and _internal_chat_message_metadata_is_account_neutral(item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD)) - and _input_item_has_only_known_fields(item, cast(str | None, item_type)) and _message_has_valid_account_neutral_content(item) ) @@ -1668,11 +625,7 @@ def _tool_call_is_self_contained(item_type: str, item: Mapping[str, JsonValue]) if item.get("status") not in (None, "completed"): return False if item_type == "function_call": - return ( - _is_nonblank_string(item.get("name")) - and isinstance(item.get("arguments"), str) - and (item.get("namespace") is None or _is_nonblank_string(item.get("namespace"))) - ) + return _is_nonblank_string(item.get("name")) and isinstance(item.get("arguments"), str) if item_type == "custom_tool_call": return _is_nonblank_string(item.get("name")) and isinstance(item.get("input"), str) operation = item.get("operation") @@ -1743,12 +696,7 @@ def _is_nonblank_string(value: JsonValue | None) -> bool: return isinstance(value, str) and bool(value.strip()) -def responses_payload_is_account_neutral_fresh_replay( - payload: Mapping[str, JsonValue], - *, - expected_session_identity: str | None = None, - expected_task_identity: str | None = None, -) -> bool: +def responses_payload_is_account_neutral_fresh_replay(payload: Mapping[str, JsonValue]) -> bool: """Return whether a full request can move accounts without stored upstream state.""" if payload.get("conversation") not in (None, ""): @@ -1765,11 +713,7 @@ def responses_payload_is_account_neutral_fresh_replay( return False if not _text_controls_are_account_neutral(payload.get("text")): return False - if not _client_metadata_is_account_neutral( - payload.get("client_metadata"), - expected_session_identity=expected_session_identity, - expected_task_identity=expected_task_identity, - ): + if not _client_metadata_is_account_neutral(payload.get("client_metadata")): return False input_value = payload.get("input") @@ -1804,11 +748,11 @@ def responses_payload_is_account_neutral_fresh_replay( def _reasoning_config_is_account_neutral(reasoning: JsonValue | None) -> bool: if reasoning is None: return True - if not isinstance(reasoning, dict) or not set(reasoning) <= _ACCOUNT_NEUTRAL_REASONING_CONFIG_FIELDS: - return False - if "context" in reasoning and reasoning["context"] != "all_turns": - return False - return all(value is None or isinstance(value, str) for key, value in reasoning.items() if key != "context") + return ( + isinstance(reasoning, dict) + and all(key in _ACCOUNT_NEUTRAL_REASONING_CONFIG_FIELDS for key in reasoning) + and all(value is None or isinstance(value, str) for value in reasoning.values()) + ) def _text_controls_are_account_neutral(text: JsonValue | None) -> bool: @@ -1843,264 +787,21 @@ def _text_controls_are_account_neutral(text: JsonValue | None) -> bool: ) -def _client_metadata_is_account_neutral( - client_metadata: JsonValue | None, - *, - expected_session_identity: str | None, - expected_task_identity: str | None, -) -> bool: +def _client_metadata_is_account_neutral(client_metadata: JsonValue | None) -> bool: if client_metadata is None: return True if not isinstance(client_metadata, dict) or not set(client_metadata) <= _ACCOUNT_NEUTRAL_CLIENT_METADATA_FIELDS: return False - if not all(_is_nonblank_string(value) for value in client_metadata.values()): - return False - if "x-codex-parent-thread-id" in client_metadata or "x-openai-subagent" in client_metadata: - return False - if ( - client_metadata.get( + return ( + all(_is_nonblank_string(value) for value in client_metadata.values()) + and client_metadata.get( "ws_request_header_x_openai_internal_codex_responses_lite", "true", ) - != "true" - ): - return False - session_id = client_metadata.get("session_id") - thread_id = client_metadata.get("thread_id") - turn_id = client_metadata.get("turn_id") - root_turn_id = client_metadata.get("root_turn_id") - if session_id is not None or thread_id is not None or turn_id is not None or root_turn_id is not None: - if ( - expected_session_identity is None - or expected_task_identity is None - or session_id != expected_session_identity - or thread_id != expected_task_identity - or not _is_nonblank_string(turn_id) - or (root_turn_id is not None and root_turn_id != turn_id) - ): - return False - turn_metadata = client_metadata.get("x-codex-turn-metadata") - if turn_metadata is None: - return root_turn_id is None and session_id is None and thread_id is None and turn_id is None - if session_id is None or thread_id is None or turn_id is None: - return False - evidence = account_neutral_codex_turn_metadata_identity( - turn_metadata, - carrier="body", - expected_session_identity=expected_session_identity, - expected_task_identity=expected_task_identity, - expected_turn_identity=cast(str | None, turn_id), - ) - if evidence is None: - return False - return ( - (root_turn_id is None or evidence.root_turn_identity == root_turn_id) - and ( - "x-codex-installation-id" not in client_metadata - or evidence.installation_identity == client_metadata["x-codex-installation-id"] - ) - and ( - "x-codex-window-id" not in client_metadata - or evidence.window_identity == client_metadata["x-codex-window-id"] - ) - ) - - -def account_neutral_codex_turn_metadata_identity( - raw_turn_metadata: JsonValue, - *, - carrier: Literal["body", "direct"], - expected_session_identity: str | None, - expected_task_identity: str | None, - expected_turn_identity: str | None, -) -> AccountNeutralCodexTurnMetadataEvidence | None: - """Validate a canonical, root-task Codex 0.149 turn-metadata carrier.""" - - max_bytes = ( - _ACCOUNT_NEUTRAL_TURN_METADATA_BODY_MAX_BYTES - if carrier == "body" - else _ACCOUNT_NEUTRAL_TURN_METADATA_DIRECT_MAX_BYTES - ) - if ( - not isinstance(raw_turn_metadata, str) - or not raw_turn_metadata.strip() - or len(raw_turn_metadata.encode("utf-8")) > max_bytes - or expected_session_identity is None - or expected_task_identity is None - ): - return None - try: - decoded = json.loads(raw_turn_metadata) - except (TypeError, ValueError): - return None - if ( - not isinstance(decoded, dict) - or not set(decoded) <= _ACCOUNT_NEUTRAL_TURN_METADATA_FIELDS - or any(field in decoded for field in _ACCOUNT_NEUTRAL_TURN_METADATA_LINEAGE_FIELDS) - or _contains_explicit_account_scoped_metadata_state(decoded) - or (carrier == "direct" and "tool_namespaces_info" in decoded) - ): - return None - - session_id = decoded.get("session_id") - thread_id = decoded.get("thread_id") - turn_id = decoded.get("turn_id") - if ( - session_id != expected_session_identity - or thread_id != expected_task_identity - or not _is_nonblank_string(turn_id) - or (expected_turn_identity is not None and turn_id != expected_turn_identity) - or decoded.get("request_kind") != "turn" - ): - return None - for key in ( - "agent_name", - "forked_from_thread_id", - "installation_id", - "sandbox", - "sandbox_mode", - "window_id", - ): - if key in decoded and not _is_nonblank_string(decoded[key]): - return None - if "workspace_kind" in decoded: - workspace_kind = decoded["workspace_kind"] - if ( - not _is_nonblank_string(workspace_kind) - or len(cast(str, workspace_kind).encode("utf-8")) > _ACCOUNT_NEUTRAL_WORKSPACE_KIND_MAX_BYTES - ): - return None - root_turn_id = decoded.get("root_turn_id") - if root_turn_id is not None and root_turn_id != turn_id: - return None - if "thread_source" in decoded and not _root_thread_source_is_account_neutral(decoded["thread_source"]): - return None - for key in ("auto_review_enabled", "node_repl_auto_review_required", "node_repl_disabled"): - if key in decoded and not isinstance(decoded[key], bool): - return None - started_at = decoded.get("turn_started_at_unix_ms") - if started_at is not None and (not isinstance(started_at, int) or isinstance(started_at, bool)): - return None - if "workspaces" in decoded and not _turn_metadata_workspaces_are_account_neutral(decoded["workspaces"]): - return None - if "tool_namespaces_info" in decoded and not _turn_tool_namespaces_info_is_account_neutral( - decoded["tool_namespaces_info"] - ): - return None - shared_projection = dict(decoded) - shared_projection.pop("tool_namespaces_info", None) - shared_projection_fingerprint = sha256( - json.dumps( - shared_projection, - ensure_ascii=True, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - ).hexdigest() - return AccountNeutralCodexTurnMetadataEvidence( - session_identity=cast(str, session_id), - task_identity=cast(str, thread_id), - turn_identity=cast(str, turn_id), - root_turn_identity=cast(str | None, root_turn_id), - installation_identity=cast(str | None, decoded.get("installation_id")), - window_identity=cast(str | None, decoded.get("window_id")), - workspace_kind=cast(str | None, decoded.get("workspace_kind")), - forked_from_thread_identity=cast(str | None, decoded.get("forked_from_thread_id")), - shared_projection_fingerprint=shared_projection_fingerprint, + == "true" ) -def _root_thread_source_is_account_neutral(value: JsonValue) -> bool: - if isinstance(value, str): - return value.lower() != "subagent" and bool(value.strip()) - if not isinstance(value, dict) or len(value) != 1: - return False - key, nested = next(iter(value.items())) - return key.lower() != "subagent" and _is_nonblank_string(nested) - - -def _turn_metadata_workspaces_are_account_neutral(value: JsonValue) -> bool: - if not isinstance(value, dict): - return False - for workspace, metadata in value.items(): - if ( - not _is_nonblank_string(workspace) - or not isinstance(metadata, dict) - or not set(metadata) - <= { - "associated_remote_urls", - "has_changes", - "latest_git_commit_hash", - } - ): - return False - urls = metadata.get("associated_remote_urls") - if urls is not None and not ( - isinstance(urls, dict) - and all(_is_nonblank_string(key) and _is_nonblank_string(url) for key, url in urls.items()) - ): - return False - if metadata.get("latest_git_commit_hash") is not None and not _is_nonblank_string( - metadata["latest_git_commit_hash"] - ): - return False - if metadata.get("has_changes") is not None and not isinstance(metadata["has_changes"], bool): - return False - return True - - -def _turn_tool_namespaces_info_is_account_neutral(value: JsonValue) -> bool: - if not isinstance(value, dict): - return False - for effective_name, namespace in value.items(): - if ( - not _is_nonblank_string(effective_name) - or not isinstance(namespace, dict) - or set(namespace) != {"functions", "name"} - or not _is_nonblank_string(namespace.get("name")) - or not isinstance(namespace.get("functions"), dict) - or not namespace.get("functions") - ): - return False - for function_name, function in cast(dict[str, JsonValue], namespace["functions"]).items(): - if ( - not _is_nonblank_string(function_name) - or not isinstance(function, dict) - or set(function) != {"code_mode_name", "deferred", "direct", "name", "source"} - or not _is_nonblank_string(function.get("name")) - or not isinstance(function.get("direct"), bool) - or not isinstance(function.get("deferred"), bool) - or ( - function.get("code_mode_name") is not None - and not _is_nonblank_string(function.get("code_mode_name")) - ) - or not _turn_tool_source_is_account_neutral(function.get("source")) - ): - return False - return True - - -def _turn_tool_source_is_account_neutral(value: JsonValue | None) -> bool: - if not isinstance(value, dict) or value.get("kind") not in {"harness", "mcp"}: - return False - if value["kind"] == "harness": - return set(value) == {"kind"} - return set(value) == {"kind", "server_name"} and _is_nonblank_string(value.get("server_name")) - - -def _contains_explicit_account_scoped_metadata_state(value: JsonValue) -> bool: - pending = [value] - while pending: - current = pending.pop() - if isinstance(current, dict): - if _mapping_has_account_scoped_reference(current): - return True - pending.extend(current.values()) - elif isinstance(current, list): - pending.extend(current) - return False - - def _tools_are_account_neutral(tools: JsonValue) -> bool: return isinstance(tools, list) and all( isinstance(tool, dict) and _tool_declaration_is_account_neutral(tool) for tool in tools @@ -2113,38 +814,12 @@ def _tool_declaration_is_account_neutral(tool: Mapping[str, JsonValue]) -> bool: return False if any(key not in _ACCOUNT_NEUTRAL_TOOL_DECLARATION_FIELDS[tool_type] for key in tool): return False - if tool_type == "namespace": - nested_tools = tool.get("tools") - return ( - set(tool) == _ACCOUNT_NEUTRAL_TOOL_DECLARATION_FIELDS["namespace"] - and _is_nonblank_string(tool.get("name")) - and isinstance(tool.get("description"), str) - and isinstance(nested_tools, list) - and bool(nested_tools) - and all( - isinstance(nested_tool, dict) and _responses_lite_namespace_tool_is_account_neutral(nested_tool) - for nested_tool in nested_tools - ) - ) - if tool_type == "tool_search": - return ( - set(tool) == _ACCOUNT_NEUTRAL_TOOL_DECLARATION_FIELDS["tool_search"] - and tool.get("execution") == "client" - and _is_nonblank_string(tool.get("description")) - and _responses_lite_tool_search_schema_is_account_neutral(tool.get("parameters")) - ) if _contains_account_scoped_tool_state(tool): return False if tool_type in {"custom", "function"} and not _is_nonblank_string(tool.get("name")): return False if tool.get("description") is not None and not isinstance(tool.get("description"), str): return False - if ( - tool_type in {"custom", "function"} - and tool.get("defer_loading") is not None - and not isinstance(tool.get("defer_loading"), bool) - ): - return False if tool_type == "function": return (tool.get("parameters") is None or isinstance(tool.get("parameters"), dict)) and ( tool.get("strict") is None or isinstance(tool.get("strict"), bool) @@ -2154,73 +829,6 @@ def _tool_declaration_is_account_neutral(tool: Mapping[str, JsonValue]) -> bool: return _web_search_tool_options_are_account_neutral(tool_type, tool) -def _responses_lite_namespace_tool_is_account_neutral(tool: Mapping[str, JsonValue]) -> bool: - tool_type = tool.get("type") - if tool_type == "function": - required_fields = {"description", "name", "parameters", "strict", "type"} - if not required_fields <= set(tool) or not set(tool) <= _ACCOUNT_NEUTRAL_TOOL_DECLARATION_FIELDS["function"]: - return False - return ( - _is_nonblank_string(tool.get("name")) - and isinstance(tool.get("description"), str) - and isinstance(tool.get("strict"), bool) - and isinstance(tool.get("parameters"), dict) - and tool.get("defer_loading", True) is True - ) - if tool_type == "custom": - required_fields = {"description", "format", "name", "type"} - if not required_fields <= set(tool) or not set(tool) <= _ACCOUNT_NEUTRAL_TOOL_DECLARATION_FIELDS["custom"]: - return False - return ( - _is_nonblank_string(tool.get("name")) - and isinstance(tool.get("description"), str) - and _responses_lite_custom_tool_format_is_account_neutral(tool.get("format")) - and tool.get("defer_loading", True) is True - ) - return False - - -def _responses_lite_custom_tool_format_is_account_neutral(format_value: JsonValue | None) -> bool: - """Validate the non-null FreeformToolFormat emitted inside a 0.149 namespace.""" - - return ( - isinstance(format_value, dict) - and format_value.get("type") == "grammar" - and _custom_tool_format_is_account_neutral(format_value) - ) - - -def _responses_lite_tool_search_schema_is_account_neutral(value: JsonValue | None) -> bool: - if not isinstance(value, dict) or set(value) != { - "additionalProperties", - "properties", - "required", - "type", - }: - return False - properties = value.get("properties") - if ( - value.get("type") != "object" - or value.get("required") != ["query"] - or value.get("additionalProperties") is not False - or not isinstance(properties, dict) - or set(properties) != {"limit", "query"} - ): - return False - query = properties.get("query") - limit = properties.get("limit") - return ( - isinstance(query, dict) - and set(query) == {"description", "type"} - and query.get("type") == "string" - and _is_nonblank_string(query.get("description")) - and isinstance(limit, dict) - and set(limit) == {"description", "type"} - and limit.get("type") == "number" - and _is_nonblank_string(limit.get("description")) - ) - - def _web_search_tool_options_are_account_neutral( tool_type: str, tool: Mapping[str, JsonValue], diff --git a/app/modules/proxy/request_policy.py b/app/modules/proxy/request_policy.py index 6769493b21..4917045f56 100644 --- a/app/modules/proxy/request_policy.py +++ b/app/modules/proxy/request_policy.py @@ -1,17 +1,21 @@ from __future__ import annotations import logging +from collections.abc import Mapping +from typing import NamedTuple from pydantic import ValidationError from app.core.errors import OpenAIErrorEnvelope, openai_error -from app.core.exceptions import ProxyModelNotAllowed +from app.core.exceptions import ProxyModelNotAllowed, ProxyReasoningEffortNotAllowed from app.core.openai.exceptions import ClientPayloadError from app.core.openai.model_registry import ModelRegistry, get_model_registry from app.core.openai.requests import ( ResponsesCompactRequest, ResponsesReasoning, ResponsesRequest, + extract_input_file_ids, + normalize_reasoning_aliases, responses_input_uses_lite_tools, ) from app.core.openai.strict_schema import ( @@ -19,10 +23,13 @@ validate_strict_json_schema, ) from app.core.openai.v1_requests import V1ResponsesRequest +from app.core.runtime_logging import safe_log_field from app.core.types import JsonValue from app.core.utils.json_guards import is_json_list, is_json_mapping from app.core.utils.request_id import get_request_id +from app.db.models import ModelSource from app.modules.api_keys.service import ApiKeyData +from app.modules.model_sources.catalog import source_model_reasoning_levels logger = logging.getLogger(__name__) @@ -125,13 +132,146 @@ def validate_model_access(api_key: ApiKeyData | None, model: str | None) -> None raise ProxyModelNotAllowed(f"This API key does not have access to model '{model}'") +def validate_reasoning_effort_access(api_key: ApiKeyData | None, effort: str | None) -> None: + if api_key is None: + return + allowed_reasoning_efforts = getattr(api_key, "allowed_reasoning_efforts", None) + if allowed_reasoning_efforts is None or effort is None: + return + normalized_effort = effort.strip().lower() + if normalized_effort in allowed_reasoning_efforts: + return + logger.info( + "api_key_reasoning_effort_not_allowed request_id=%s key_id=%s reasoning_effort=%s", + get_request_id(), + "", + safe_log_field(normalized_effort), + ) + raise ProxyReasoningEffortNotAllowed( + f"This API key does not have access to reasoning effort '{normalized_effort}'", + param="reasoning.effort", + ) + + +def _client_reasoning_effort(payload: ResponsesRequest | ResponsesCompactRequest) -> str | None: + """Return the effort selected by the client before wire normalization. + + Cursor encodes its effort in an accepted model alias, where ``xhigh`` is + later lowered to the upstream's ``high`` value. API-key policies are an + operator-facing client-plane control, so they must compare against the + original selection rather than that wire representation. + """ + model_effort = _client_reasoning_effort_from_model(payload.model) + if model_effort is not None: + return model_effort + + reasoning = payload.reasoning.model_dump(mode="json", exclude_none=True) if payload.reasoning is not None else None + if is_json_mapping(reasoning): + effort = reasoning.get("effort") + if isinstance(effort, str) and effort.strip(): + return effort.strip().lower() + return _client_reasoning_effort_from_provider_aliases(payload) + + +def _client_reasoning_effort_from_provider_aliases( + payload: ResponsesRequest | ResponsesCompactRequest, +) -> str | None: + reasoning = payload.reasoning.model_dump(mode="json", exclude_none=True) if payload.reasoning is not None else None + extra = payload.model_extra + if isinstance(extra, dict): + alias_payload = dict(extra) + if reasoning is not None: + alias_payload["reasoning"] = reasoning + normalize_reasoning_aliases(alias_payload) + normalized_reasoning = alias_payload.get("reasoning") + if is_json_mapping(normalized_reasoning): + extra_effort = normalized_reasoning.get("effort") + if isinstance(extra_effort, str) and extra_effort.strip(): + return extra_effort.strip().lower() + + return None + + +def _materialize_provider_reasoning_effort( + payload: ResponsesRequest | ResponsesCompactRequest, + effort: str | None, +) -> None: + existing_effort = payload.reasoning.effort if payload.reasoning is not None else None + if effort is None or (isinstance(existing_effort, str) and existing_effort.strip()): + return + if payload.reasoning is None: + payload.reasoning = ResponsesReasoning(effort=effort) + else: + payload.reasoning.effort = effort + if isinstance(payload, ResponsesRequest): + payload._codex_lb_provider_reasoning_effort_materialized = True + + +def _client_reasoning_effort_from_model(model: str | None) -> str | None: + alias = _resolve_model_alias_parts(model) + if alias is not None: + normalized_model = model.strip().lower() if isinstance(model, str) else "" + suffix = normalized_model[len(alias[0]) + 1 :] + tokens = {token for token in suffix.split("-") if token} + if "xhigh" in tokens or "extra" in tokens: + return "xhigh" + if alias[1] is not None: + return alias[1] + return None + + +def normalize_source_reasoning_aliases(payload: dict[str, JsonValue]) -> None: + """Align effort-bearing aliases while preserving unrelated source controls.""" + provider_thinking = payload.get("thinking") + preserve_provider_thinking = False + if "thinking" in payload: + probe: dict[str, JsonValue] = {"thinking": provider_thinking} + normalize_reasoning_aliases(probe) + normalized_reasoning = probe.get("reasoning") + normalized_effort = normalized_reasoning.get("effort") if is_json_mapping(normalized_reasoning) else None + thinking_mapping = provider_thinking if is_json_mapping(provider_thinking) else None + thinking_type = thinking_mapping.get("type") if thinking_mapping is not None else None + is_inactive = thinking_mapping is not None and ( + thinking_mapping.get("enabled") is False + or (isinstance(thinking_type, str) and thinking_type.strip().lower() == "disabled") + ) + preserve_provider_thinking = ( + thinking_mapping is not None + and not is_inactive + and not (isinstance(normalized_effort, str) and bool(normalized_effort.strip())) + ) + if preserve_provider_thinking: + payload.pop("thinking", None) + normalize_reasoning_aliases(payload) + if preserve_provider_thinking and thinking_mapping is not None: + preserved_thinking = dict(thinking_mapping.items()) + preserved_effort = preserved_thinking.get("effort") + if isinstance(preserved_effort, str) and not preserved_effort.strip(): + preserved_thinking.pop("effort") + payload["thinking"] = preserved_thinking + + +class ApiKeyEnforcementResult(NamedTuple): + """What :func:`apply_api_key_enforcement` observed while mutating the payload. + + ``pre_normalization_reasoning_effort`` carries the effort that + :func:`normalize_unsupported_reasoning_effort` replaced, so a caller that + later routes the request to an OpenAI-compatible model source can restore + it. It is the post-enforcement value: restoring it cannot resurrect an + effort an API key overrode. + """ + + service_tier_was_enforced: bool + pre_normalization_reasoning_effort: str | None + + def apply_api_key_enforcement( payload: ResponsesRequest | ResponsesCompactRequest, api_key: ApiKeyData | None, *, registry: ModelRegistry | None = None, prohibit_fast_mode: bool = False, -) -> bool: +) -> ApiKeyEnforcementResult: """Apply API-key policy and report whether it supplied the service tier. The returned provenance is captured before mutating ``payload``. Callers @@ -139,23 +279,31 @@ def apply_api_key_enforcement( equal the enforced value (including after ``fast`` canonicalizes to ``priority``). """ + client_reasoning_effort = payload._codex_lb_client_reasoning_effort or _client_reasoning_effort(payload) + payload._codex_lb_client_reasoning_effort = client_reasoning_effort + provider_reasoning_effort = _client_reasoning_effort_from_provider_aliases(payload) normalize_upstream_model_alias(payload, prohibit_fast_mode=prohibit_fast_mode) if api_key is None: - normalize_unsupported_reasoning_effort(payload) - return False + _materialize_provider_reasoning_effort(payload, provider_reasoning_effort) + pre_normalization_effort = normalize_unsupported_reasoning_effort(payload, registry=registry) + return ApiKeyEnforcementResult(False, pre_normalization_effort) if api_key.enforced_model: + enforced_model_reasoning_effort = _client_reasoning_effort_from_model(api_key.enforced_model) requested_model = payload.model if requested_model != api_key.enforced_model: logger.info( "api_key_model_enforced request_id=%s key_id=%s requested_model=%s enforced_model=%s", get_request_id(), - api_key.id, - requested_model, - api_key.enforced_model, + "", + safe_log_field(requested_model), + safe_log_field(api_key.enforced_model), ) payload.model = api_key.enforced_model + if enforced_model_reasoning_effort is not None: + client_reasoning_effort = enforced_model_reasoning_effort + payload._codex_lb_client_reasoning_effort = client_reasoning_effort normalize_upstream_model_alias(payload, prohibit_fast_mode=prohibit_fast_mode) if ( responses_input_uses_lite_tools(payload.input) @@ -180,12 +328,21 @@ def apply_api_key_enforcement( logger.info( "api_key_reasoning_enforced request_id=%s key_id=%s requested_effort=%s enforced_effort=%s", get_request_id(), - api_key.id, - requested_effort, - api_key.enforced_reasoning_effort, + "", + safe_log_field(requested_effort), + safe_log_field(api_key.enforced_reasoning_effort), ) - normalize_unsupported_reasoning_effort(payload) + _materialize_provider_reasoning_effort(payload, provider_reasoning_effort) + if client_reasoning_effort is not None: + validate_reasoning_effort_access(api_key, client_reasoning_effort) + if ( + payload.reasoning is not None + and isinstance(payload.reasoning.effort, str) + and payload.reasoning.effort.strip().lower() == client_reasoning_effort + ): + payload.reasoning.effort = client_reasoning_effort + pre_normalization_effort = normalize_unsupported_reasoning_effort(payload, registry=registry) service_tier_was_enforced = False if api_key.enforced_service_tier is not None: @@ -210,12 +367,12 @@ def apply_api_key_enforcement( "requested_service_tier=%s enforced_service_tier=%s " "outbound_service_tier=%s", get_request_id(), - api_key.id, - requested_service_tier, - api_key.enforced_service_tier, - effective_service_tier, + "", + safe_log_field(requested_service_tier), + safe_log_field(api_key.enforced_service_tier), + safe_log_field(effective_service_tier), ) - return service_tier_was_enforced + return ApiKeyEnforcementResult(service_tier_was_enforced, pre_normalization_effort) def apply_enforced_service_tier_model_fallback( @@ -243,8 +400,8 @@ def apply_enforced_service_tier_model_fallback( logger.info( "api_key_enforced_service_tier_model_fallback request_id=%s model=%s enforced_service_tier=%s", get_request_id(), - payload.model, - service_tier, + safe_log_field(payload.model), + safe_log_field(service_tier), ) payload.service_tier = None return True @@ -301,6 +458,9 @@ def sanitize_source_chat_payload( def apply_api_key_enforcement_to_chat_payload( payload: dict[str, JsonValue], api_key: ApiKeyData | None, + *, + allowed_reasoning_effort: str | None = None, + materialize_allowed_reasoning_effort: bool = False, ) -> None: """Mirror :func:`apply_api_key_enforcement` onto a chat-completions wire payload. @@ -309,6 +469,58 @@ def apply_api_key_enforcement_to_chat_payload( applied to the outbound dict as well or the upstream receives the caller's values while accounting uses the enforced ones. """ + if allowed_reasoning_effort is not None: + wire_effort = resolve_wire_reasoning_effort(allowed_reasoning_effort) + # Chat requests can express the same setting through several provider + # aliases. Once the Responses conversion has authorized one effective + # choice, make every caller-supplied alias agree without adding fields + # that the selected source may not accept. + if "reasoning_effort" in payload: + payload["reasoning_effort"] = wire_effort + if "reasoningEffort" in payload: + payload["reasoningEffort"] = wire_effort + if "thinking" in payload: + thinking = payload["thinking"] + if isinstance(thinking, dict): + thinking_effort = thinking.get("effort") + if isinstance(thinking_effort, str) and not thinking_effort.strip(): + thinking = {**thinking} + thinking.pop("effort") + payload["thinking"] = thinking + if isinstance(thinking_effort, str) and thinking_effort.strip(): + aligned_thinking = {**thinking, "effort": wire_effort} + thinking_type = aligned_thinking.get("type") + if isinstance(thinking_type, str) and thinking_type.strip().lower() == "disabled": + aligned_thinking.pop("type") + if aligned_thinking.get("enabled") is False: + aligned_thinking.pop("enabled") + payload["thinking"] = aligned_thinking + else: + thinking_type = thinking.get("type") + is_inactive = thinking.get("enabled") is False or ( + isinstance(thinking_type, str) and thinking_type.strip().lower() == "disabled" + ) + selects_implicit_medium = thinking.get("enabled") is True or ( + isinstance(thinking_type, str) and thinking_type.strip().lower() == "enabled" + ) + if is_inactive or (selects_implicit_medium and wire_effort != "medium"): + payload.pop("thinking") + else: + payload["thinking"] = wire_effort + if "enable_thinking" in payload: + if wire_effort == "medium": + payload["enable_thinking"] = True + else: + payload.pop("enable_thinking", None) + reasoning = payload.get("reasoning") + if isinstance(reasoning, dict): + payload["reasoning"] = {**reasoning, "effort": wire_effort} + if materialize_allowed_reasoning_effort and not any( + key in payload + for key in ("reasoning_effort", "reasoningEffort", "thinking", "enable_thinking", "reasoning") + ): + payload["reasoning_effort"] = wire_effort + if api_key is None: return @@ -355,8 +567,8 @@ def normalize_upstream_model_alias( logger.info( "model_alias_normalized request_id=%s requested_model=%s normalized_model=%s", get_request_id(), - payload.model, - canonical_model, + safe_log_field(payload.model), + safe_log_field(canonical_model), ) payload.model = canonical_model @@ -371,10 +583,10 @@ def normalize_upstream_model_alias( "model_alias_reasoning_normalized request_id=%s requested_model=%s " "normalized_model=%s requested_effort=%s normalized_effort=%s", get_request_id(), - requested_model, - canonical_model, - requested_effort, - alias_effort, + safe_log_field(requested_model), + safe_log_field(canonical_model), + safe_log_field(requested_effort), + safe_log_field(alias_effort), ) if alias_service_tier is not None and getattr(payload, "service_tier", None) is None: @@ -382,8 +594,8 @@ def normalize_upstream_model_alias( logger.info( "model_alias_fast_mode_prohibited request_id=%s requested_model=%s normalized_model=%s", get_request_id(), - requested_model, - canonical_model, + safe_log_field(requested_model), + safe_log_field(canonical_model), ) return setattr(payload, "service_tier", alias_service_tier) @@ -391,9 +603,9 @@ def normalize_upstream_model_alias( "model_alias_service_tier_normalized request_id=%s requested_model=%s " "normalized_model=%s normalized_service_tier=%s", get_request_id(), - requested_model, - canonical_model, - alias_service_tier, + safe_log_field(requested_model), + safe_log_field(canonical_model), + safe_log_field(alias_service_tier), ) @@ -448,7 +660,7 @@ def normalize_unsupported_reasoning_effort( payload: ResponsesRequest | ResponsesCompactRequest, *, registry: ModelRegistry | None = None, -) -> None: +) -> str | None: """Rewrite ``reasoning.effort`` values the upstream backend rejects. Some efforts that codex-lb accepts at the API surface (notably @@ -461,10 +673,25 @@ def normalize_unsupported_reasoning_effort( Client-plane efforts the reference Codex client aliases before sending (``ultra`` -> ``max``) are rewritten the same way here. + + Returns the effort that the unsupported-effort fallback replaced, in + normalized (trimmed, lowercased) form, or ``None`` when nothing restorable + was rewritten. Model sources do not have the backend quirk that fallback + works around, but whether a request is served by one is only known after + source selection, which happens later; callers that can reach a source + carry this value forward and restore it there (see + ``restore_source_reasoning_effort``). + + The ``ultra`` -> ``max`` wire alias is never reported. That aliasing mirrors + the reference client and is required on every upstream surface, so it must + survive source routing too. + + The reported value is the post-enforcement effort rather than the client's + original, so restoring it cannot resurrect an effort an API key overrode. """ if payload.reasoning is None or payload.reasoning.effort is None: - return + return None requested_effort = payload.reasoning.effort normalized_effort = requested_effort.strip().lower() @@ -475,14 +702,16 @@ def normalize_unsupported_reasoning_effort( logger.info( "reasoning_effort_wire_aliased request_id=%s model=%s requested_effort=%s aliased_effort=%s", get_request_id(), - payload.model, - requested_effort, - wire_alias, + safe_log_field(payload.model), + safe_log_field(requested_effort), + safe_log_field(wire_alias), ) - return + # Deliberately not reported as restorable: the ultra -> max alias must + # hold on every surface, source-routed payloads included. + return None if normalized_effort not in _UNSUPPORTED_UPSTREAM_REASONING_EFFORTS: - return + return None fallback = _resolve_reasoning_effort_fallback( payload.model, @@ -490,12 +719,44 @@ def normalize_unsupported_reasoning_effort( ) payload.reasoning.effort = fallback logger.info( - "reasoning_effort_normalized request_id=%s model=%s requested_effort=%s normalized_effort=%s", + "reasoning_effort_normalized request_id=%s", get_request_id(), - payload.model, - requested_effort, - fallback, ) + return normalized_effort + + +def restore_source_reasoning_effort( + payload: ResponsesRequest | ResponsesCompactRequest, + source: ModelSource, + *, + pre_normalization_effort: str | None, +) -> None: + """Undo :func:`normalize_unsupported_reasoning_effort` for a source-routed request. + + The rewrite exists solely to work around a ChatGPT/Codex backend quirk, so + it must not reach an OpenAI-compatible model source. This runs at the point + where the source has actually been selected, which is the only place the + routing outcome is known -- inferring it earlier from registry membership + misfires in both directions (a subscription model missing from a populated + snapshot, and a source model whose slug shadows a subscription one). + + The restore is gated on the operator having declared the effort for this + model: sources without reasoning metadata keep the pre-existing behaviour, + and an effort the backend never advertised is not sent to it. + """ + if pre_normalization_effort is None or payload.reasoning is None: + return + if payload.model is None: + return + restored_effort = pre_normalization_effort.strip().lower() + declared = {level.effort for level in source_model_reasoning_levels(source, payload.model)} + if restored_effort not in declared: + return + # Normalized on assignment rather than trusting the caller: the sole + # producer already reports the normalized form, but that invariant is + # non-local and a casing variant must never reach the wire. + payload.reasoning.effort = restored_effort + logger.info("reasoning_effort_restored_for_source request_id=%s", get_request_id()) def _resolve_reasoning_effort_fallback( @@ -570,18 +831,43 @@ def normalize_responses_request_payload( return responses -def strip_terminal_compaction_trigger_input(payload: ResponsesRequest) -> list[JsonValue] | None: +def validate_top_level_compaction_trigger_input_shape(payload: Mapping[str, JsonValue]) -> None: + input_value = payload.get("input") + if not is_json_list(input_value): + return + _validate_terminal_compaction_trigger_input_items(input_value) + + +def strip_terminal_compaction_trigger_input( + payload: ResponsesRequest | ResponsesCompactRequest, + *, + strip_trigger: bool = True, +) -> list[JsonValue] | None: input_value = payload.input if not is_json_list(input_value): return None + return _strip_terminal_compaction_trigger_input_items(input_value, strip_trigger=strip_trigger) + - stripped_input: list[JsonValue] = [] +def _strip_terminal_compaction_trigger_input_items( + input_value: list[JsonValue], + *, + strip_trigger: bool, +) -> list[JsonValue] | None: + trigger_seen = _validate_terminal_compaction_trigger_input_items(input_value) + if not trigger_seen: + return None + if not strip_trigger: + return input_value + return [item for item in input_value if not (is_json_mapping(item) and item.get("type") == "compaction_trigger")] + + +def _validate_terminal_compaction_trigger_input_items(input_value: list[JsonValue]) -> bool: trigger_seen = False last_index = len(input_value) - 1 for index, item in enumerate(input_value): if not (is_json_mapping(item) and item.get("type") == "compaction_trigger"): - stripped_input.append(item) continue if trigger_seen or index != last_index: @@ -593,9 +879,25 @@ def strip_terminal_compaction_trigger_input(payload: ResponsesRequest) -> list[J ) trigger_seen = True - if not trigger_seen: - return None - return stripped_input + return trigger_seen + + +def responses_source_route_excluded(payload: ResponsesRequest) -> bool: + """True when a Responses request must stay on subscription accounts. + + A terminal compaction trigger is served by the upstream compact flow on the + turn's owner account, and an ``input_file``/``input_image`` file reference + is pinned to the subscription account that received the upload — neither + can be dispatched to an OpenAI-compatible model source. The HTTP + ``/responses`` route and the WebSocket source-ownership guards share this + predicate so their notion of source-route eligibility cannot drift. + + Raises ``ClientPayloadError`` for a malformed compaction trigger, exactly + like ``strip_terminal_compaction_trigger_input``. + """ + if strip_terminal_compaction_trigger_input(payload) is not None: + return True + return bool(extract_input_file_ids(payload.input)) def enforce_strict_text_format(request: ResponsesRequest) -> None: diff --git a/app/modules/proxy/response_transition_manifest.py b/app/modules/proxy/response_transition_manifest.py deleted file mode 100644 index 8ef133fc41..0000000000 --- a/app/modules/proxy/response_transition_manifest.py +++ /dev/null @@ -1,253 +0,0 @@ -"""Content-free durable proofs for completed Responses output transitions.""" - -from __future__ import annotations - -import json -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from hashlib import sha256 -from typing import Any - -from app.core.openai.public_output import normalize_public_output_item -from app.core.types import JsonValue - -RESPONSE_TRANSITION_MANIFEST_SCHEMA = "qk_http_bridge_response_transition_manifest_v1" -_MAX_MANIFEST_ITEMS = 4096 -_SUPPORTED_OUTPUT_ITEM_TYPES = frozenset( - { - "agent_message", - "apply_patch_call", - "custom_tool_call", - "function_call", - "image_generation_call", - "message", - "reasoning", - "tool_search_call", - "tool_search_output", - "web_search_call", - } -) -_CLIENT_SETTLED_CALL_TYPES = frozenset( - { - "apply_patch_call", - "custom_tool_call", - "function_call", - } -) -_SUPPORTED_MANIFEST_ITEM_KINDS = (_SUPPORTED_OUTPUT_ITEM_TYPES - {"message"}) | {"message:assistant"} - - -def _canonical_json(value: object) -> str: - return json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True) - - -def _canonical_sha256(value: object) -> str: - return sha256(_canonical_json(value).encode("utf-8")).hexdigest() - - -def _is_sha256(value: object) -> bool: - return isinstance(value, str) and len(value) == 64 and all(character in "0123456789abcdef" for character in value) - - -def _item_kind(item: Mapping[str, JsonValue]) -> str | None: - item_type = item.get("type") - if not isinstance(item_type, str) or item_type not in _SUPPORTED_OUTPUT_ITEM_TYPES: - return None - if item_type != "message": - return item_type - role = item.get("role") - if role != "assistant": - return None - phase = item.get("phase") - if phase is not None and (not isinstance(phase, str) or not phase): - return None - return "message:assistant" - - -def pending_tool_calls_digest(pending_tool_calls: Mapping[str, str]) -> str: - return _canonical_sha256(dict(sorted(pending_tool_calls.items()))) - - -@dataclass(frozen=True, slots=True) -class ResponseTransitionManifestItem: - kind: str - fingerprint: str - - def canonical_payload(self) -> dict[str, str]: - return {"fingerprint": self.fingerprint, "kind": self.kind} - - -@dataclass(frozen=True, slots=True) -class ResponseTransitionManifest: - response_id_hash: str - terminal_status: str - pending_tool_calls_digest: str - items: tuple[ResponseTransitionManifestItem, ...] - schema: str = RESPONSE_TRANSITION_MANIFEST_SCHEMA - - def canonical_payload(self) -> dict[str, JsonValue]: - return { - "items": [item.canonical_payload() for item in self.items], - "pending_tool_calls_digest": self.pending_tool_calls_digest, - "response_id_hash": self.response_id_hash, - "schema": self.schema, - "terminal_status": self.terminal_status, - } - - @property - def digest(self) -> str: - return _canonical_sha256(self.canonical_payload()) - - @property - def item_kinds(self) -> tuple[str, ...]: - return tuple(item.kind for item in self.items) - - -def build_response_transition_manifest( - payload: Mapping[str, JsonValue] | None, - *, - pending_tool_calls: Mapping[str, str], - normalize_for_public_contract: bool = False, -) -> ResponseTransitionManifest | None: - """Build a bounded manifest without retaining response or tool content.""" - - response = payload.get("response") if isinstance(payload, Mapping) else None - if not isinstance(response, Mapping): - return None - response_id = response.get("id") - terminal_status = response.get("status") - output = response.get("output") - if ( - not isinstance(response_id, str) - or not response_id - or terminal_status != "completed" - or not isinstance(output, list) - or not output - or len(output) > _MAX_MANIFEST_ITEMS - ): - return None - - manifest_items: list[ResponseTransitionManifestItem] = [] - observed_client_calls: dict[str, str] = {} - for raw_item in output: - if not isinstance(raw_item, Mapping): - return None - item = dict(raw_item) - replay_item = normalize_public_output_item(item) if normalize_for_public_contract else item - if replay_item is None: - return None - kind = _item_kind(replay_item) - if kind is None: - return None - item_type = item.get("type") - if item_type in _CLIENT_SETTLED_CALL_TYPES: - call_id = item.get("call_id") - if not isinstance(call_id, str) or not call_id or call_id in observed_client_calls: - return None - observed_client_calls[call_id] = str(item_type) - manifest_items.append( - ResponseTransitionManifestItem( - kind=kind, - fingerprint=_canonical_sha256(replay_item), - ) - ) - - if observed_client_calls != dict(pending_tool_calls): - return None - return ResponseTransitionManifest( - response_id_hash=sha256(response_id.encode("utf-8")).hexdigest(), - terminal_status="completed", - pending_tool_calls_digest=pending_tool_calls_digest(pending_tool_calls), - items=tuple(manifest_items), - ) - - -def response_transition_manifest_item_matches( - item: Mapping[str, JsonValue], - expected: ResponseTransitionManifestItem, -) -> bool: - return _item_kind(item) == expected.kind and _canonical_sha256(dict(item)) == expected.fingerprint - - -def encode_response_transition_manifest(manifest: ResponseTransitionManifest | None) -> str | None: - if manifest is None: - return None - return _canonical_json(manifest.canonical_payload()) - - -def decode_response_transition_manifest(value: str | None) -> ResponseTransitionManifest | None: - if value is None: - return None - try: - payload: Any = json.loads(value) - except (TypeError, ValueError): - return None - if not isinstance(payload, dict) or set(payload) != { - "items", - "pending_tool_calls_digest", - "response_id_hash", - "schema", - "terminal_status", - }: - return None - if ( - payload.get("schema") != RESPONSE_TRANSITION_MANIFEST_SCHEMA - or payload.get("terminal_status") != "completed" - or not _is_sha256(payload.get("response_id_hash")) - or not _is_sha256(payload.get("pending_tool_calls_digest")) - ): - return None - raw_items = payload.get("items") - if not isinstance(raw_items, list) or not raw_items or len(raw_items) > _MAX_MANIFEST_ITEMS: - return None - items: list[ResponseTransitionManifestItem] = [] - for raw_item in raw_items: - if ( - not isinstance(raw_item, dict) - or set(raw_item) != {"fingerprint", "kind"} - or not isinstance(raw_item.get("kind"), str) - or raw_item["kind"] not in _SUPPORTED_MANIFEST_ITEM_KINDS - or not _is_sha256(raw_item.get("fingerprint")) - ): - return None - items.append( - ResponseTransitionManifestItem( - kind=raw_item["kind"], - fingerprint=raw_item["fingerprint"], - ) - ) - return ResponseTransitionManifest( - response_id_hash=payload["response_id_hash"], - terminal_status=payload["terminal_status"], - pending_tool_calls_digest=payload["pending_tool_calls_digest"], - items=tuple(items), - schema=payload["schema"], - ) - - -def response_transition_manifest_matches_context( - manifest: ResponseTransitionManifest, - *, - response_id: str, - pending_tool_calls: Mapping[str, str], -) -> bool: - return manifest.response_id_hash == sha256( - response_id.encode("utf-8") - ).hexdigest() and manifest.pending_tool_calls_digest == pending_tool_calls_digest(pending_tool_calls) - - -def match_response_transition_manifest_prefix( - input_items: Sequence[JsonValue], - *, - stored_count: int, - manifest: ResponseTransitionManifest, -) -> int | None: - """Return the first item after an exact manifest-bound output prefix.""" - - manifest_end = stored_count + len(manifest.items) - if stored_count < 0 or manifest_end > len(input_items): - return None - for actual, expected in zip(input_items[stored_count:manifest_end], manifest.items, strict=True): - if not isinstance(actual, Mapping) or not response_transition_manifest_item_matches(actual, expected): - return None - return manifest_end diff --git a/app/modules/proxy/rowless_recovery.py b/app/modules/proxy/rowless_recovery.py deleted file mode 100644 index 9af6c98711..0000000000 --- a/app/modules/proxy/rowless_recovery.py +++ /dev/null @@ -1,249 +0,0 @@ -"""Content-free proofs for an operator-acknowledged rowless semantic rebase.""" - -from __future__ import annotations - -import hashlib -import json -from dataclasses import dataclass -from typing import cast - -from app.core.openai.requests import ResponsesRequest -from app.core.types import JsonValue -from app.modules.proxy.replay_safety import ( - normalize_responses_input_for_rowless_replay, - project_responses_input_for_account_neutral_fresh_replay, - responses_direct_call_ledger_summary, - responses_input_items_are_self_contained_rowless_replay, - responses_input_retains_prior_output_and_fresh_followup, - responses_payload_is_account_neutral_fresh_replay, -) - -ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT = "operator_acknowledged_semantic_rebase" -ROWLESS_FORWARDING_PAYLOAD_SCHEMA = "qk_http_bridge_rowless_forwarding_payload_v1" -ROWLESS_AUTHORIZATION_MODE_AUTOMATIC = "automatic_live_request" -ROWLESS_AUTHORIZATION_MODE_OPERATOR = "operator_checkpoint" - - -@dataclass(frozen=True, slots=True) -class RowlessRecoveryCaptureFacts: - input_item_count: int - input_fingerprint: str - contract_fingerprint: str - direct_call_ledger_digest: str - projected_payload_fingerprint: str - actual_wire_fingerprint: str - unresolved_count: int - projected_input: list[JsonValue] - self_contained: bool - account_neutral: bool - retains_prior_output: bool - - -@dataclass(frozen=True, slots=True) -class RowlessRecoveryCaptureIntent: - api_key_scope: str - session_key_kind: str - strong_session_hash: str - task_authority_digest: str - task_identity: str - session_identity: str - facts: RowlessRecoveryCaptureFacts - automatic_live_recovery: bool = False - - -def canonical_json_sha256(value: object) -> str: - canonical = json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest() - - -def rowless_strong_session_hash(session_key_kind: str, session_key_value: str) -> str: - return canonical_json_sha256({"kind": session_key_kind, "value": session_key_value}) - - -def rowless_task_authority_digest( - *, - session_id: str, - prompt_cache_key: str, - thread_id: str, -) -> str: - """Bind the stable Codex task identity independently of turn-state routing.""" - - payload = bytearray(b"qk-http-bridge-task-authority-v1\0") - for tag, value in ( - ("session-id", session_id), - ("prompt_cache_key", prompt_cache_key), - ("thread-id", thread_id), - ): - tag_bytes = tag.encode("utf-8") - value_bytes = value.encode("utf-8") - payload.extend(len(tag_bytes).to_bytes(2, "big")) - payload.extend(tag_bytes) - payload.extend(len(value_bytes).to_bytes(4, "big")) - payload.extend(value_bytes) - return hashlib.sha256(payload).hexdigest() - - -def responses_non_input_contract_fingerprint(payload: ResponsesRequest) -> str: - contract = dict(payload.model_dump_for_forwarding()) - contract.pop("input", None) - contract.pop("previous_response_id", None) - return canonical_json_sha256(contract) - - -def rowless_forwarding_payload_fingerprint(payload: ResponsesRequest) -> str: - """Bind the anchor-free logical payload and projection version.""" - - return canonical_json_sha256( - { - "schema": ROWLESS_FORWARDING_PAYLOAD_SCHEMA, - "payload": payload.model_dump_for_forwarding(), - } - ) - - -def rowless_actual_wire_fingerprint(request_text: str) -> str: - """Bind the exact serialized response.create bytes sent upstream.""" - - digest = hashlib.sha256() - digest.update(b"qk-http-bridge-rowless-actual-wire-v1\0") - digest.update(request_text.encode("utf-8")) - return digest.hexdigest() - - -def rowless_projected_actual_wire_fingerprint( - request_text: str, - projected_input: list[JsonValue], -) -> str: - """Hash the exact transformed first wire after applying the approved rebase projection.""" - - return rowless_actual_wire_fingerprint(rowless_projected_actual_wire_text(request_text, projected_input)) - - -def rowless_projected_actual_wire_text( - request_text: str, - projected_input: list[JsonValue], -) -> str: - """Return the exact anchor-free wire retained only by the live request.""" - - payload = json.loads(request_text) - if not isinstance(payload, dict): - raise ValueError("rowless wire payload must be an object") - payload["input"] = projected_input - payload.pop("previous_response_id", None) - return json.dumps(payload, ensure_ascii=True, separators=(",", ":")) - - -def _contains_transformable_external_image(value: JsonValue) -> bool: - if isinstance(value, list): - return any(_contains_transformable_external_image(item) for item in value) - if not isinstance(value, dict): - return False - if value.get("type") == "input_image": - image_url = value.get("image_url") - if isinstance(image_url, str) and image_url.lower().startswith(("http://", "https://")): - return True - return any(_contains_transformable_external_image(item) for item in value.values()) - - -def build_rowless_recovery_capture_facts( - payload: ResponsesRequest, - *, - expected_session_identity: str | None = None, - expected_task_identity: str | None = None, -) -> RowlessRecoveryCaptureFacts | None: - if not isinstance(payload.input, list) or not payload.input: - return None - input_items = cast(list[JsonValue], payload.input) - if _contains_transformable_external_image(input_items): - return None - ledger = responses_direct_call_ledger_summary(input_items) - if ledger is None: - return None - projection = project_responses_input_for_account_neutral_fresh_replay( - input_items, - stored_count=len(input_items), - ) - evidence_projection = project_responses_input_for_account_neutral_fresh_replay( - input_items, - stored_count=len(input_items), - preserve_response_owned_agent_message_ids=True, - ) - if projection is None or evidence_projection is None: - return None - projected_input = normalize_responses_input_for_rowless_replay(projection.input_items) - evidence_input = normalize_responses_input_for_rowless_replay(evidence_projection.input_items) - if projected_input is None or evidence_input is None: - return None - projected_payload = payload.model_copy(update={"input": projected_input, "previous_response_id": None}) - self_contained = responses_input_items_are_self_contained_rowless_replay( - input_items, - projected_input, - ) - # The wire projection strips response-owned IDs. Keep them only in this - # classification copy so retained agent output can still be proven. - retains_prior_output = responses_input_retains_prior_output_and_fresh_followup(evidence_input) - account_neutral_input = [ - item for item in projected_input if not (isinstance(item, dict) and item.get("type") == "agent_message") - ] - account_neutral_payload = projected_payload.model_copy(update={"input": account_neutral_input}) - return RowlessRecoveryCaptureFacts( - input_item_count=len(input_items), - input_fingerprint=canonical_json_sha256(input_items), - contract_fingerprint=responses_non_input_contract_fingerprint(payload), - direct_call_ledger_digest=ledger.digest, - projected_payload_fingerprint=rowless_forwarding_payload_fingerprint(projected_payload), - actual_wire_fingerprint=rowless_forwarding_payload_fingerprint(projected_payload), - unresolved_count=ledger.unresolved_count, - projected_input=projected_input, - self_contained=self_contained, - account_neutral=( - self_contained - and responses_payload_is_account_neutral_fresh_replay( - account_neutral_payload.to_replay_safety_payload(), - expected_session_identity=expected_session_identity, - expected_task_identity=expected_task_identity, - ) - ), - retains_prior_output=retains_prior_output, - ) - - -def approved_rowless_recovery_projection( - payload: ResponsesRequest, - *, - captured_input_item_count: int, - captured_input_fingerprint: str, - non_input_contract_fingerprint: str, - direct_call_ledger_digest: str, - projected_payload_fingerprint: str, -) -> list[JsonValue] | None: - """Return the one safe anchor-free projection or fail closed.""" - - if not isinstance(payload.input, list): - return None - input_items = cast(list[JsonValue], payload.input) - if captured_input_item_count <= 0 or len(input_items) != captured_input_item_count: - return None - if canonical_json_sha256(input_items[:captured_input_item_count]) != captured_input_fingerprint: - return None - if responses_non_input_contract_fingerprint(payload) != non_input_contract_fingerprint: - return None - ledger = responses_direct_call_ledger_summary(input_items[:captured_input_item_count]) - if ledger is None or ledger.unresolved_count != 0 or ledger.digest != direct_call_ledger_digest: - return None - projection = project_responses_input_for_account_neutral_fresh_replay( - input_items, - stored_count=captured_input_item_count, - ) - if projection is None: - return None - projected_input = normalize_responses_input_for_rowless_replay(projection.input_items) - if projected_input is None or not responses_input_items_are_self_contained_rowless_replay( - input_items, - projected_input, - ): - return None - projected_payload = payload.model_copy(update={"input": projected_input, "previous_response_id": None}) - if rowless_forwarding_payload_fingerprint(projected_payload) != projected_payload_fingerprint: - return None - return projected_input diff --git a/app/modules/proxy/rowless_recovery_api.py b/app/modules/proxy/rowless_recovery_api.py deleted file mode 100644 index 4510dc9ec7..0000000000 --- a/app/modules/proxy/rowless_recovery_api.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Authenticated dashboard control plane for rowless semantic rebases.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Literal - -from fastapi import APIRouter, Depends, HTTPException, Request, status -from pydantic import BaseModel, ConfigDict, Field - -from app.core.auth.dashboard_access import DashboardPrincipal -from app.core.auth.dashboard_mode import DashboardAuthMode -from app.core.auth.dependencies import require_dashboard_admin_access, set_dashboard_error_format -from app.db.session import SessionLocal -from app.modules.proxy.rowless_recovery import canonical_json_sha256 -from app.modules.proxy.rowless_recovery_repository import ( - RowlessCheckpointReceipt, - RowlessRecoveryRepository, - RowlessRecoveryStateError, -) - -router = APIRouter( - prefix="/api/http-bridge/rowless-recovery", - tags=["dashboard"], - dependencies=[Depends(set_dashboard_error_format)], -) - - -class ChallengeRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - generation: int = Field(ge=1) - - -class ApproveRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - generation: int = Field(ge=1) - acknowledgement: Literal["operator_acknowledged_semantic_rebase"] - challenge: str = Field(min_length=32, max_length=512) - receipt_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") - receipt: RowlessCheckpointReceipt - - -async def require_authenticated_rebase_admin(request: Request) -> DashboardPrincipal: - principal = await require_dashboard_admin_access(request) - if principal.auth_mode != DashboardAuthMode.TRUSTED_HEADER: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="A trusted proxy operator is required for semantic rebase approval", - ) - if not (principal.actor or "").strip(): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="A trusted proxy operator identity is required for semantic rebase approval", - ) - return principal - - -@router.get("") -async def list_rowless_authorities( - principal: DashboardPrincipal = Depends(require_authenticated_rebase_admin), -) -> list[dict[str, object]]: - del principal - async with SessionLocal() as session: - authorities = await RowlessRecoveryRepository(session).list_authorities() - return [ - { - "id": item.id, - "state": item.state.value, - "generation": item.generation, - "strongSessionHash": item.strong_session_hash, - "taskAuthorityDigest": item.captured_task_authority_digest, - "staleAnchorHash": item.stale_anchor_hash, - "markerOriginBound": item.origin_marker_session_id is not None, - "authorizationMode": item.authorization_mode, - "inputItemCount": item.captured_input_item_count, - "createdAt": item.created_at, - "updatedAt": item.updated_at, - } - for item in authorities - ] - - -@router.get("/status") -async def rowless_recovery_status( - principal: DashboardPrincipal = Depends(require_authenticated_rebase_admin), -) -> dict[str, object]: - del principal - async with SessionLocal() as session: - repository = RowlessRecoveryRepository(session) - counts = await repository.authority_state_counts() - marker_bound_counts = await repository.authority_state_counts(marker_bound_only=True) - active_automatic_count = await repository.active_automatic_authority_count() - replay_fence_count = counts["approved"] + counts["unknown"] + counts["consumed"] - marker_bound_count = sum(marker_bound_counts.values()) - return { - "stateCounts": counts, - "markerBoundStateCounts": marker_bound_counts, - "replayFenceCount": replay_fence_count, - "markerBoundAuthorityCount": marker_bound_count, - "activeAutomaticAuthorityCount": active_automatic_count, - "preRowlessImageCompatible": replay_fence_count == 0 and marker_bound_count == 0, - "preMarkerRecoveryImageCompatible": marker_bound_count == 0, - "preAutomaticRecoveryImageCompatible": active_automatic_count == 0, - "minimumRollbackCapability": ( - "rowless_automatic_recovery_v3" - if active_automatic_count - else ( - "rowless_marker_recovery_v2" - if marker_bound_count - else ("rowless_recovery_v1" if replay_fence_count else None) - ) - ), - } - - -@router.post("/{authority_id}/challenge") -async def issue_rowless_challenge( - authority_id: str, - body: ChallengeRequest, - principal: DashboardPrincipal = Depends(require_authenticated_rebase_admin), -) -> dict[str, str | int | bool | datetime]: - del principal - try: - async with SessionLocal() as session: - challenge = await RowlessRecoveryRepository(session).issue_challenge( - authority_id=authority_id, - generation=body.generation, - ) - except RowlessRecoveryStateError as exc: - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc - return { - "authorityId": challenge.authority.id, - "generation": challenge.authority.generation, - "challenge": challenge.challenge, - "expiresAt": challenge.expires_at, - "strongSessionHash": challenge.authority.strong_session_hash, - "taskAuthorityDigest": challenge.authority.captured_task_authority_digest, - "capturedInputItemCount": challenge.authority.captured_input_item_count, - "capturedInputFingerprint": challenge.authority.captured_input_fingerprint, - "nonInputContractFingerprint": challenge.authority.non_input_contract_fingerprint, - "retainedRequestLedgerDigest": challenge.authority.settled_direct_call_ledger_digest, - "projectedPayloadFingerprint": challenge.authority.projected_payload_fingerprint, - "actualWireFingerprint": challenge.authority.actual_wire_fingerprint, - "retainedUnresolvedCount": challenge.authority.settled_direct_call_unresolved_count, - "requestSelfContained": challenge.authority.request_self_contained, - "requestAccountNeutral": challenge.authority.request_account_neutral, - "selectedAccountIntentHash": canonical_json_sha256(challenge.authority.selected_account_intent), - } - - -@router.post("/{authority_id}/approve") -async def approve_rowless_recovery( - authority_id: str, - body: ApproveRequest, - request: Request, - principal: DashboardPrincipal = Depends(require_authenticated_rebase_admin), -) -> dict[str, str | int]: - actor = principal.actor or "verified_standard_dashboard_admin" - try: - async with SessionLocal() as session: - approved = await RowlessRecoveryRepository(session).approve( - authority_id=authority_id, - generation=body.generation, - challenge=body.challenge, - declared_receipt_sha256=body.receipt_sha256, - receipt=body.receipt, - acknowledgement=body.acknowledgement, - approved_actor=actor, - request_id=getattr(request.state, "request_id", None), - ) - except RowlessRecoveryStateError as exc: - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc - return {"authorityId": approved.id, "generation": approved.generation, "state": approved.state.value} diff --git a/app/modules/proxy/rowless_recovery_repository.py b/app/modules/proxy/rowless_recovery_repository.py deleted file mode 100644 index cda76b27c8..0000000000 --- a/app/modules/proxy/rowless_recovery_repository.py +++ /dev/null @@ -1,1708 +0,0 @@ -"""Durable lifecycle for an operator-acknowledged rowless semantic rebase.""" - -from __future__ import annotations - -import asyncio -import json -import re -import secrets -from dataclasses import dataclass -from datetime import datetime, timedelta -from hashlib import sha256 -from typing import TypeVar - -from sqlalchemy import delete, func, select, text -from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncSession - -from app.core.clients.proxy_websocket import UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE -from app.core.utils.time import to_utc_naive, utcnow -from app.db.models import ( - AuditLog, - HttpBridgeRecoveryAttemptRecord, - HttpBridgeRecoveryAttemptState, - HttpBridgeRowlessRecoveryAuthority, - HttpBridgeRowlessRecoveryState, - HttpBridgeSessionRecord, -) -from app.db.session import sqlite_writer_section -from app.modules.proxy.durable_bridge_repository import ( - DurableBridgeRepository, - _encode_pending_tool_calls, - durable_bridge_hash, -) -from app.modules.proxy.response_transition_manifest import ( - ResponseTransitionManifest, - encode_response_transition_manifest, -) -from app.modules.proxy.rowless_recovery import ( - ROWLESS_AUTHORIZATION_MODE_AUTOMATIC, - ROWLESS_AUTHORIZATION_MODE_OPERATOR, - ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - RowlessRecoveryCaptureFacts, - canonical_json_sha256, -) - -ROWLESS_RECOVERY_CHALLENGE_TTL_SECONDS = 900 -ROWLESS_RECOVERY_CAPTURED_RETENTION_SECONDS = 7 * 24 * 3600 -ROWLESS_RECOVERY_MAX_CAPTURED_PER_API_SCOPE = 100 -_TaskResultT = TypeVar("_TaskResultT") - - -async def _await_repository_task_deferring_cancellation( - task: asyncio.Task[_TaskResultT], -) -> tuple[_TaskResultT, asyncio.CancelledError | None]: - """Finish one transaction-critical task while retaining cancellation.""" - - cancellation: asyncio.CancelledError | None = None - while True: - try: - return await asyncio.shield(task), cancellation - except asyncio.CancelledError as exc: - if task.cancelled(): - raise - cancellation = cancellation or exc - - -class RowlessRecoveryConflictError(RuntimeError): - """The durable authority exists but does not bind the same contract.""" - - -class RowlessRecoveryStateError(RuntimeError): - """A fail-closed lifecycle precondition did not match.""" - - -async def _flush_automatic_claim_or_reject(session: AsyncSession) -> None: - """Map every pre-commit uniqueness race to one stable proof rejection.""" - - try: - await session.flush() - except IntegrityError as exc: - await session.rollback() - raise RowlessRecoveryStateError("automatic_live_request_claim_conflict") from exc - - -@dataclass(frozen=True, slots=True) -class RowlessRecoveryAuthoritySnapshot: - id: str - api_key_scope: str - session_key_kind: str - strong_session_hash: str - stale_anchor_hash: str - generation: int - generation_nonce: str - state: HttpBridgeRowlessRecoveryState - captured_input_item_count: int - captured_input_fingerprint: str - non_input_contract_fingerprint: str - settled_direct_call_ledger_digest: str - projected_payload_fingerprint: str - actual_wire_fingerprint: str - origin_marker_session_id: str | None - settled_direct_call_unresolved_count: int - selected_account_intent: str - captured_task_identity_hash: str - captured_session_identity_hash: str - captured_task_authority_digest: str - request_self_contained: bool - request_account_neutral: bool - checkpoint_receipt_sha256: str | None - authorization_mode: str | None - authorization_proof_sha256: str | None - replacement_session_id: str | None - dispatch_request_id: str | None - dispatch_send_started_at: datetime | None - wire_request_fingerprint: str | None - created_at: datetime - updated_at: datetime - - -@dataclass(frozen=True, slots=True) -class RowlessRecoveryChallenge: - authority: RowlessRecoveryAuthoritySnapshot - challenge: str - expires_at: datetime - - -@dataclass(frozen=True, slots=True) -class RowlessCheckpointReceipt: - schema: str - remote_session_jsonl_sha256: str - remote_session_jsonl_size_bytes: int - remote_session_jsonl_last_offset: int - full_checkpoint_tool_ledger_digest: str - unresolved_count: int - task_identity: str - session_identity: str - strong_session_hash: str - task_authority_digest: str - captured_input_item_count: int - captured_input_fingerprint: str - non_input_contract_fingerprint: str - retained_request_direct_call_ledger_digest: str - captured_projected_payload_fingerprint: str - captured_actual_wire_fingerprint: str - captured_request_binding_provenance: str - - def canonical_payload(self) -> dict[str, str | int]: - return { - "full_checkpoint_tool_ledger_digest": self.full_checkpoint_tool_ledger_digest, - "remote_session_jsonl_last_offset": self.remote_session_jsonl_last_offset, - "remote_session_jsonl_sha256": self.remote_session_jsonl_sha256, - "remote_session_jsonl_size_bytes": self.remote_session_jsonl_size_bytes, - "schema": self.schema, - "session_identity": self.session_identity, - "strong_session_hash": self.strong_session_hash, - "task_identity": self.task_identity, - "task_authority_digest": self.task_authority_digest, - "captured_input_item_count": self.captured_input_item_count, - "captured_input_fingerprint": self.captured_input_fingerprint, - "non_input_contract_fingerprint": self.non_input_contract_fingerprint, - "retained_request_direct_call_ledger_digest": self.retained_request_direct_call_ledger_digest, - "captured_projected_payload_fingerprint": self.captured_projected_payload_fingerprint, - "captured_actual_wire_fingerprint": self.captured_actual_wire_fingerprint, - "captured_request_binding_provenance": self.captured_request_binding_provenance, - "unresolved_count": self.unresolved_count, - } - - def sha256(self) -> str: - return canonical_json_sha256(self.canonical_payload()) - - -class RowlessRecoveryRepository: - """Own the non-cascading semantic-rebase authority and its CAS fences.""" - - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def capture( - self, - *, - api_key_scope: str, - session_key_kind: str, - strong_session_hash: str, - stale_anchor_hash: str, - selected_account_intent: str, - task_identity: str, - session_identity: str, - task_authority_digest: str, - facts: RowlessRecoveryCaptureFacts, - origin_marker_session_id: str | None = None, - ) -> RowlessRecoveryAuthoritySnapshot: - if ( - not api_key_scope - or not session_key_kind - or not selected_account_intent - or not task_identity - or not session_identity - or not _is_sha256(task_authority_digest) - ): - raise ValueError("rowless recovery identity must be complete") - generation_nonce = secrets.token_hex(32) - # A rowless reject proves only which account rejected the stale - # anchor. It is not proof that this account owned the now-purged - # historical anchor. Only an account-neutral self-contained request - # can therefore enter this operator rebase flow. - if not facts.self_contained or not facts.account_neutral: - raise RowlessRecoveryStateError("rowless_request_not_account_neutral") - async with sqlite_writer_section(): - if self._session.get_bind().dialect.name == "postgresql": - await self._session.execute( - text("SELECT pg_advisory_xact_lock(hashtext(:key))"), - {"key": f"http-bridge-rowless-capture:{api_key_scope}"}, - ) - existing = await self._find_for_update( - api_key_scope=api_key_scope, - strong_session_hash=strong_session_hash, - stale_anchor_hash=stale_anchor_hash, - ) - if existing is not None: - self._require_same_capture( - existing, - selected_account_intent, - task_identity, - session_identity, - task_authority_digest, - facts, - origin_marker_session_id, - ) - snapshot = _snapshot(existing) - await self._session.rollback() - return snapshot - existing = await self._find_exact_request_contract_for_update( - api_key_scope=api_key_scope, - strong_session_hash=strong_session_hash, - facts=facts, - ) - if existing is not None: - self._require_same_capture( - existing, - selected_account_intent, - task_identity, - session_identity, - task_authority_digest, - facts, - origin_marker_session_id, - ) - snapshot = _snapshot(existing) - await self._session.rollback() - return snapshot - captured_count = await self._session.scalar( - select(func.count()) - .select_from(HttpBridgeRowlessRecoveryAuthority) - .where( - HttpBridgeRowlessRecoveryAuthority.api_key_scope == api_key_scope, - HttpBridgeRowlessRecoveryAuthority.state == HttpBridgeRowlessRecoveryState.CAPTURED, - ) - ) - if int(captured_count or 0) >= ROWLESS_RECOVERY_MAX_CAPTURED_PER_API_SCOPE: - await self._session.rollback() - raise RowlessRecoveryStateError("rowless_capture_scope_limit_reached") - if origin_marker_session_id is not None: - await self._require_live_origin_marker( - session_id=origin_marker_session_id, - api_key_scope=api_key_scope, - selected_account_intent=selected_account_intent, - stale_anchor_hash=stale_anchor_hash, - ) - row = HttpBridgeRowlessRecoveryAuthority( - api_key_scope=api_key_scope, - session_key_kind=session_key_kind, - strong_session_hash=strong_session_hash, - stale_anchor_hash=stale_anchor_hash, - generation=1, - generation_nonce=generation_nonce, - state=HttpBridgeRowlessRecoveryState.CAPTURED, - captured_input_item_count=facts.input_item_count, - captured_input_fingerprint=facts.input_fingerprint, - non_input_contract_fingerprint=facts.contract_fingerprint, - settled_direct_call_ledger_digest=facts.direct_call_ledger_digest, - projected_payload_fingerprint=facts.projected_payload_fingerprint, - actual_wire_fingerprint=facts.actual_wire_fingerprint, - origin_marker_session_id=origin_marker_session_id, - settled_direct_call_unresolved_count=facts.unresolved_count, - selected_account_intent=selected_account_intent, - captured_task_identity_hash=canonical_json_sha256(task_identity), - captured_session_identity_hash=canonical_json_sha256(session_identity), - captured_task_authority_digest=task_authority_digest, - request_self_contained=facts.self_contained, - request_account_neutral=facts.account_neutral, - ) - self._session.add(row) - try: - await self._session.commit() - except IntegrityError: - await self._session.rollback() - existing = await self._find_for_update( - api_key_scope=api_key_scope, - strong_session_hash=strong_session_hash, - stale_anchor_hash=stale_anchor_hash, - ) - if existing is None: - existing = await self._find_exact_request_contract_for_update( - api_key_scope=api_key_scope, - strong_session_hash=strong_session_hash, - facts=facts, - ) - if existing is None: - raise - self._require_same_capture( - existing, - selected_account_intent, - task_identity, - session_identity, - task_authority_digest, - facts, - origin_marker_session_id, - ) - snapshot = _snapshot(existing) - await self._session.rollback() - return snapshot - await self._session.refresh(row) - return _snapshot(row) - - async def get(self, authority_id: str) -> RowlessRecoveryAuthoritySnapshot | None: - row = await self._session.get(HttpBridgeRowlessRecoveryAuthority, authority_id) - return _snapshot(row) if row is not None else None - - async def lookup( - self, - *, - api_key_scope: str, - strong_session_hash: str, - stale_anchor_hash: str, - ) -> RowlessRecoveryAuthoritySnapshot | None: - row = await self._session.scalar( - select(HttpBridgeRowlessRecoveryAuthority).where( - HttpBridgeRowlessRecoveryAuthority.api_key_scope == api_key_scope, - HttpBridgeRowlessRecoveryAuthority.strong_session_hash == strong_session_hash, - HttpBridgeRowlessRecoveryAuthority.stale_anchor_hash == stale_anchor_hash, - ) - ) - return _snapshot(row) if row is not None else None - - async def lookup_stale_anchor_in_scope( - self, - *, - api_key_scope: str, - stale_anchor_hash: str, - ) -> RowlessRecoveryAuthoritySnapshot | None: - """Resolve an anchor fence even when incoming task headers are incomplete.""" - - rows = list( - ( - await self._session.scalars( - select(HttpBridgeRowlessRecoveryAuthority) - .where( - HttpBridgeRowlessRecoveryAuthority.api_key_scope == api_key_scope, - HttpBridgeRowlessRecoveryAuthority.stale_anchor_hash == stale_anchor_hash, - ) - .limit(2) - ) - ).all() - ) - if len(rows) > 1: - raise RowlessRecoveryStateError("rowless_anchor_authority_ambiguous") - return _snapshot(rows[0]) if rows else None - - async def lookup_exact_request_contract( - self, - *, - api_key_scope: str, - strong_session_hash: str, - facts: RowlessRecoveryCaptureFacts, - ) -> RowlessRecoveryAuthoritySnapshot | None: - """Resolve one task-bound authority without trusting the incoming anchor.""" - - rows = list( - ( - await self._session.scalars( - select(HttpBridgeRowlessRecoveryAuthority) - .where( - HttpBridgeRowlessRecoveryAuthority.api_key_scope == api_key_scope, - HttpBridgeRowlessRecoveryAuthority.strong_session_hash == strong_session_hash, - HttpBridgeRowlessRecoveryAuthority.captured_input_item_count == facts.input_item_count, - HttpBridgeRowlessRecoveryAuthority.captured_input_fingerprint == facts.input_fingerprint, - HttpBridgeRowlessRecoveryAuthority.non_input_contract_fingerprint == facts.contract_fingerprint, - HttpBridgeRowlessRecoveryAuthority.settled_direct_call_ledger_digest - == facts.direct_call_ledger_digest, - HttpBridgeRowlessRecoveryAuthority.projected_payload_fingerprint - == facts.projected_payload_fingerprint, - ) - .order_by(HttpBridgeRowlessRecoveryAuthority.created_at.asc()) - .limit(2) - ) - ).all() - ) - if len(rows) > 1: - raise RowlessRecoveryStateError("rowless_request_authority_ambiguous") - return _snapshot(rows[0]) if rows else None - - async def list_authorities( - self, - *, - states: tuple[HttpBridgeRowlessRecoveryState, ...] = ( - HttpBridgeRowlessRecoveryState.CAPTURED, - HttpBridgeRowlessRecoveryState.APPROVED, - HttpBridgeRowlessRecoveryState.UNKNOWN, - ), - limit: int = 100, - ) -> list[RowlessRecoveryAuthoritySnapshot]: - rows = ( - await self._session.scalars( - select(HttpBridgeRowlessRecoveryAuthority) - .where(HttpBridgeRowlessRecoveryAuthority.state.in_(states)) - .order_by(HttpBridgeRowlessRecoveryAuthority.updated_at.desc()) - .limit(max(1, min(limit, 500))) - ) - ).all() - return [_snapshot(row) for row in rows] - - async def authority_state_counts(self, *, marker_bound_only: bool = False) -> dict[str, int]: - """Return content-free counts used to enforce the image rollback floor.""" - - statement = select( - HttpBridgeRowlessRecoveryAuthority.state, - func.count(HttpBridgeRowlessRecoveryAuthority.id), - ) - if marker_bound_only: - statement = statement.where(HttpBridgeRowlessRecoveryAuthority.origin_marker_session_id.is_not(None)) - statement = statement.group_by(HttpBridgeRowlessRecoveryAuthority.state) - rows = (await self._session.execute(statement)).all() - counts = {state.value: 0 for state in HttpBridgeRowlessRecoveryState} - for state, count in rows: - counts[state.value] = int(count) - return counts - - async def active_automatic_authority_count(self) -> int: - """Return automatic authorities that require v3 dispatch semantics.""" - - count = await self._session.scalar( - select(func.count(HttpBridgeRowlessRecoveryAuthority.id)).where( - HttpBridgeRowlessRecoveryAuthority.authorization_mode == ROWLESS_AUTHORIZATION_MODE_AUTOMATIC, - HttpBridgeRowlessRecoveryAuthority.state.in_( - ( - HttpBridgeRowlessRecoveryState.APPROVED, - HttpBridgeRowlessRecoveryState.UNKNOWN, - ) - ), - ) - ) - return int(count or 0) - - async def purge_expired_audit_rows( - self, - *, - captured_cutoff: datetime, - batch_size: int = 100, - ) -> dict[str, int]: - """Bound abandoned captures while retaining all replay tombstones.""" - - counts = {"captured": 0} - policies = ( - ( - "captured", - HttpBridgeRowlessRecoveryAuthority.state == HttpBridgeRowlessRecoveryState.CAPTURED, - HttpBridgeRowlessRecoveryAuthority.checkpoint_receipt_sha256.is_(None), - HttpBridgeRowlessRecoveryAuthority.updated_at < captured_cutoff, - ), - ) - for label, *filters in policies: - while True: - ids = list( - ( - await self._session.scalars( - select(HttpBridgeRowlessRecoveryAuthority.id) - .where(*filters) - .order_by(HttpBridgeRowlessRecoveryAuthority.updated_at.asc()) - .limit(max(1, min(batch_size, 500))) - ) - ).all() - ) - if not ids: - break - async with sqlite_writer_section(): - deleted = await self._session.execute( - delete(HttpBridgeRowlessRecoveryAuthority) - .where(HttpBridgeRowlessRecoveryAuthority.id.in_(ids), *filters) - .returning(HttpBridgeRowlessRecoveryAuthority.id) - ) - await self._session.commit() - counts[label] += len(deleted.scalars().all()) - return counts - - async def issue_challenge( - self, - *, - authority_id: str, - generation: int, - ) -> RowlessRecoveryChallenge: - challenge = secrets.token_urlsafe(32) - challenge_hash = sha256(challenge.encode("utf-8")).hexdigest() - expires_at = utcnow() + timedelta(seconds=ROWLESS_RECOVERY_CHALLENGE_TTL_SECONDS) - async with sqlite_writer_section(): - row = await self._find_id_for_update(authority_id) - if row is None or row.generation != generation or row.state != HttpBridgeRowlessRecoveryState.CAPTURED: - await self._session.rollback() - raise RowlessRecoveryStateError("captured_generation_not_found") - row.challenge_nonce_hash = challenge_hash - row.challenge_expires_at = expires_at - await self._session.commit() - await self._session.refresh(row) - return RowlessRecoveryChallenge(authority=_snapshot(row), challenge=challenge, expires_at=expires_at) - - async def approve( - self, - *, - authority_id: str, - generation: int, - challenge: str, - declared_receipt_sha256: str, - receipt: RowlessCheckpointReceipt, - acknowledgement: str, - approved_actor: str, - request_id: str | None, - ) -> RowlessRecoveryAuthoritySnapshot: - if acknowledgement != ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT: - raise RowlessRecoveryStateError("semantic_rebase_acknowledgement_required") - receipt_sha256 = receipt.sha256() - if receipt_sha256 != declared_receipt_sha256: - raise RowlessRecoveryStateError("checkpoint_receipt_digest_mismatch") - if ( - receipt.schema != "qk_http_bridge_rowless_checkpoint_receipt_v1" - or receipt.unresolved_count != 0 - or receipt.remote_session_jsonl_size_bytes <= 0 - or receipt.remote_session_jsonl_last_offset != receipt.remote_session_jsonl_size_bytes - or receipt.strong_session_hash == "" - or not _is_sha256(receipt.remote_session_jsonl_sha256) - or not _is_sha256(receipt.full_checkpoint_tool_ledger_digest) - or not _is_sha256(receipt.strong_session_hash) - or not _is_sha256(receipt.task_authority_digest) - or receipt.captured_input_item_count <= 0 - or not _is_sha256(receipt.captured_input_fingerprint) - or not _is_sha256(receipt.non_input_contract_fingerprint) - or not _is_sha256(receipt.retained_request_direct_call_ledger_digest) - or not _is_sha256(receipt.captured_projected_payload_fingerprint) - or not _is_sha256(receipt.captured_actual_wire_fingerprint) - or receipt.captured_request_binding_provenance != "server_challenge" - or not receipt.task_identity.strip() - or not receipt.session_identity.strip() - ): - raise RowlessRecoveryStateError("checkpoint_receipt_invalid") - now = utcnow() - async with sqlite_writer_section(): - row = await self._find_id_for_update(authority_id) - if ( - row is None - or row.generation != generation - or row.state != HttpBridgeRowlessRecoveryState.CAPTURED - or row.challenge_nonce_hash is None - or row.challenge_expires_at is None - or not secrets.compare_digest( - row.challenge_nonce_hash, - sha256(challenge.encode("utf-8")).hexdigest(), - ) - or to_utc_naive(row.challenge_expires_at) < to_utc_naive(now) - ): - await self._session.rollback() - raise RowlessRecoveryStateError("challenge_or_generation_invalid") - if ( - row.settled_direct_call_unresolved_count != 0 - or not row.request_self_contained - or not row.request_account_neutral - or receipt.strong_session_hash != row.strong_session_hash - or receipt.task_authority_digest != row.captured_task_authority_digest - or canonical_json_sha256(receipt.task_identity) != row.captured_task_identity_hash - or canonical_json_sha256(receipt.session_identity) != row.captured_session_identity_hash - or receipt.captured_input_item_count != row.captured_input_item_count - or receipt.captured_input_fingerprint != row.captured_input_fingerprint - or receipt.non_input_contract_fingerprint != row.non_input_contract_fingerprint - or receipt.retained_request_direct_call_ledger_digest != row.settled_direct_call_ledger_digest - or receipt.captured_projected_payload_fingerprint != row.projected_payload_fingerprint - or receipt.captured_actual_wire_fingerprint != row.actual_wire_fingerprint - ): - await self._session.rollback() - raise RowlessRecoveryStateError("checkpoint_receipt_contract_mismatch") - row.state = HttpBridgeRowlessRecoveryState.APPROVED - row.authorization_mode = ROWLESS_AUTHORIZATION_MODE_OPERATOR - row.authorization_proof_sha256 = receipt_sha256 - row.checkpoint_receipt_sha256 = receipt_sha256 - row.checkpoint_jsonl_sha256 = receipt.remote_session_jsonl_sha256 - row.checkpoint_jsonl_size_bytes = receipt.remote_session_jsonl_size_bytes - row.checkpoint_jsonl_last_offset = receipt.remote_session_jsonl_last_offset - row.checkpoint_task_identity_hash = canonical_json_sha256(receipt.task_identity) - row.checkpoint_session_identity_hash = canonical_json_sha256(receipt.session_identity) - row.checkpoint_strong_session_hash = receipt.strong_session_hash - row.checkpoint_task_authority_digest = receipt.task_authority_digest - row.checkpoint_tool_ledger_digest = receipt.full_checkpoint_tool_ledger_digest - row.approved_by_actor = canonical_json_sha256(approved_actor) - row.approved_at = now - row.challenge_nonce_hash = None - row.challenge_expires_at = None - self._session.add( - AuditLog( - action="http_bridge_rowless_semantic_rebase_approved", - details=json.dumps( - { - "authority_id": row.id, - "generation": row.generation, - "receipt_sha256": receipt_sha256, - "actor_hash": row.approved_by_actor, - }, - separators=(",", ":"), - sort_keys=True, - ), - request_id=request_id, - ) - ) - await self._session.commit() - await self._session.refresh(row) - return _snapshot(row) - - async def capture_and_claim_automatic_preflight( - self, - *, - api_key_scope: str, - session_key_kind: str, - strong_session_hash: str, - stale_anchor_hash: str, - selected_account_intent: str, - task_identity: str, - session_identity: str, - task_authority_digest: str, - facts: RowlessRecoveryCaptureFacts, - request_id: str, - wire_request_fingerprint: str, - origin_marker_session_id: str | None = None, - expected_authority_id: str | None = None, - expected_generation: int | None = None, - ) -> RowlessRecoveryAuthoritySnapshot: - """Capture one official live request and claim its only safe dispatch.""" - - if ( - not api_key_scope - or not session_key_kind - or not selected_account_intent - or not task_identity - or not session_identity - or not request_id - or not _is_sha256(task_authority_digest) - or not _is_sha256(strong_session_hash) - or not _is_sha256(stale_anchor_hash) - or not _is_sha256(wire_request_fingerprint) - or facts.actual_wire_fingerprint != wire_request_fingerprint - or facts.unresolved_count != 0 - or not facts.self_contained - or not facts.account_neutral - or not facts.retains_prior_output - ): - raise RowlessRecoveryStateError("automatic_live_request_proof_invalid") - - async with sqlite_writer_section(): - if self._session.get_bind().dialect.name == "postgresql": - await self._session.execute( - text("SELECT pg_advisory_xact_lock(hashtext(:key))"), - {"key": f"http-bridge-rowless-auto:{api_key_scope}:{strong_session_hash}"}, - ) - row = await self._find_for_update( - api_key_scope=api_key_scope, - strong_session_hash=strong_session_hash, - stale_anchor_hash=stale_anchor_hash, - ) - if row is None: - row = await self._find_exact_request_contract_for_update( - api_key_scope=api_key_scope, - strong_session_hash=strong_session_hash, - facts=facts, - ) - if row is None: - if expected_authority_id is not None or expected_generation is not None: - await self._session.rollback() - raise RowlessRecoveryStateError("automatic_expected_generation_not_found") - if origin_marker_session_id is not None: - await self._require_live_origin_marker( - session_id=origin_marker_session_id, - api_key_scope=api_key_scope, - selected_account_intent=selected_account_intent, - stale_anchor_hash=stale_anchor_hash, - ) - row = HttpBridgeRowlessRecoveryAuthority( - api_key_scope=api_key_scope, - session_key_kind=session_key_kind, - strong_session_hash=strong_session_hash, - stale_anchor_hash=stale_anchor_hash, - generation=1, - generation_nonce=secrets.token_hex(32), - state=HttpBridgeRowlessRecoveryState.CAPTURED, - captured_input_item_count=facts.input_item_count, - captured_input_fingerprint=facts.input_fingerprint, - non_input_contract_fingerprint=facts.contract_fingerprint, - settled_direct_call_ledger_digest=facts.direct_call_ledger_digest, - projected_payload_fingerprint=facts.projected_payload_fingerprint, - actual_wire_fingerprint=facts.actual_wire_fingerprint, - origin_marker_session_id=origin_marker_session_id, - settled_direct_call_unresolved_count=facts.unresolved_count, - selected_account_intent=selected_account_intent, - captured_task_identity_hash=canonical_json_sha256(task_identity), - captured_session_identity_hash=canonical_json_sha256(session_identity), - captured_task_authority_digest=task_authority_digest, - request_self_contained=facts.self_contained, - request_account_neutral=facts.account_neutral, - ) - self._session.add(row) - await _flush_automatic_claim_or_reject(self._session) - else: - origin_marker = await self._load_origin_marker_for_update(row) - marker_journal_exists = False - if row.origin_marker_session_id is not None: - marker_journal_exists = ( - await self._session.scalar( - select(HttpBridgeRecoveryAttemptRecord.id) - .where( - HttpBridgeRecoveryAttemptRecord.session_id == row.origin_marker_session_id, - HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.UNKNOWN, - ) - .limit(1) - ) - ) is not None - if ( - row.stale_anchor_hash != stale_anchor_hash - or row.state - not in { - HttpBridgeRowlessRecoveryState.CAPTURED, - HttpBridgeRowlessRecoveryState.APPROVED, - } - or row.dispatch_request_id is not None - or row.replacement_session_id is not None - or row.dispatch_send_started_at is not None - or row.wire_request_fingerprint is not None - or row.consumed_at is not None - or marker_journal_exists - or row.selected_account_intent != selected_account_intent - or row.origin_marker_session_id != origin_marker_session_id - or row.captured_task_identity_hash != canonical_json_sha256(task_identity) - or row.captured_session_identity_hash != canonical_json_sha256(session_identity) - or row.captured_task_authority_digest != task_authority_digest - or (expected_authority_id is not None and row.id != expected_authority_id) - or (expected_generation is not None and row.generation != expected_generation) - or ( - row.origin_marker_session_id is not None - and not self._origin_marker_matches_authority(origin_marker, row) - ) - ): - await self._session.rollback() - raise RowlessRecoveryStateError("automatic_unsent_generation_fence_rejected") - if not _capture_matches(row, facts): - self._session.add( - AuditLog( - action="http_bridge_rowless_automatic_generation_superseded", - details=json.dumps( - { - "authority_id": row.id, - "generation": row.generation, - "state": row.state.value, - "authorization_mode": row.authorization_mode, - "authorization_proof_sha256": row.authorization_proof_sha256, - "checkpoint_receipt_sha256": row.checkpoint_receipt_sha256, - "captured_input_fingerprint": row.captured_input_fingerprint, - "projected_payload_fingerprint": row.projected_payload_fingerprint, - "actual_wire_fingerprint": row.actual_wire_fingerprint, - }, - separators=(",", ":"), - sort_keys=True, - ), - request_id=request_id, - ) - ) - row.generation += 1 - row.generation_nonce = secrets.token_hex(32) - _replace_capture(row, facts) - _clear_operator_authorization(row) - - self._require_same_capture( - row, - selected_account_intent, - task_identity, - session_identity, - task_authority_digest, - facts, - origin_marker_session_id, - ) - authorization_proof = _automatic_authorization_proof( - row, - wire_request_fingerprint=wire_request_fingerprint, - ) - if row.origin_marker_session_id is not None: - origin_marker = await self._load_origin_marker_for_update(row) - if not self._origin_marker_matches_authority(origin_marker, row): - await self._session.rollback() - raise RowlessRecoveryStateError("automatic_origin_marker_fence_rejected") - if origin_marker is None: # pragma: no cover - guarded above - raise AssertionError("automatic marker authority requires its origin") - origin_marker.recovery_required_attempt_fingerprint = _origin_marker_attempt_fingerprint( - row, - wire_request_fingerprint, - ) - origin_marker.recovery_required_attempt_request_id = request_id - row.authorization_mode = ROWLESS_AUTHORIZATION_MODE_AUTOMATIC - row.authorization_proof_sha256 = authorization_proof - row.state = HttpBridgeRowlessRecoveryState.UNKNOWN - row.dispatch_request_id = request_id - row.wire_request_fingerprint = wire_request_fingerprint - row.dispatch_send_started_at = None - self._session.add( - AuditLog( - action="http_bridge_rowless_automatic_live_request_claimed", - details=json.dumps( - { - "authority_id": row.id, - "generation": row.generation, - "authorization_proof_sha256": authorization_proof, - }, - separators=(",", ":"), - sort_keys=True, - ), - request_id=request_id, - ) - ) - try: - await _flush_automatic_claim_or_reject(self._session) - await self._session.refresh(row) - snapshot = _snapshot(row) - commit_task = asyncio.create_task(self._session.commit()) - _, cancellation = await _await_repository_task_deferring_cancellation(commit_task) - except IntegrityError as exc: - await self._session.rollback() - raise RowlessRecoveryStateError("automatic_live_request_claim_conflict") from exc - if cancellation is not None: - rollback_task = asyncio.create_task( - self.rollback_preflight_setup_failure( - authority_id=snapshot.id, - generation=snapshot.generation, - request_id=request_id, - wire_request_fingerprint=wire_request_fingerprint, - ) - ) - restored, _rollback_cancellation = await _await_repository_task_deferring_cancellation(rollback_task) - if not restored: - raise RowlessRecoveryStateError("cancelled_automatic_claim_restore_failed") from cancellation - raise cancellation - return snapshot - - async def claim_dispatch( - self, - *, - authority_id: str, - generation: int, - replacement_session_id: str, - request_id: str, - wire_request_fingerprint: str, - model: str | None, - task_authority_digest: str, - ) -> RowlessRecoveryAuthoritySnapshot: - """Bind a preflight-claimed authority to its durable replacement.""" - - async with sqlite_writer_section(): - row = await self._find_id_for_update(authority_id) - replacement = await self._session.scalar( - select(HttpBridgeSessionRecord) - .where(HttpBridgeSessionRecord.id == replacement_session_id) - .with_for_update() - ) - rejection = next( - ( - reason - for reason, rejected in ( - ("authority_missing", row is None), - ("generation", row is not None and row.generation != generation), - ("state", row is not None and row.state != HttpBridgeRowlessRecoveryState.UNKNOWN), - ( - "authorization", - row is not None - and not _dispatch_authorized( - row, - wire_request_fingerprint=wire_request_fingerprint, - ), - ), - ("request_id", row is not None and row.dispatch_request_id != request_id), - ( - "wire_fingerprint", - row is not None and row.wire_request_fingerprint != wire_request_fingerprint, - ), - ("replacement_already_bound", row is not None and row.replacement_session_id is not None), - ( - "task_authority", - row is not None and row.captured_task_authority_digest != task_authority_digest, - ), - ( - "origin_marker_session", - row is not None - and row.origin_marker_session_id is not None - and replacement is not None - and replacement.id != row.origin_marker_session_id, - ), - ( - "origin_marker_generation", - row is not None - and row.origin_marker_session_id is not None - and not self._origin_marker_matches_authority( - replacement, - row, - expected_attempt_fingerprint=_origin_marker_attempt_fingerprint( - row, - wire_request_fingerprint, - ), - expected_request_id=request_id, - ), - ), - ("replacement_missing", replacement is None), - ( - "api_scope", - row is not None - and replacement is not None - and replacement.api_key_scope != row.api_key_scope, - ), - ( - "account", - row is not None - and replacement is not None - and replacement.account_id != row.selected_account_intent, - ), - ) - if rejected - ), - None, - ) - if rejection is not None: - await self._session.rollback() - raise RowlessRecoveryStateError(f"approved_generation_dispatch_fence_rejected:{rejection}") - if row is None or replacement is None: # pragma: no cover - guarded above - raise AssertionError("validated dispatch rows are required") - existing = await self._session.scalar( - select(HttpBridgeRecoveryAttemptRecord).where( - HttpBridgeRecoveryAttemptRecord.session_id == replacement_session_id, - HttpBridgeRecoveryAttemptRecord.request_fingerprint == wire_request_fingerprint, - ) - ) - if existing is not None: - await self._session.rollback() - raise RowlessRecoveryStateError("dispatch_already_claimed") - self._session.add( - HttpBridgeRecoveryAttemptRecord( - session_id=replacement_session_id, - request_fingerprint=wire_request_fingerprint, - request_id=request_id, - account_id=row.selected_account_intent, - model=model, - replay_safe=False, - state=HttpBridgeRecoveryAttemptState.UNKNOWN, - ) - ) - row.replacement_session_id = replacement_session_id - try: - await self._session.commit() - except IntegrityError as exc: - await self._session.rollback() - raise RowlessRecoveryStateError("dispatch_already_claimed") from exc - await self._session.refresh(row) - return _snapshot(row) - - async def claim_dispatch_preflight( - self, - *, - authority_id: str, - generation: int, - request_id: str, - wire_request_fingerprint: str, - task_authority_digest: str, - ) -> RowlessRecoveryAuthoritySnapshot: - """CAS before account selection so concurrent losers never connect.""" - - async with sqlite_writer_section(): - row = await self._find_id_for_update(authority_id) - origin_marker = None - if row is not None and row.origin_marker_session_id is not None: - origin_marker = await self._session.scalar( - select(HttpBridgeSessionRecord) - .where(HttpBridgeSessionRecord.id == row.origin_marker_session_id) - .with_for_update() - ) - if ( - row is None - or row.generation != generation - or row.state != HttpBridgeRowlessRecoveryState.APPROVED - or not _dispatch_authorized( - row, - wire_request_fingerprint=wire_request_fingerprint, - ) - or row.captured_task_authority_digest != task_authority_digest - or ( - row.origin_marker_session_id is not None - and not self._origin_marker_matches_authority(origin_marker, row) - ) - ): - await self._session.rollback() - raise RowlessRecoveryStateError("approved_generation_preflight_fence_rejected") - if origin_marker is not None: - origin_marker.recovery_required_attempt_fingerprint = _origin_marker_attempt_fingerprint( - row, - wire_request_fingerprint, - ) - origin_marker.recovery_required_attempt_request_id = request_id - row.state = HttpBridgeRowlessRecoveryState.UNKNOWN - row.dispatch_request_id = request_id - row.wire_request_fingerprint = wire_request_fingerprint - row.dispatch_send_started_at = None - await self._session.commit() - await self._session.refresh(row) - return _snapshot(row) - - async def rollback_preflight_setup_failure( - self, - *, - authority_id: str, - generation: int, - request_id: str, - wire_request_fingerprint: str, - ) -> bool: - """Restore a claim only when setup failed before a replacement existed.""" - - async with sqlite_writer_section(): - row = await self._find_id_for_update(authority_id) - origin_marker = await self._load_origin_marker_for_update(row) - if ( - row is None - or row.generation != generation - or row.state != HttpBridgeRowlessRecoveryState.UNKNOWN - or row.dispatch_request_id != request_id - or row.wire_request_fingerprint != wire_request_fingerprint - or row.replacement_session_id is not None - or row.dispatch_send_started_at is not None - or ( - row.origin_marker_session_id is not None - and not self._origin_marker_matches_authority( - origin_marker, - row, - expected_attempt_fingerprint=_origin_marker_attempt_fingerprint( - row, - wire_request_fingerprint, - ), - expected_request_id=request_id, - ) - ) - ): - await self._session.rollback() - return False - journal_exists = await self._session.scalar( - select(HttpBridgeRecoveryAttemptRecord.id).where( - HttpBridgeRecoveryAttemptRecord.request_id == request_id, - HttpBridgeRecoveryAttemptRecord.request_fingerprint == wire_request_fingerprint, - ) - ) - if journal_exists is not None: - await self._session.rollback() - return False - row.state = HttpBridgeRowlessRecoveryState.APPROVED - row.dispatch_request_id = None - row.wire_request_fingerprint = None - if origin_marker is not None: - origin_marker.recovery_required_attempt_fingerprint = None - origin_marker.recovery_required_attempt_request_id = None - await self._session.commit() - return True - - async def mark_dispatch_send_started( - self, - *, - authority_id: str, - generation: int, - request_id: str, - wire_request_fingerprint: str, - ) -> bool: - """Durably close the only proven-unsent rollback window before send.""" - - async with sqlite_writer_section(): - row = await self._find_id_for_update(authority_id) - origin_marker = await self._load_origin_marker_for_update(row) - journal = None - if row is not None and row.replacement_session_id is not None: - journal = await self._session.scalar( - select(HttpBridgeRecoveryAttemptRecord) - .where( - HttpBridgeRecoveryAttemptRecord.session_id == row.replacement_session_id, - HttpBridgeRecoveryAttemptRecord.request_id == request_id, - HttpBridgeRecoveryAttemptRecord.request_fingerprint == wire_request_fingerprint, - ) - .with_for_update() - ) - if ( - row is None - or row.generation != generation - or row.state != HttpBridgeRowlessRecoveryState.UNKNOWN - or row.dispatch_request_id != request_id - or row.wire_request_fingerprint != wire_request_fingerprint - or row.dispatch_send_started_at is not None - or journal is None - or journal.state != HttpBridgeRecoveryAttemptState.UNKNOWN - or journal.response_id is not None - or ( - row.origin_marker_session_id is not None - and not self._origin_marker_matches_authority( - origin_marker, - row, - expected_attempt_fingerprint=_origin_marker_attempt_fingerprint( - row, - wire_request_fingerprint, - ), - expected_request_id=request_id, - ) - ) - ): - await self._session.rollback() - return False - row.dispatch_send_started_at = utcnow() - await self._session.commit() - return True - - async def rollback_proven_unsent( - self, - *, - authority_id: str, - generation: int, - request_id: str, - wire_request_fingerprint: str, - ) -> bool: - async with sqlite_writer_section(): - row = await self._find_id_for_update(authority_id) - origin_marker = await self._load_origin_marker_for_update(row) - if ( - row is None - or row.generation != generation - or row.state != HttpBridgeRowlessRecoveryState.UNKNOWN - or row.dispatch_request_id != request_id - or row.wire_request_fingerprint != wire_request_fingerprint - or row.dispatch_send_started_at is not None - or ( - row.origin_marker_session_id is not None - and not self._origin_marker_matches_authority( - origin_marker, - row, - expected_attempt_fingerprint=_origin_marker_attempt_fingerprint( - row, - wire_request_fingerprint, - ), - expected_request_id=request_id, - ) - ) - ): - await self._session.rollback() - return False - if row.replacement_session_id is None: - await self._session.rollback() - return False - deleted = await self._session.execute( - delete(HttpBridgeRecoveryAttemptRecord) - .where( - HttpBridgeRecoveryAttemptRecord.session_id == row.replacement_session_id, - HttpBridgeRecoveryAttemptRecord.request_fingerprint == wire_request_fingerprint, - HttpBridgeRecoveryAttemptRecord.request_id == request_id, - HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.UNKNOWN, - HttpBridgeRecoveryAttemptRecord.response_id.is_(None), - ) - .returning(HttpBridgeRecoveryAttemptRecord.id) - ) - if deleted.scalar_one_or_none() is None: - await self._session.rollback() - return False - row.state = HttpBridgeRowlessRecoveryState.APPROVED - row.replacement_session_id = None - row.dispatch_request_id = None - row.wire_request_fingerprint = None - if origin_marker is not None: - origin_marker.recovery_required_attempt_fingerprint = None - origin_marker.recovery_required_attempt_request_id = None - await self._session.commit() - return True - - async def rollback_physically_unsent_after_send_marker( - self, - *, - authority_id: str, - generation: int, - request_id: str, - wire_request_fingerprint: str, - transport_proof_code: str, - ) -> bool: - """Restore APPROVED after every attempted socket proved no bytes sent.""" - - if transport_proof_code != UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE: - return False - async with sqlite_writer_section(): - row = await self._find_id_for_update(authority_id) - origin_marker = await self._load_origin_marker_for_update(row) - if ( - row is None - or row.generation != generation - or row.state != HttpBridgeRowlessRecoveryState.UNKNOWN - or row.dispatch_request_id != request_id - or row.wire_request_fingerprint != wire_request_fingerprint - or row.dispatch_send_started_at is None - or row.replacement_session_id is None - or ( - row.origin_marker_session_id is not None - and not self._origin_marker_matches_authority( - origin_marker, - row, - expected_attempt_fingerprint=_origin_marker_attempt_fingerprint( - row, - wire_request_fingerprint, - ), - expected_request_id=request_id, - ) - ) - ): - await self._session.rollback() - return False - deleted = await self._session.execute( - delete(HttpBridgeRecoveryAttemptRecord) - .where( - HttpBridgeRecoveryAttemptRecord.session_id == row.replacement_session_id, - HttpBridgeRecoveryAttemptRecord.request_fingerprint == wire_request_fingerprint, - HttpBridgeRecoveryAttemptRecord.request_id == request_id, - HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.UNKNOWN, - HttpBridgeRecoveryAttemptRecord.response_id.is_(None), - ) - .returning(HttpBridgeRecoveryAttemptRecord.id) - ) - if deleted.scalar_one_or_none() is None: - await self._session.rollback() - return False - row.state = HttpBridgeRowlessRecoveryState.APPROVED - row.replacement_session_id = None - row.dispatch_request_id = None - row.dispatch_send_started_at = None - row.wire_request_fingerprint = None - if origin_marker is not None: - origin_marker.recovery_required_attempt_fingerprint = None - origin_marker.recovery_required_attempt_request_id = None - await self._session.commit() - return True - - async def rollback_before_send_primitive( - self, - *, - authority_id: str, - generation: int, - request_id: str, - wire_request_fingerprint: str, - ) -> bool: - """Restore an exact UNKNOWN journal before its send helper is invoked.""" - - async with sqlite_writer_section(): - row = await self._find_id_for_update(authority_id) - origin_marker = await self._load_origin_marker_for_update(row) - if ( - row is None - or row.generation != generation - or row.state != HttpBridgeRowlessRecoveryState.UNKNOWN - or row.dispatch_request_id != request_id - or row.wire_request_fingerprint != wire_request_fingerprint - or row.replacement_session_id is None - or ( - row.origin_marker_session_id is not None - and not self._origin_marker_matches_authority( - origin_marker, - row, - expected_attempt_fingerprint=_origin_marker_attempt_fingerprint( - row, - wire_request_fingerprint, - ), - expected_request_id=request_id, - ) - ) - ): - await self._session.rollback() - return False - deleted = await self._session.execute( - delete(HttpBridgeRecoveryAttemptRecord) - .where( - HttpBridgeRecoveryAttemptRecord.session_id == row.replacement_session_id, - HttpBridgeRecoveryAttemptRecord.request_fingerprint == wire_request_fingerprint, - HttpBridgeRecoveryAttemptRecord.request_id == request_id, - HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.UNKNOWN, - HttpBridgeRecoveryAttemptRecord.response_id.is_(None), - ) - .returning(HttpBridgeRecoveryAttemptRecord.id) - ) - if deleted.scalar_one_or_none() is None: - await self._session.rollback() - return False - row.state = HttpBridgeRowlessRecoveryState.APPROVED - row.replacement_session_id = None - row.dispatch_request_id = None - row.dispatch_send_started_at = None - row.wire_request_fingerprint = None - if origin_marker is not None: - origin_marker.recovery_required_attempt_fingerprint = None - origin_marker.recovery_required_attempt_request_id = None - await self._session.commit() - return True - - async def settle_completed( - self, - *, - authority_id: str, - generation: int, - replacement_session_id: str, - owner_instance_id: str, - owner_epoch: int, - request_id: str, - response_id: str, - input_item_count: int, - input_full_fingerprint: str, - pending_tool_calls: dict[str, str], - response_transition_manifest: ResponseTransitionManifest | None, - ) -> bool: - """Atomically publish the new checkpoint and consume one authority.""" - - async with sqlite_writer_section(): - row = await self._find_id_for_update(authority_id) - replacement = await self._session.scalar( - select(HttpBridgeSessionRecord) - .where(HttpBridgeSessionRecord.id == replacement_session_id) - .with_for_update() - ) - journal = await self._session.scalar( - select(HttpBridgeRecoveryAttemptRecord) - .where( - HttpBridgeRecoveryAttemptRecord.session_id == replacement_session_id, - HttpBridgeRecoveryAttemptRecord.request_id == request_id, - ) - .with_for_update() - ) - if ( - row is None - or row.generation != generation - or row.state != HttpBridgeRowlessRecoveryState.UNKNOWN - or row.replacement_session_id != replacement_session_id - or row.dispatch_request_id != request_id - or row.dispatch_send_started_at is None - or replacement is None - or replacement.api_key_scope != row.api_key_scope - or replacement.owner_instance_id != owner_instance_id - or replacement.owner_epoch != owner_epoch - or replacement.account_id != row.selected_account_intent - or journal is None - or journal.state != HttpBridgeRecoveryAttemptState.UNKNOWN - or journal.request_fingerprint != row.wire_request_fingerprint - or journal.response_id is not None - or input_item_count != row.captured_input_item_count - or input_full_fingerprint != row.captured_input_fingerprint - or ( - row.origin_marker_session_id is not None - and ( - replacement.id != row.origin_marker_session_id - or not self._origin_marker_matches_authority( - replacement, - row, - expected_attempt_fingerprint=_origin_marker_attempt_fingerprint( - row, - row.wire_request_fingerprint, - ), - expected_request_id=request_id, - ) - ) - ) - ): - await self._session.rollback() - return False - replacement.latest_response_id = response_id - replacement.latest_input_item_count = input_item_count - replacement.latest_input_full_fingerprint = input_full_fingerprint - replacement.latest_pending_tool_calls_json = _encode_pending_tool_calls( - response_id, - pending_tool_calls, - ) - replacement.latest_response_transition_manifest_json = encode_response_transition_manifest( - response_transition_manifest - ) - replacement.recovery_required_anchor_hash = None - replacement.recovery_required_account_id = None - replacement.recovery_required_attempt_fingerprint = None - replacement.recovery_required_attempt_request_id = None - replacement.recovery_required_at = None - registered = await DurableBridgeRepository(self._session)._execute_alias_upsert( - session_id=replacement_session_id, - alias_kind="previous_response_id", - alias_value=response_id, - api_key_scope=row.api_key_scope, - target_account_neutral_replay=False, - ) - if not registered: - await self._session.rollback() - return False - journal.state = HttpBridgeRecoveryAttemptState.REPLAYED - journal.response_id = response_id - row.state = HttpBridgeRowlessRecoveryState.CONSUMED - row.consumed_response_id_hash = durable_bridge_hash(response_id) - row.consumed_at = utcnow() - self._session.add( - AuditLog( - action="http_bridge_rowless_semantic_rebase_consumed", - details=json.dumps( - { - "authority_id": row.id, - "generation": row.generation, - "response_id_hash": row.consumed_response_id_hash, - }, - separators=(",", ":"), - sort_keys=True, - ), - request_id=request_id, - ) - ) - await self._session.commit() - return True - - async def _find_for_update( - self, - *, - api_key_scope: str, - strong_session_hash: str, - stale_anchor_hash: str, - ) -> HttpBridgeRowlessRecoveryAuthority | None: - return await self._session.scalar( - select(HttpBridgeRowlessRecoveryAuthority) - .where( - HttpBridgeRowlessRecoveryAuthority.api_key_scope == api_key_scope, - HttpBridgeRowlessRecoveryAuthority.strong_session_hash == strong_session_hash, - HttpBridgeRowlessRecoveryAuthority.stale_anchor_hash == stale_anchor_hash, - ) - .with_for_update() - ) - - async def _find_id_for_update(self, authority_id: str) -> HttpBridgeRowlessRecoveryAuthority | None: - return await self._session.scalar( - select(HttpBridgeRowlessRecoveryAuthority) - .where(HttpBridgeRowlessRecoveryAuthority.id == authority_id) - .with_for_update() - ) - - async def _require_live_origin_marker( - self, - *, - session_id: str, - api_key_scope: str, - selected_account_intent: str, - stale_anchor_hash: str, - ) -> None: - marker = await self._session.scalar( - select(HttpBridgeSessionRecord).where(HttpBridgeSessionRecord.id == session_id).with_for_update() - ) - if ( - marker is None - or marker.api_key_scope != api_key_scope - or marker.account_id != selected_account_intent - or marker.recovery_required_account_id != selected_account_intent - or marker.recovery_required_anchor_hash != stale_anchor_hash - or marker.latest_response_id is None - or durable_bridge_hash(marker.latest_response_id) != stale_anchor_hash - or marker.recovery_required_attempt_fingerprint is not None - or marker.recovery_required_attempt_request_id is not None - ): - raise RowlessRecoveryStateError("durable_marker_capture_fence_rejected") - - @staticmethod - def _origin_marker_matches_authority( - marker: HttpBridgeSessionRecord | None, - authority: HttpBridgeRowlessRecoveryAuthority, - *, - expected_attempt_fingerprint: str | None = None, - expected_request_id: str | None = None, - ) -> bool: - return bool( - marker is not None - and marker.id == authority.origin_marker_session_id - and marker.api_key_scope == authority.api_key_scope - and marker.account_id == authority.selected_account_intent - and marker.recovery_required_account_id == authority.selected_account_intent - and marker.recovery_required_anchor_hash == authority.stale_anchor_hash - and marker.latest_response_id is not None - and durable_bridge_hash(marker.latest_response_id) == authority.stale_anchor_hash - and marker.recovery_required_attempt_fingerprint == expected_attempt_fingerprint - and marker.recovery_required_attempt_request_id == expected_request_id - ) - - async def _load_origin_marker_for_update( - self, - authority: HttpBridgeRowlessRecoveryAuthority | None, - ) -> HttpBridgeSessionRecord | None: - if authority is None or authority.origin_marker_session_id is None: - return None - return await self._session.scalar( - select(HttpBridgeSessionRecord) - .where(HttpBridgeSessionRecord.id == authority.origin_marker_session_id) - .with_for_update() - ) - - async def _find_exact_request_contract_for_update( - self, - *, - api_key_scope: str, - strong_session_hash: str, - facts: RowlessRecoveryCaptureFacts, - ) -> HttpBridgeRowlessRecoveryAuthority | None: - return await self._session.scalar( - select(HttpBridgeRowlessRecoveryAuthority) - .where( - HttpBridgeRowlessRecoveryAuthority.api_key_scope == api_key_scope, - HttpBridgeRowlessRecoveryAuthority.strong_session_hash == strong_session_hash, - HttpBridgeRowlessRecoveryAuthority.captured_input_fingerprint == facts.input_fingerprint, - HttpBridgeRowlessRecoveryAuthority.non_input_contract_fingerprint == facts.contract_fingerprint, - HttpBridgeRowlessRecoveryAuthority.settled_direct_call_ledger_digest == facts.direct_call_ledger_digest, - HttpBridgeRowlessRecoveryAuthority.projected_payload_fingerprint == facts.projected_payload_fingerprint, - ) - .with_for_update() - ) - - @staticmethod - def _require_same_capture( - row: HttpBridgeRowlessRecoveryAuthority, - selected_account_intent: str, - task_identity: str, - session_identity: str, - task_authority_digest: str, - facts: RowlessRecoveryCaptureFacts, - origin_marker_session_id: str | None, - ) -> None: - if ( - row.selected_account_intent != selected_account_intent - or row.origin_marker_session_id != origin_marker_session_id - or row.captured_task_identity_hash != canonical_json_sha256(task_identity) - or row.captured_session_identity_hash != canonical_json_sha256(session_identity) - or row.captured_task_authority_digest != task_authority_digest - or row.captured_input_item_count != facts.input_item_count - or row.captured_input_fingerprint != facts.input_fingerprint - or row.non_input_contract_fingerprint != facts.contract_fingerprint - or row.settled_direct_call_ledger_digest != facts.direct_call_ledger_digest - or row.projected_payload_fingerprint != facts.projected_payload_fingerprint - or row.actual_wire_fingerprint != facts.actual_wire_fingerprint - or row.settled_direct_call_unresolved_count != facts.unresolved_count - or row.request_self_contained != facts.self_contained - or row.request_account_neutral != facts.account_neutral - ): - raise RowlessRecoveryConflictError("rowless recovery contract changed") - - -def _snapshot(row: HttpBridgeRowlessRecoveryAuthority) -> RowlessRecoveryAuthoritySnapshot: - return RowlessRecoveryAuthoritySnapshot( - id=row.id, - api_key_scope=row.api_key_scope, - session_key_kind=row.session_key_kind, - strong_session_hash=row.strong_session_hash, - stale_anchor_hash=row.stale_anchor_hash, - generation=row.generation, - generation_nonce=row.generation_nonce, - state=row.state, - captured_input_item_count=row.captured_input_item_count, - captured_input_fingerprint=row.captured_input_fingerprint, - non_input_contract_fingerprint=row.non_input_contract_fingerprint, - settled_direct_call_ledger_digest=row.settled_direct_call_ledger_digest, - projected_payload_fingerprint=row.projected_payload_fingerprint, - actual_wire_fingerprint=row.actual_wire_fingerprint, - origin_marker_session_id=row.origin_marker_session_id, - settled_direct_call_unresolved_count=row.settled_direct_call_unresolved_count, - selected_account_intent=row.selected_account_intent, - captured_task_identity_hash=row.captured_task_identity_hash, - captured_session_identity_hash=row.captured_session_identity_hash, - captured_task_authority_digest=row.captured_task_authority_digest, - request_self_contained=row.request_self_contained, - request_account_neutral=row.request_account_neutral, - checkpoint_receipt_sha256=row.checkpoint_receipt_sha256, - authorization_mode=row.authorization_mode, - authorization_proof_sha256=row.authorization_proof_sha256, - replacement_session_id=row.replacement_session_id, - dispatch_request_id=row.dispatch_request_id, - dispatch_send_started_at=row.dispatch_send_started_at, - wire_request_fingerprint=row.wire_request_fingerprint, - created_at=row.created_at, - updated_at=row.updated_at, - ) - - -def _capture_matches( - row: HttpBridgeRowlessRecoveryAuthority, - facts: RowlessRecoveryCaptureFacts, -) -> bool: - return bool( - row.captured_input_item_count == facts.input_item_count - and row.captured_input_fingerprint == facts.input_fingerprint - and row.non_input_contract_fingerprint == facts.contract_fingerprint - and row.settled_direct_call_ledger_digest == facts.direct_call_ledger_digest - and row.projected_payload_fingerprint == facts.projected_payload_fingerprint - and row.actual_wire_fingerprint == facts.actual_wire_fingerprint - and row.settled_direct_call_unresolved_count == facts.unresolved_count - and row.request_self_contained == facts.self_contained - and row.request_account_neutral == facts.account_neutral - ) - - -def _replace_capture( - row: HttpBridgeRowlessRecoveryAuthority, - facts: RowlessRecoveryCaptureFacts, -) -> None: - row.captured_input_item_count = facts.input_item_count - row.captured_input_fingerprint = facts.input_fingerprint - row.non_input_contract_fingerprint = facts.contract_fingerprint - row.settled_direct_call_ledger_digest = facts.direct_call_ledger_digest - row.projected_payload_fingerprint = facts.projected_payload_fingerprint - row.actual_wire_fingerprint = facts.actual_wire_fingerprint - row.settled_direct_call_unresolved_count = facts.unresolved_count - row.request_self_contained = facts.self_contained - row.request_account_neutral = facts.account_neutral - - -def _clear_operator_authorization(row: HttpBridgeRowlessRecoveryAuthority) -> None: - row.authorization_mode = None - row.authorization_proof_sha256 = None - row.challenge_nonce_hash = None - row.challenge_expires_at = None - row.checkpoint_receipt_sha256 = None - row.checkpoint_jsonl_sha256 = None - row.checkpoint_jsonl_size_bytes = None - row.checkpoint_jsonl_last_offset = None - row.checkpoint_task_identity_hash = None - row.checkpoint_session_identity_hash = None - row.checkpoint_strong_session_hash = None - row.checkpoint_task_authority_digest = None - row.checkpoint_tool_ledger_digest = None - row.approved_by_actor = None - row.approved_at = None - - -def _automatic_authorization_proof( - row: HttpBridgeRowlessRecoveryAuthority, - *, - wire_request_fingerprint: str, -) -> str: - return canonical_json_sha256( - { - "schema": "qk_http_bridge_rowless_live_request_authorization_v1", - "authority_id": row.id, - "generation": row.generation, - "generation_nonce": row.generation_nonce, - "strong_session_hash": row.strong_session_hash, - "task_authority_digest": row.captured_task_authority_digest, - "captured_input_item_count": row.captured_input_item_count, - "captured_input_fingerprint": row.captured_input_fingerprint, - "non_input_contract_fingerprint": row.non_input_contract_fingerprint, - "direct_call_ledger_digest": row.settled_direct_call_ledger_digest, - "projected_payload_fingerprint": row.projected_payload_fingerprint, - "actual_wire_fingerprint": row.actual_wire_fingerprint, - "selected_account_intent": row.selected_account_intent, - "wire_request_fingerprint": wire_request_fingerprint, - } - ) - - -def _dispatch_authorized( - row: HttpBridgeRowlessRecoveryAuthority, - *, - wire_request_fingerprint: str, -) -> bool: - if row.authorization_mode == ROWLESS_AUTHORIZATION_MODE_AUTOMATIC: - if not _is_sha256(row.authorization_proof_sha256 or "") or not _is_sha256(wire_request_fingerprint): - return False - return secrets.compare_digest( - row.authorization_proof_sha256 or "", - _automatic_authorization_proof( - row, - wire_request_fingerprint=wire_request_fingerprint, - ), - ) - if row.authorization_mode in {None, ROWLESS_AUTHORIZATION_MODE_OPERATOR}: - return _is_sha256(row.checkpoint_receipt_sha256 or "") - return False - - -def _origin_marker_attempt_fingerprint( - authority: HttpBridgeRowlessRecoveryAuthority, - wire_request_fingerprint: str | None, -) -> str: - if wire_request_fingerprint is None: - raise RowlessRecoveryStateError("rowless_marker_wire_fingerprint_missing") - return canonical_json_sha256( - { - "domain": "qk_http_bridge_rowless_marker_attempt_v1", - "authority_id": authority.id, - "generation": authority.generation, - "wire_request_fingerprint": wire_request_fingerprint, - } - ) - - -_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") - - -def _is_sha256(value: str) -> bool: - return _SHA256_RE.fullmatch(value) is not None diff --git a/app/modules/proxy/service.py b/app/modules/proxy/service.py index 03f2769685..cf6fcbafd0 100644 --- a/app/modules/proxy/service.py +++ b/app/modules/proxy/service.py @@ -5,7 +5,7 @@ import re import time from collections.abc import Awaitable, Callable, Collection -from contextlib import AbstractAsyncContextManager, asynccontextmanager +from contextlib import asynccontextmanager from typing import Any, AsyncIterator, Literal, Mapping, NoReturn, TypeVar, cast import aiohttp @@ -513,6 +513,9 @@ from app.modules.proxy._service.streaming.helpers import ( _classify_upstream_close as _classify_upstream_close, ) +from app.modules.proxy._service.streaming.helpers import ( + _is_account_neutral_transport_drop as _is_account_neutral_transport_drop, +) from app.modules.proxy._service.streaming.helpers import ( _push_stream_attempt_timeout_overrides as _push_stream_attempt_timeout_overrides, ) @@ -559,7 +562,6 @@ _clear_websocket_request_error_overrides, # noqa: F401 _DownstreamWebSocketActivity, # noqa: F401 _event_type_from_payload, # noqa: F401 - _FilePinEntry, _finalize_ttft_reasoning_deltas, # noqa: F401 _http_error_status_from_payload, # noqa: F401 _HTTPBridgeSession, @@ -650,6 +652,7 @@ from app.modules.proxy._service.websocket.helpers import ( _app_error_to_websocket_event, # noqa: F401 _assign_websocket_response_id, # noqa: F401 + _clear_websocket_stale_previous_response_cache, # noqa: F401 _draining_websocket_request_states, # noqa: F401 _find_websocket_request_state_by_response_id, # noqa: F401 _is_websocket_previous_response_output_item, # noqa: F401 @@ -701,6 +704,7 @@ _websocket_precreated_auth_error_code, # noqa: F401 _websocket_precreated_retry_error_code, # noqa: F401 _websocket_receive_timeout_for_pending_requests, # noqa: F401 + _websocket_request_text_is_account_neutral_fresh_replay, # noqa: F401 _websocket_response_id, # noqa: F401 _websocket_top_level_error_payload, # noqa: F401 _wrapped_websocket_error_event, # noqa: F401 @@ -708,7 +712,7 @@ from app.modules.proxy.affinity import ( _AffinityPolicy, _CodexSessionSource, - _sticky_key_for_codex_control_request, + _sticky_key_for_thread_goal_request, _sticky_key_from_session_header, # noqa: F401 ) from app.modules.proxy.affinity import ( @@ -731,6 +735,7 @@ _parse_openai_error, _upstream_error_from_openai, ) +from app.modules.proxy.http_bridge_event_batcher import HttpBridgeOperationEventBatcher from app.modules.proxy.http_bridge_forwarding import ( HTTPBridgeForwardContext as HTTPBridgeForwardContext, ) @@ -927,20 +932,20 @@ def __init__( self, repo_factory: ProxyRepoFactory, *, - refresh_repo_factory: Callable[[], AbstractAsyncContextManager[AccountsRepositoryPort]] | None = None, live_websocket_connector: LiveWebSocketConnector = connect_live_websocket, ) -> None: self._repo_factory = repo_factory - self._refresh_repo_factory = refresh_repo_factory or self._accounts_refresh_scope self._encryptor = TokenEncryptor() self._load_balancer = LoadBalancer(repo_factory) self._capability_router = CapabilityRouter(repo_factory) self._live_websocket_connector = live_websocket_connector self._ring_membership = RingMembershipService(SessionLocal) self._durable_bridge = DurableBridgeSessionCoordinator(SessionLocal) + self._http_bridge_operation_event_batcher = HttpBridgeOperationEventBatcher.from_settings(self._durable_bridge) self._http_bridge_owner_client = HTTPBridgeOwnerClient() - self._http_bridge_sessions: dict[_HTTPBridgeSessionKey, _HTTPBridgeSession] = {} - _initialize_http_bridge_retry_circuit(self) + self._initialize_http_bridge_session_registry() + _initialize_http_bridge_retry_circuit(self, _clear_websocket_stale_previous_response_cache) + self._http_bridge_account_timeout_failures, self._http_bridge_account_timeout_lock = {}, asyncio.Lock() self._http_bridge_inflight_sessions: dict[_HTTPBridgeSessionKey, asyncio.Future[_HTTPBridgeSession]] = {} self._http_bridge_turn_state_index: dict[tuple[str, str | None], _HTTPBridgeSessionKey] = {} self._http_bridge_previous_response_index: dict[tuple[str, str | None], _HTTPBridgeSessionKey] = {} @@ -948,15 +953,7 @@ def __init__( self._websocket_continuity_index: dict[tuple[str, str | None], _WebSocketContinuityState] = {} self._background_cleanup_tasks: set[asyncio.Task[None]] = set() self._stream_api_key_release_retry_semaphore = asyncio.Semaphore(_STREAM_API_KEY_RELEASE_RETRY_MAX_CONCURRENCY) - # In-memory pin from upstream-issued file_id -> codex-lb account_id. - # Used so ``finalize_file`` for a given ``file_id`` is routed to - # the same account that handled ``create_file``. Cross-instance - # routing is best-effort: if the finalize request lands on a - # different replica with no pin, we fall back to a fresh load- - # balancer selection. The TTL is short enough (5 min) that we - # never hold stale pins after the upstream upload window closes. - self._file_account_pins: dict[str, _FilePinEntry] = {} - self._file_account_pin_lock = asyncio.Lock() + self._file_pin_session_factory = SessionLocal self._http_bridge_lock = anyio.Lock() self._work_admission: WorkAdmissionController | None = None self._request_log_tasks: set[asyncio.Task[None]] = set() @@ -994,9 +991,8 @@ async def thread_goal_request( base_settings = get_settings() deadline = start + base_settings.proxy_request_budget_seconds settings = await get_settings_cache().get() - affinity = _sticky_key_for_codex_control_request( - headers, - codex_session_affinity=codex_session_affinity, + affinity = _sticky_key_for_thread_goal_request( + payload, headers, codex_session_affinity, settings.openai_cache_affinity_max_age_seconds ) selection_model = api_key.enforced_model if api_key is not None else None routing_strategy = _routing_strategy(settings) @@ -1092,6 +1088,9 @@ async def _select_goal_failover(excluded_account_ids: set[str]) -> AccountSelect reallocate_sticky=affinity.reallocate_sticky, sticky_source=affinity.codex_session_source, legacy_sticky_key=affinity.legacy_selection_key, + legacy_continuity_source=affinity.legacy_continuity_source, + sticky_seed_key=affinity.seed_selection_key, + sticky_seed_kind=affinity.seed_selection_kind, sticky_max_age_seconds=affinity.max_age_seconds, prefer_earlier_reset_accounts=settings.prefer_earlier_reset_accounts, routing_strategy=routing_strategy, @@ -1308,25 +1307,19 @@ async def _acquire_request_state_response_create_admission( pending_request_ages_seconds: list[float] | None = None should_retire_stuck_session = False stale_pending_requests_to_fail: list[_WebSocketRequestState] = [] + retry_circuit_attempt_selection = None if bridge_session is not None: now = time.monotonic() - async with bridge_session.pending_lock: - pending_states = list(bridge_session.pending_requests) - pending_count = len(pending_states) - queued_count = bridge_session.queued_request_count + stale_gate_snapshot = await self._snapshot_http_bridge_stale_gate_state(bridge_session, now=now) + pending_states = stale_gate_snapshot.pending_states + pending_count = len(pending_states) + queued_count = stale_gate_snapshot.queued_count + threshold_seconds = stale_gate_snapshot.threshold_seconds + stale_pending_requests_to_fail = stale_gate_snapshot.stale_request_states + should_retire_stuck_session = stale_gate_snapshot.should_retire + retry_circuit_attempt_selection = stale_gate_snapshot.retry_circuit_attempt_selection pending_request_ids = [state.request_log_id or state.request_id for state in pending_states] pending_request_ages_seconds = [max(0.0, now - state.started_at) for state in pending_states] - threshold_seconds = float( - getattr(get_settings(), "http_responses_session_bridge_stuck_gate_retire_after_seconds", 300.0) - ) - stale_pending_requests_to_fail, should_retire_stuck_session = ( - self._classify_http_bridge_stale_gate_holders( - pending_states, - now=now, - threshold_seconds=threshold_seconds, - session_closed=bridge_session.closed, - ) - ) if not should_retire_stuck_session and any( max(0.0, now - state.started_at) >= threshold_seconds for state in pending_states ): @@ -1374,6 +1367,7 @@ async def _acquire_request_state_response_create_admission( bridge_session, stale_pending_requests_to_fail, detail="response_create_gate_timeout_stuck_pending", + retry_circuit_attempt_selection=retry_circuit_attempt_selection, ) elif bridge_session is not None and should_retire_stuck_session: _record_http_bridge_stuck_retire( @@ -1383,6 +1377,7 @@ async def _acquire_request_state_response_create_admission( await self._retire_stale_pending_http_bridge_session( bridge_session, detail="response_create_gate_timeout_stuck_pending", + retry_circuit_attempt_selection=retry_circuit_attempt_selection, ) raise _http_bridge_startup_wait_timeout_error( "http_bridge_response_create_gate", @@ -1418,24 +1413,11 @@ async def _release_request_state_account_response_create_lease( request_state.account_response_create_release = None await self._load_balancer.release_account_lease(lease) - async def _select_account_with_budget_compatible( - self, - deadline: float, - **kwargs: object, - ) -> AccountSelection: + async def _select_account_with_budget_compatible(self, deadline: float, **kwargs: object) -> AccountSelection: affinity_policy = kwargs.pop("affinity_policy", None) if isinstance(affinity_policy, _AffinityPolicy): # Expand once at the compatibility edge so transport callers cannot drift. - kwargs.update( - sticky_key=affinity_policy.selection_key, - sticky_kind=affinity_policy.kind, - reallocate_sticky=affinity_policy.reallocate_sticky, - sticky_source=affinity_policy.codex_session_source, - legacy_sticky_key=affinity_policy.legacy_selection_key, - spill_bare_session_on_account_cap=affinity_policy.spill_on_account_cap, - require_unambiguous_account=affinity_policy.require_unambiguous_account, - sticky_max_age_seconds=affinity_policy.max_age_seconds, - ) + kwargs.update(affinity_policy.selection_kwargs()) required_capability_kwargs = {} if kwargs.get("require_security_work_authorized") is True: required_capability_kwargs["require_security_work_authorized"] = kwargs.pop( @@ -1450,6 +1432,8 @@ async def _select_account_with_budget_compatible( @asynccontextmanager async def _accounts_refresh_scope(self) -> AsyncIterator[AccountsRepositoryPort]: + # A self-contained repo prevents request cancellation from closing the + # session under AuthManager's shielded refresh and stranding a connection. async with self._repo_factory() as repos: yield repos.accounts @@ -1463,11 +1447,11 @@ async def _ensure_fresh( ) -> Account: token = push_token_refresh_timeout_override(timeout_seconds) try: - async with self._refresh_repo_factory() as accounts_repo: + async with self._repo_factory() as repos: auth_manager = AuthManager( - accounts_repo, + repos.accounts, acquire_refresh_admission=self._get_work_admission().acquire_token_refresh, - refresh_repo_factory=self._refresh_repo_factory, + refresh_repo_factory=self._accounts_refresh_scope, redact_sensitive_details=redact_sensitive_details, ) refresh = auth_manager.ensure_fresh(account, force=force) @@ -1719,7 +1703,11 @@ async def _select_account_with_budget( reallocate_sticky: bool = False, sticky_source: _CodexSessionSource | None = None, legacy_sticky_key: str | None = None, + legacy_continuity_source: _CodexSessionSource | None = None, + sticky_seed_key: str | None = None, + sticky_seed_kind: StickySessionKind | None = None, spill_bare_session_on_account_cap: bool = False, + abandon_unavailable_legacy_owner: bool = False, require_unambiguous_account: bool = False, sticky_max_age_seconds: int | None = None, prefer_earlier_reset_accounts: bool = False, @@ -1876,6 +1864,12 @@ def log_account_id(account_id: str | None) -> str | None: sticky_max_age_seconds=preferred_sticky_inputs[3], sticky_source=preferred_sticky_inputs[4], legacy_sticky_key=preferred_sticky_inputs[5], + legacy_continuity_source=legacy_continuity_source, + # Exact ownership chooses the account; a first-ever thread + # still seeds atomically without overwriting a process default. + sticky_seed_key=sticky_seed_key, + sticky_seed_kind=sticky_seed_kind, + abandon_unavailable_legacy_owner=abandon_unavailable_legacy_owner, prefer_earlier_reset_accounts=prefer_earlier_reset_accounts, prefer_earlier_reset_window=prefer_earlier_reset_window, routing_strategy=routing_strategy, @@ -1933,11 +1927,15 @@ def log_account_id(account_id: str | None) -> str | None: reallocate_sticky=reallocate_sticky, sticky_source=sticky_source, legacy_sticky_key=legacy_sticky_key, + legacy_continuity_source=legacy_continuity_source, + sticky_seed_key=sticky_seed_key, + sticky_seed_kind=sticky_seed_kind, spill_bare_session_on_account_cap=_AffinityPolicy.cap_spillover_allowed( spill_bare_session_on_account_cap, preferred_account_id, request_stage, ), + abandon_unavailable_legacy_owner=abandon_unavailable_legacy_owner, require_unambiguous_account=require_unambiguous_account, sticky_max_age_seconds=sticky_max_age_seconds, prefer_earlier_reset_accounts=prefer_earlier_reset_accounts, diff --git a/app/modules/proxy/sticky_repository.py b/app/modules/proxy/sticky_repository.py index 0948da94c5..23ff2b6685 100644 --- a/app/modules/proxy/sticky_repository.py +++ b/app/modules/proxy/sticky_repository.py @@ -3,6 +3,7 @@ from collections.abc import Sequence from dataclasses import dataclass from datetime import datetime, timedelta +from typing import Literal from sqlalchemy import and_, delete, func, or_, select, update from sqlalchemy.dialects.postgresql import insert as pg_insert @@ -24,6 +25,21 @@ # bind parameters, which this chunk size also respects. _DELETE_ENTRIES_CHUNK_SIZE = 250 +_ContinuitySource = Literal["session_header", "thread_header", "turn_state"] +_SESSION_HEADER_ABANDONMENT_SCOPE = "session_header" + +# A same-owner TTL refresh upsert only rewrites ``updated_at``. On hot +# (key, kind) rows, concurrent requests serialize on that row lock, so the +# selection path may skip the rewrite while the row is younger than this +# window, revalidating the observed deadline at write time. The window is +# bounded to at most 1% of the mapping TTL (so expiry moves by at most 1% of +# the window it protects) and to a small absolute ceiling; a rebind to a +# different owner, a row carrying any abandonment marker, or a row stamped in +# the future is never skippable because those writes change state beyond +# freshness (or the observation itself is untrustworthy). +_REFRESH_SKIP_TTL_FRACTION = 0.01 +_REFRESH_SKIP_MAX_SECONDS = 15.0 + # Only the Live-call ownership namespace is reserved. Other LF-prefixed keys # (e.g. the pre-existing "\ncodex-lb-affinity-v1" selection affinities) remain # ordinary operator-manageable sessions. @@ -49,12 +65,102 @@ class StickyOwnerLookup: account_id: str | None continuity_abandoned: bool - - -def _owner_lookup_from_row(row: StickySession) -> StickyOwnerLookup: - if row.continuity_abandoned_at is not None: - return StickyOwnerLookup(account_id=None, continuity_abandoned=True) - return StickyOwnerLookup(account_id=row.account_id, continuity_abandoned=False) + # Source-qualified abandonment makes account_id ownerless only for the + # matching source, but selection must still remember which durable owner + # was retired. Global stale-hard tombstones leave this unset because their + # established recovery path may legitimately reselect a recovered owner. + abandoned_account_id: str | None = None + # Set only when the row was observed in this lookup with a fresh + # ``updated_at`` (within the refresh-skip window derived from + # ``max_age_seconds``) and no abandonment marker, so a same-owner TTL + # refresh upsert would be a pure ``updated_at`` rewrite. The value is the + # naive-UTC instant (``observed_updated_at`` + skip window) after which + # the skip is no longer valid; consumers must isinstance-check + # ``datetime`` (test doubles may auto-vivify attributes), must revalidate + # the deadline against the clock immediately before omitting the write, + # and must never skip a write that changes the owner account. + refresh_skip_deadline: datetime | None = None + + +def _continuity_is_abandoned_for_source( + abandoned_at: datetime | None, + abandonment_scope: str | None, + continuity_source: _ContinuitySource | None, +) -> bool: + if abandonment_scope is not None: + # A scope is itself the source-qualified marker. Goal-restart writers + # deliberately leave the legacy timestamp NULL so pre-scope binaries + # keep treating account_id as hard ownership during rollout/rollback. + # Unknown and nonmatching typed callers likewise fail closed. + return abandonment_scope == continuity_source + # Historical stale-hard tombstones have a timestamp and NULL scope, and + # therefore continue to abandon ownership globally for every source. + return abandoned_at is not None + + +def _source_scoped_abandoned_account_id( + account_id: str, + abandonment_scope: str | None, + continuity_source: _ContinuitySource | None, +) -> str | None: + if abandonment_scope is not None and abandonment_scope == continuity_source: + return account_id + return None + + +def _owner_lookup_from_row( + row: StickySession, + *, + continuity_source: _ContinuitySource | None, + refresh_skip_deadline: datetime | None = None, +) -> StickyOwnerLookup: + if _continuity_is_abandoned_for_source( + row.continuity_abandoned_at, + row.continuity_abandonment_scope, + continuity_source, + ): + return StickyOwnerLookup( + account_id=None, + continuity_abandoned=True, + abandoned_account_id=_source_scoped_abandoned_account_id( + row.account_id, + row.continuity_abandonment_scope, + continuity_source, + ), + ) + return StickyOwnerLookup( + account_id=row.account_id, + continuity_abandoned=False, + refresh_skip_deadline=refresh_skip_deadline, + ) + + +def _same_owner_refresh_skip_deadline( + row: StickySession, + *, + observed_updated_at: datetime, + now: datetime, + max_age_seconds: int, +) -> datetime | None: + """Deadline until which a same-owner upsert of this row stays skippable. + + Any abandonment marker disqualifies the skip: an upsert re-establishes + ownership by clearing both marker columns, so that write is semantic even + when the owner account is unchanged. A row whose ``updated_at`` sits in + the future (database clock ahead of this process, or a restored row) is + also never skippable: an upper-bound-only age comparison would let such a + row satisfy the window for longer than the documented bound. + """ + + if row.continuity_abandoned_at is not None or row.continuity_abandonment_scope is not None: + return None + age_seconds = (now - observed_updated_at).total_seconds() + if age_seconds < 0: + return None + skip_window_seconds = min(_REFRESH_SKIP_MAX_SECONDS, max_age_seconds * _REFRESH_SKIP_TTL_FRACTION) + if age_seconds > skip_window_seconds: + return None + return observed_updated_at + timedelta(seconds=skip_window_seconds) class StickySessionsRepository: @@ -67,8 +173,14 @@ async def get_account_id( *, kind: StickySessionKind, max_age_seconds: int | None = None, + continuity_source: _ContinuitySource | None = None, ) -> str | None: - lookup = await self.get_account_id_and_abandonment(key, kind=kind, max_age_seconds=max_age_seconds) + lookup = await self.get_account_id_and_abandonment( + key, + kind=kind, + max_age_seconds=max_age_seconds, + continuity_source=continuity_source, + ) return lookup.account_id async def get_account_id_and_abandonment( @@ -77,17 +189,14 @@ async def get_account_id_and_abandonment( *, kind: StickySessionKind, max_age_seconds: int | None = None, + continuity_source: _ContinuitySource | None = None, ) -> StickyOwnerLookup: - """Resolve a mapping's owner, and whether it's a purge tombstone. - - A tombstoned row (``continuity_abandoned_at`` set — see - ``purge_stale_hard_codex_session_mappings``) is deliberately reported - as ownerless here, same as a missing row, so every existing caller of - ``get_account_id`` keeps treating it as "no live pin" without change. - The extra flag lets ``run_sticky_selection_path`` additionally - distinguish "this key was purged" from "this key was never seen", - which matters only for the `conversation`-continuity ambiguous-owner - check that has no other index to fall back on. + """Resolve a mapping's owner and applicable abandonment marker. + + Global stale-hard tombstones remain ownerless for every source. A + source-scoped marker is ownerless only for its matching typed source; + explicit turn-state and unknown callers retain the stored owner when a + goal restart abandoned only session-header interpretation. """ if not key: return StickyOwnerLookup(account_id=None, continuity_abandoned=False) @@ -95,11 +204,21 @@ async def get_account_id_and_abandonment( if row is None: return StickyOwnerLookup(account_id=None, continuity_abandoned=False) if max_age_seconds is None: - return _owner_lookup_from_row(row) - cutoff = utcnow() - timedelta(seconds=max_age_seconds) + return _owner_lookup_from_row(row, continuity_source=continuity_source) + now = utcnow() + cutoff = now - timedelta(seconds=max_age_seconds) observed_updated_at = to_utc_naive(row.updated_at) if observed_updated_at >= cutoff: - return _owner_lookup_from_row(row) + return _owner_lookup_from_row( + row, + continuity_source=continuity_source, + refresh_skip_deadline=_same_owner_refresh_skip_deadline( + row, + observed_updated_at=observed_updated_at, + now=now, + max_age_seconds=max_age_seconds, + ), + ) # Release the read snapshot before attempting a SQLite write upgrade. # The DELETE remains safe because every value observed above participates @@ -116,7 +235,7 @@ async def get_account_id_and_abandonment( ) .returning(StickySession.key) ) - current: tuple[str, datetime, datetime | None] | None = None + current: tuple[str, datetime, datetime | None, str | None] | None = None async with sqlite_writer_section(): deleted_key = (await self._session.execute(statement)).scalar_one_or_none() if deleted_key is None: @@ -127,6 +246,7 @@ async def get_account_id_and_abandonment( StickySession.account_id, StickySession.updated_at, StickySession.continuity_abandoned_at, + StickySession.continuity_abandonment_scope, ).where( StickySession.key == key, StickySession.kind == kind, @@ -140,13 +260,46 @@ async def get_account_id_and_abandonment( if deleted_key is not None or current is None: return StickyOwnerLookup(account_id=None, continuity_abandoned=False) - current_account_id, current_updated_at, current_continuity_abandoned_at = current + ( + current_account_id, + current_updated_at, + current_continuity_abandoned_at, + current_continuity_abandonment_scope, + ) = current if to_utc_naive(current_updated_at) < cutoff: return StickyOwnerLookup(account_id=None, continuity_abandoned=False) - if current_continuity_abandoned_at is not None: - return StickyOwnerLookup(account_id=None, continuity_abandoned=True) + if _continuity_is_abandoned_for_source( + current_continuity_abandoned_at, + current_continuity_abandonment_scope, + continuity_source, + ): + return StickyOwnerLookup( + account_id=None, + continuity_abandoned=True, + abandoned_account_id=_source_scoped_abandoned_account_id( + current_account_id, + current_continuity_abandonment_scope, + continuity_source, + ), + ) return StickyOwnerLookup(account_id=current_account_id, continuity_abandoned=False) + async def release_read_snapshot(self) -> None: + """End the session's current read transaction. + + On the default SQLite/WAL configuration one transaction pins one read + snapshot at its first SELECT, so a session shared across successive + ownership lookups would leave every later lookup blind to owners + committed concurrently after the first read. Committing ends that + snapshot so the next SELECT begins a fresh transaction; on PostgreSQL + READ COMMITTED each statement already reads fresh committed state, so + this is a near-free no-op. COMMIT (not rollback) on purpose: rollback + expires all tracked ORM state regardless of ``expire_on_commit``, + while commit under the session factory's ``expire_on_commit=False`` + keeps rows loaded by earlier lookups readable. + """ + await self._session.commit() + async def get_entry(self, key: str, *, kind: StickySessionKind) -> StickySession | None: if not key: return None @@ -191,6 +344,42 @@ async def insert_if_absent(self, key: str, account_id: str, kind: StickySessionK raise RuntimeError("StickySession immutable insert did not resolve an owner") return owner_id + async def upsert_with_seed_if_absent( + self, + key: str, + account_id: str, + *, + kind: StickySessionKind, + seed_key: str, + seed_kind: StickySessionKind, + ) -> StickySession: + """Upsert one mapping and initialize its immutable seed atomically.""" + + # Keep these writes in one transaction. A process seed without the + # initiating thread row is false placement evidence, while a thread + # row without its seed makes the first admitted thread invisible to + # later siblings. Do not replace the seed's DO NOTHING with an upsert: + # another thread may have won first-writer initialization already. + seed_statement = self._build_insert_do_nothing_statement(seed_key, account_id, seed_kind) + mapping_statement = self._build_upsert_statement(key, account_id, kind).returning(StickySession) + async with sqlite_writer_section(): + try: + await self._session.execute(seed_statement) + result = await self._session.execute( + mapping_statement, + execution_options={"populate_existing": True}, + ) + row = result.scalar_one_or_none() + if row is None: + raise RuntimeError(f"StickySession seeded upsert failed for key={key!r} kind={kind.value!r}") + await self._session.commit() + except BaseException: + # This method owns both writes as one unit even when a caller + # catches the error and keeps using the same session. + await self._session.rollback() + raise + return row + async def delete(self, key: str, *, kind: StickySessionKind) -> bool: if not key: return False @@ -203,6 +392,69 @@ async def delete(self, key: str, *, kind: StickySessionKind) -> bool: await self._session.commit() return result.scalar_one_or_none() is not None + async def abandon_legacy_session_header_owner_if_unavailable( + self, + key: str, + *, + kind: StickySessionKind, + expected_account_id: str, + ) -> bool: + """Abandon only session-header interpretation of an unavailable raw owner.""" + + if not key or not expected_account_id: + return False + unavailable_statuses = ( + AccountStatus.PAUSED, + AccountStatus.RATE_LIMITED, + AccountStatus.QUOTA_EXCEEDED, + ) + # PostgreSQL evaluates the status subquery from the UPDATE statement's + # snapshot. Without first locking the Account row, a concurrent status + # recovery can commit while that statement waits for the StickySession + # row and the stale snapshot can still authorize a tombstone. Locking + # the status owner makes recovery and retirement serialize; the sticky + # owner predicate below independently keeps concurrent rebinds safe. + owner_status_lock = select(Account.status).where(Account.id == expected_account_id).with_for_update() + # Retain account status inside the UPDATE as a second, database-level + # invariant. The lock is the concurrency guarantee; this predicate + # prevents future refactors from turning a prior status observation + # into unconditional retirement. + unavailable_owner = select(Account.id).where( + Account.id == expected_account_id, + Account.status.in_(unavailable_statuses), + ) + statement = ( + update(StickySession) + .where( + StickySession.key == key, + StickySession.kind == kind, + StickySession.account_id == expected_account_id, + StickySession.continuity_abandoned_at.is_(None), + StickySession.continuity_abandonment_scope.is_(None), + StickySession.account_id.in_(unavailable_owner), + ) + # The scope column is the new reader's marker. Keep the legacy + # timestamp NULL: older replicas know only that timestamp, so they + # continue to treat account_id as hard ownership instead of + # globally abandoning and rebinding a colliding explicit turn + # state. New readers use typed scope, never key shape, to decide + # which source may ignore the retained owner. + .values( + updated_at=func.now(), + continuity_abandoned_at=None, + continuity_abandonment_scope=_SESSION_HEADER_ABANDONMENT_SCOPE, + ) + .returning(StickySession.key) + ) + async with sqlite_writer_section(): + owner_status = await self._session.scalar(owner_status_lock) + if owner_status not in unavailable_statuses: + await self._session.commit() + return False + result = await self._session.execute(statement) + await self._session.commit() + return result.scalar_one_or_none() is not None + async def restore_if_current( self, key: str, @@ -239,7 +491,12 @@ async def restore_if_current( StickySession.kind == kind, StickySession.account_id == expected_account_id, ) - .values(account_id=restore_account_id, updated_at=func.now(), continuity_abandoned_at=None) + .values( + account_id=restore_account_id, + updated_at=func.now(), + continuity_abandoned_at=None, + continuity_abandonment_scope=None, + ) .returning(StickySession.key) ) @@ -452,15 +709,25 @@ async def purge_stale_hard_codex_session_mappings(self, cutoff: datetime, *, now update(StickySession) .where( StickySession.kind == StickySessionKind.CODEX_SESSION, - StickySession.continuity_abandoned_at.is_(None), + or_( + StickySession.continuity_abandoned_at.is_(None), + StickySession.continuity_abandonment_scope.is_not(None), + ), StickySession.updated_at < cutoff_naive, StickySession.account_id.in_(unavailable_account_ids), ) - .values(continuity_abandoned_at=to_utc_naive(now)) + # Stale-hard cleanup is global. It may promote a younger + # session-header-only marker once the original row itself crosses + # the normal stale-hard threshold. + .values( + continuity_abandoned_at=to_utc_naive(now), + continuity_abandonment_scope=None, + ) ) delete_stmt = delete(StickySession).where( StickySession.kind == StickySessionKind.CODEX_SESSION, StickySession.continuity_abandoned_at.is_not(None), + StickySession.continuity_abandonment_scope.is_(None), StickySession.continuity_abandoned_at < cutoff_naive, ) async with sqlite_writer_section(): @@ -490,6 +757,7 @@ def _build_upsert_statement(self, key: str, account_id: str, kind: StickySession # no longer applies — otherwise this row would keep reporting # itself as abandoned even though it now has a live owner. "continuity_abandoned_at": None, + "continuity_abandonment_scope": None, }, ) diff --git a/app/modules/proxy/tool_call_dedupe.py b/app/modules/proxy/tool_call_dedupe.py index 6c1c676799..6c2aa756d4 100644 --- a/app/modules/proxy/tool_call_dedupe.py +++ b/app/modules/proxy/tool_call_dedupe.py @@ -7,7 +7,7 @@ from app.core.openai import tool_call_safety from app.core.openai.models import OpenAIEvent -from app.core.openai.parsing import parse_sse_event_payload +from app.core.openai.parsing import classify_event_type, parse_sse_event_payload from app.core.types import JsonValue from app.core.utils.sse import format_sse_event @@ -36,14 +36,7 @@ def is_downstream_side_effect_tool_call(item: Mapping[str, JsonValue]) -> bool: def event_type_from_payload(event: OpenAIEvent | None, payload: dict[str, JsonValue] | None) -> str | None: if event is not None: return event.type - if payload is None: - return None - payload_type = payload.get("type") - if isinstance(payload_type, str): - return payload_type - if isinstance(payload.get("error"), dict): - return "error" - return None + return classify_event_type(payload) def response_id_from_payload(payload: dict[str, JsonValue] | None) -> str | None: @@ -785,10 +778,10 @@ def rewrite_parallel_tool_call_text( ) -> tuple[str, dict[str, JsonValue] | None, OpenAIEvent | None, str | None, str]: rewritten_payload, changed, _removed_count = rewrite_parallel_tool_call_payload(payload) if not changed: - # Reuse the caller's parsed event; validating the payload directly - # avoids re-parsing the raw block when a caller has neither. - if event is None: - event = parse_sse_event_payload(payload) + # Reuse the caller's parsed event as-is. Hot streaming paths validate + # only lifecycle frames, so ``event`` is intentionally None for the + # delta bulk; the event type is classified from the payload dict + # instead of re-validating per frame. return text, payload, event, event_type_from_payload(event, payload), event_block assert rewritten_payload is not None rewritten_text = json.dumps(rewritten_payload, ensure_ascii=True, separators=(",", ":")) @@ -811,8 +804,8 @@ def rewrite_parallel_tool_call_sse_line( ) -> tuple[str, dict[str, JsonValue] | None, OpenAIEvent | None, str | None]: rewritten_payload, changed, _removed_count = rewrite_parallel_tool_call_payload(payload) if not changed: - if event is None: - event = parse_sse_event_payload(payload) + # See rewrite_parallel_tool_call_text: no per-frame re-validation on + # the unchanged path; callers own lifecycle-gated validation. return line, payload, event, event_type_from_payload(event, payload) assert rewritten_payload is not None rewritten_line = format_sse_event(rewritten_payload) diff --git a/app/modules/quota_planner/warmup.py b/app/modules/quota_planner/warmup.py index 4b57e6debb..3e75776ae2 100644 --- a/app/modules/quota_planner/warmup.py +++ b/app/modules/quota_planner/warmup.py @@ -14,6 +14,7 @@ from app.core.crypto import TokenEncryptor from app.core.openai.parsing import parse_sse_event from app.core.openai.requests import ResponsesRequest +from app.core.runtime_logging import safe_log_field from app.core.utils.time import naive_utc_to_epoch, utcnow from app.db.models import Account, AccountStatus, QuotaPlannerDecision from app.modules.accounts.repository import AccountsRepository @@ -151,7 +152,9 @@ async def warm_now( output_tokens=WARMUP_DEFAULT_OUTPUT_BUDGET, ), ) - reservation_id = reservation.reservation_id + # ``None`` means no configured limit applies to the warmup + # probe; there is nothing to finalize afterwards. + reservation_id = reservation.reservation_id if reservation is not None else None except ApiKeyNotFoundError: row = await self._planner.update_decision_status( decision.id, @@ -309,7 +312,11 @@ async def _try_record_warmup_effect( try: await self._record_warmup_effect(account, model, source=source, confidence=confidence) except Exception: - logger.exception("Failed to record quota warmup effect", extra={"account_id": account.id, "model": model}) + logger.exception( + "Failed to record quota warmup effect account_id=%s model=%s", + safe_log_field(account.id), + safe_log_field(model), + ) async def _resolve_refused_claim( self, diff --git a/app/modules/rate_limit_reset_credits/api.py b/app/modules/rate_limit_reset_credits/api.py index 034c83e2a2..01bb4d431f 100644 --- a/app/modules/rate_limit_reset_credits/api.py +++ b/app/modules/rate_limit_reset_credits/api.py @@ -132,7 +132,8 @@ async def get_rate_limit_reset_credits( ) -> RateLimitResetCreditsSnapshotResponse | None: store = get_rate_limit_reset_credits_store() account = await context.repository.get_by_id(account_id) - if account is None: + # A pending-deletion account is gone from the operator's point of view. + if account is None or account.delete_requested_at is not None: await store.invalidate(account_id) return None if account.status in _NON_REDEEMABLE_STATUSES or not account.chatgpt_account_id: @@ -158,7 +159,9 @@ async def consume_rate_limit_reset_credit( context: AccountsContext = Depends(get_accounts_context), ) -> ConsumeResetCreditResponseSchema: account = await context.repository.get_by_id(account_id) - if account is None: + # A pending-deletion account is gone from the operator's point of view: + # the synchronous delete 404'd here once the row was removed. + if account is None or account.delete_requested_at is not None: raise DashboardNotFoundError("Account not found", code="account_not_found") store = get_rate_limit_reset_credits_store() diff --git a/app/modules/reports/api.py b/app/modules/reports/api.py index 8b8aa1731e..857ffa8871 100644 --- a/app/modules/reports/api.py +++ b/app/modules/reports/api.py @@ -29,6 +29,7 @@ async def get_reports( end_date: Annotated[date | None, Query()] = None, report_timezone: Annotated[str | None, Query(alias="timezone")] = None, account_id: Annotated[list[str] | None, Query()] = None, + api_key_id: Annotated[list[str] | None, Query()] = None, model: Annotated[str | None, Query()] = None, useragent_group: Annotated[str | None, Query()] = None, ) -> ReportsResponse: @@ -38,6 +39,7 @@ async def get_reports( end_date=end_date, report_timezone=report_timezone, account_ids=account_id, + api_key_ids=api_key_id, model=model, useragent_group=useragent_group, ) diff --git a/app/modules/reports/repository.py b/app/modules/reports/repository.py index a7d0f172a6..ffadccaf5e 100644 --- a/app/modules/reports/repository.py +++ b/app/modules/reports/repository.py @@ -34,6 +34,7 @@ class DailyReportAggregateRow: requests: int input_tokens: int output_tokens: int + reasoning_tokens: int | None cached_input_tokens: int cost_usd: float active_accounts: int @@ -50,6 +51,8 @@ class SummaryAggregateRow: total_cost_usd: float total_input_tokens: int total_output_tokens: int + total_reasoning_tokens: int + reasoning_usage_known_requests: int total_cached_tokens: int total_requests: int total_errors: int @@ -96,6 +99,7 @@ async def aggregate_daily_rows( account_ids: list[str] | None = None, model: str | None = None, useragent_group: str | None = None, + api_key_ids: list[str] | None = None, ) -> list[DailyReportAggregateRow]: window_days = (end_date - start_date).days + 1 if window_days > MAX_DAILY_REPORT_DAYS: @@ -108,14 +112,14 @@ async def aggregate_daily_rows( # unfiltered per-day conversation counts are served from the # conversation satellite (one extra statement per batch, replacing # the raw COUNT(DISTINCT ...) column in the main statement). - use_rollup = not (account_ids or model or useragent_group) + use_rollup = not (account_ids or model or useragent_group or api_key_ids) rows: list[DailyReportAggregateRow] = [] # SQLite caps compound SELECTs at 500 terms, so long report ranges are # executed in chunks instead of building a single oversized UNION ALL. for day_ranges_batch in batched(day_ranges, _SQLITE_COMPOUND_SELECT_LIMIT): day_ranges_list = list(day_ranges_batch) speed_result = await self._session.execute( - _daily_speed_medians_stmt(day_ranges_list, account_ids, model, useragent_group) + _daily_speed_medians_stmt(day_ranges_list, account_ids, model, useragent_group, api_key_ids) ) speed_values = { speed_row.report_date: ( @@ -139,7 +143,12 @@ async def aggregate_daily_rows( result = await self._session.execute( _daily_rows_stmt( - day_ranges_list, account_ids, model, useragent_group, include_conversations=not use_rollup + day_ranges_list, + account_ids, + model, + useragent_group, + api_key_ids, + include_conversations=not use_rollup, ) ) rows.extend( @@ -148,6 +157,7 @@ async def aggregate_daily_rows( requests=int(row.requests or 0), input_tokens=int(row.input_tokens or 0), output_tokens=int(row.output_tokens or 0), + reasoning_tokens=int(row.reasoning_tokens) if row.reasoning_tokens is not None else None, cached_input_tokens=int(row.cached_input_tokens or 0), cost_usd=float(row.cost_usd or 0.0), active_accounts=int(row.active_accounts or 0), @@ -171,19 +181,25 @@ async def aggregate_summary( account_ids: list[str] | None = None, model: str | None = None, useragent_group: str | None = None, + api_key_ids: list[str] | None = None, ) -> SummaryAggregateRow: - conditions = _report_conditions(start_date, end_date, account_ids, model, useragent_group) + conditions = _report_conditions(start_date, end_date, account_ids, model, useragent_group, api_key_ids) # The conversation satellite carries no model/useragent dimensions # and pre-merges accounts, so only the unfiltered read is served from # it (rollup + raw tail, split out of the single statement the same # way the dashboard activity read splits its conversation metrics); # filtered summaries keep the legacy raw single statement. - use_rollup = not (account_ids or model or useragent_group) + use_rollup = not (account_ids or model or useragent_group or api_key_ids) columns = [ func.coalesce(func.sum(RequestLog.cost_usd), 0.0).label("total_cost_usd"), func.coalesce(func.sum(RequestLog.input_tokens), 0).label("total_input_tokens"), - func.coalesce(func.sum(RequestLog.output_tokens), 0).label("total_output_tokens"), + func.coalesce( + func.sum(func.coalesce(RequestLog.output_tokens, RequestLog.reasoning_tokens, 0)), + 0, + ).label("total_output_tokens"), + func.coalesce(func.sum(RequestLog.reasoning_tokens), 0).label("total_reasoning_tokens"), + func.count(RequestLog.reasoning_tokens).label("reasoning_usage_known_requests"), func.coalesce(func.sum(RequestLog.cached_input_tokens), 0).label("total_cached_tokens"), func.count().label("total_requests"), func.coalesce( @@ -216,6 +232,8 @@ async def aggregate_summary( total_cost_usd=float(row.total_cost_usd), total_input_tokens=int(row.total_input_tokens), total_output_tokens=int(row.total_output_tokens), + total_reasoning_tokens=int(row.total_reasoning_tokens), + reasoning_usage_known_requests=int(row.reasoning_usage_known_requests), total_cached_tokens=int(row.total_cached_tokens), total_requests=int(row.total_requests), total_errors=int(row.total_errors), @@ -231,9 +249,10 @@ async def aggregate_by_model( account_ids: list[str] | None = None, model: str | None = None, useragent_group: str | None = None, + api_key_ids: list[str] | None = None, ) -> list[ModelAggregateRow]: conditions = [ - *_report_conditions(start_date, end_date, account_ids, model, useragent_group), + *_report_conditions(start_date, end_date, account_ids, model, useragent_group, api_key_ids), RequestLog.model.is_not(None), ] @@ -264,8 +283,9 @@ async def aggregate_by_account( account_ids: list[str] | None = None, model: str | None = None, useragent_group: str | None = None, + api_key_ids: list[str] | None = None, ) -> list[AccountAggregateRow]: - conditions = _report_conditions(start_date, end_date, account_ids, model, useragent_group) + conditions = _report_conditions(start_date, end_date, account_ids, model, useragent_group, api_key_ids) stmt = ( select( @@ -305,10 +325,11 @@ async def aggregate_by_useragent( account_ids: list[str] | None = None, model: str | None = None, useragent_group: str | None = None, + api_key_ids: list[str] | None = None, ) -> list[UserAgentAggregateRow]: useragent_group_bucket = _useragent_group_bucket_expr() conditions = [ - *_report_conditions(start_date, end_date, account_ids, model, useragent_group), + *_report_conditions(start_date, end_date, account_ids, model, useragent_group, api_key_ids), or_(RequestLog.useragent_group.is_(None), func.trim(RequestLog.useragent_group) != ""), ] @@ -339,9 +360,10 @@ async def count_active_accounts( account_ids: list[str] | None = None, model: str | None = None, useragent_group: str | None = None, + api_key_ids: list[str] | None = None, ) -> int: conditions = [ - *_report_conditions(start_date, end_date, account_ids, model, useragent_group), + *_report_conditions(start_date, end_date, account_ids, model, useragent_group, api_key_ids), RequestLog.account_id.is_not(None), ] @@ -355,6 +377,7 @@ async def earliest_report_activity_at( account_ids: list[str] | None = None, model: str | None = None, useragent_group: str | None = None, + api_key_ids: list[str] | None = None, ) -> datetime | None: conditions = [_normal_traffic_clause()] if account_ids: @@ -364,6 +387,8 @@ async def earliest_report_activity_at( useragent_group_clause = _useragent_group_filter_clause(useragent_group) if useragent_group_clause is not None: conditions.append(useragent_group_clause) + if api_key_ids: + conditions.append(RequestLog.api_key_id.in_(api_key_ids)) result = await self._session.execute(select(func.min(RequestLog.requested_at)).where(and_(*conditions))) value = result.scalar_one_or_none() @@ -376,6 +401,7 @@ def _report_conditions( account_ids: list[str] | None, model: str | None, useragent_group: str | None, + api_key_ids: list[str] | None = None, ) -> list: conditions = [ RequestLog.requested_at >= start_date, @@ -389,6 +415,8 @@ def _report_conditions( useragent_group_clause = _useragent_group_filter_clause(useragent_group) if useragent_group_clause is not None: conditions.append(useragent_group_clause) + if api_key_ids: + conditions.append(RequestLog.api_key_id.in_(api_key_ids)) return conditions @@ -435,6 +463,7 @@ def _daily_speed_medians_stmt( account_ids: list[str] | None, model: str | None, useragent_group: str | None, + api_key_ids: list[str] | None = None, ): useragent_group_clause = _useragent_group_filter_clause(useragent_group) day_ranges_cte = _day_ranges_cte(day_ranges) @@ -447,6 +476,7 @@ def _daily_speed_medians_stmt( *([RequestLog.account_id.in_(account_ids)] if account_ids else []), *([RequestLog.model == model] if model else []), *([useragent_group_clause] if useragent_group_clause is not None else []), + *([RequestLog.api_key_id.in_(api_key_ids)] if api_key_ids else []), ), ) token_count = RequestLog.output_tokens - func.coalesce(RequestLog.reasoning_tokens, 0) @@ -579,6 +609,7 @@ def _daily_rows_stmt( account_ids: list[str] | None, model: str | None, useragent_group: str | None, + api_key_ids: list[str] | None = None, *, include_conversations: bool = True, ): @@ -588,7 +619,11 @@ def _daily_rows_stmt( day_ranges_cte.c.report_date, func.count(RequestLog.id).label("requests"), func.coalesce(func.sum(RequestLog.input_tokens), 0).label("input_tokens"), - func.coalesce(func.sum(RequestLog.output_tokens), 0).label("output_tokens"), + func.coalesce( + func.sum(func.coalesce(RequestLog.output_tokens, RequestLog.reasoning_tokens, 0)), + 0, + ).label("output_tokens"), + func.sum(RequestLog.reasoning_tokens).label("reasoning_tokens"), func.coalesce(func.sum(RequestLog.cached_input_tokens), 0).label("cached_input_tokens"), func.coalesce(func.sum(RequestLog.cost_usd), 0.0).label("cost_usd"), func.count(func.distinct(RequestLog.account_id)).label("active_accounts"), @@ -615,6 +650,7 @@ def _daily_rows_stmt( *([RequestLog.account_id.in_(account_ids)] if account_ids else []), *([RequestLog.model == model] if model else []), *([useragent_group_clause] if useragent_group_clause is not None else []), + *([RequestLog.api_key_id.in_(api_key_ids)] if api_key_ids else []), ), ) ) diff --git a/app/modules/reports/schemas.py b/app/modules/reports/schemas.py index 2bcafd60d6..de10ba0a00 100644 --- a/app/modules/reports/schemas.py +++ b/app/modules/reports/schemas.py @@ -10,6 +10,7 @@ class DailyReportRow(DashboardModel): requests: int input_tokens: int output_tokens: int + reasoning_tokens: int | None cached_input_tokens: int cost_usd: float active_accounts: int @@ -46,6 +47,8 @@ class ReportSummary(DashboardModel): total_cost_usd: float total_input_tokens: int total_output_tokens: int + total_reasoning_tokens: int + reasoning_usage_known_requests: int total_cached_tokens: int total_requests: int total_errors: int diff --git a/app/modules/reports/service.py b/app/modules/reports/service.py index 12a65d2674..cfff634326 100644 --- a/app/modules/reports/service.py +++ b/app/modules/reports/service.py @@ -33,6 +33,7 @@ async def get_reports( account_ids: list[str] | None = None, model: str | None = None, useragent_group: str | None = None, + api_key_ids: list[str] | None = None, ) -> ReportsResponse: timezone_info = _resolve_timezone(report_timezone) now = utcnow().replace(tzinfo=timezone.utc).astimezone(timezone_info) @@ -53,15 +54,20 @@ async def get_reports( previous_start_at = _local_midnight_to_utc_naive(previous_start_date, timezone_info) previous_end_at = _local_midnight_to_utc_naive(previous_end_date + timedelta(days=1), timezone_info) - summary = await self._repository.aggregate_summary(start_at, end_at, account_ids, model, useragent_group) + summary = await self._repository.aggregate_summary( + start_at, end_at, account_ids, model, useragent_group, api_key_ids + ) previous_summary = await self._repository.aggregate_summary( previous_start_at, previous_end_at, account_ids, model, useragent_group, + api_key_ids, + ) + earliest_activity_at = await self._repository.earliest_report_activity_at( + account_ids, model, useragent_group, api_key_ids ) - earliest_activity_at = await self._repository.earliest_report_activity_at(account_ids, model, useragent_group) daily_rows = await self._repository.aggregate_daily_rows( start_date, end_date, @@ -69,6 +75,7 @@ async def get_reports( account_ids, model, useragent_group, + api_key_ids, ) daily = [ DailyReportRow( @@ -76,6 +83,7 @@ async def get_reports( requests=row.requests, input_tokens=row.input_tokens, output_tokens=row.output_tokens, + reasoning_tokens=row.reasoning_tokens, cached_input_tokens=row.cached_input_tokens, cost_usd=round(row.cost_usd, 4), active_accounts=row.active_accounts, @@ -88,14 +96,19 @@ async def get_reports( ) for row in daily_rows ] - by_model = await self._repository.aggregate_by_model(start_at, end_at, account_ids, model, useragent_group) - by_account = await self._repository.aggregate_by_account(start_at, end_at, account_ids, model, useragent_group) + by_model = await self._repository.aggregate_by_model( + start_at, end_at, account_ids, model, useragent_group, api_key_ids + ) + by_account = await self._repository.aggregate_by_account( + start_at, end_at, account_ids, model, useragent_group, api_key_ids + ) by_useragent = await self._repository.aggregate_by_useragent( start_at, end_at, account_ids, model, useragent_group, + api_key_ids, ) model_total = sum(m.cost_usd for m in by_model) @@ -114,6 +127,8 @@ async def get_reports( total_cost_usd=round(summary.total_cost_usd, 4), total_input_tokens=summary.total_input_tokens, total_output_tokens=summary.total_output_tokens, + total_reasoning_tokens=summary.total_reasoning_tokens, + reasoning_usage_known_requests=summary.reasoning_usage_known_requests, total_cached_tokens=summary.total_cached_tokens, total_requests=summary.total_requests, total_errors=summary.total_errors, diff --git a/app/modules/request_logs/mappers.py b/app/modules/request_logs/mappers.py index b61206075d..c6faa5c30c 100644 --- a/app/modules/request_logs/mappers.py +++ b/app/modules/request_logs/mappers.py @@ -3,6 +3,7 @@ from typing import cast as typing_cast from app.core.usage.logs import ( + CANCELLED_STATUS, RequestLogLike, cached_input_tokens_from_log, cost_breakdown_from_log, @@ -19,6 +20,8 @@ def normalize_log_status(status: str, error_code: str | None) -> str: if status == "success": return "ok" + if status == CANCELLED_STATUS: + return "cancelled" if error_code in RATE_LIMIT_CODES: return "rate_limit" if error_code in QUOTA_CODES: @@ -58,6 +61,11 @@ def to_request_log_entry( client_ip=log.client_ip if include_sensitive_metadata else None, transport=log.transport, upstream_transport=log.upstream_transport, + upstream_proxy_route_mode=log.upstream_proxy_route_mode, + upstream_proxy_pool_id=log.upstream_proxy_pool_id, + upstream_proxy_endpoint_id=log.upstream_proxy_endpoint_id, + upstream_proxy_fallback_used=log.upstream_proxy_fallback_used, + upstream_proxy_fail_closed_reason=log.upstream_proxy_fail_closed_reason, service_tier=log.service_tier, requested_service_tier=log.requested_service_tier, actual_service_tier=log.actual_service_tier, diff --git a/app/modules/request_logs/repository.py b/app/modules/request_logs/repository.py index 8a0676126b..79a65dac72 100644 --- a/app/modules/request_logs/repository.py +++ b/app/modules/request_logs/repository.py @@ -3,13 +3,15 @@ import time from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from typing import Any from typing import cast as typing_cast import anyio -from sqlalchemy import Integer, String, and_, case, cast, func, or_, select +from sqlalchemy import Integer, String, and_, case, cast, func, insert, or_, select from sqlalchemy import exc as sa_exc +from sqlalchemy.engine import CursorResult from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import InstrumentedAttribute +from sqlalchemy.orm import InstrumentedAttribute, make_transient_to_detached from sqlalchemy.sql.elements import ColumnElement from app.core.usage.logs import ( @@ -67,6 +69,14 @@ class _RequestLogFilters: # Earliest representable listing lower bound for the rollup-count window. _ROLLUP_EPOCH = datetime(1970, 1, 1) +# Column keys for the Core request-log insert in ``add_log``. Every non-PK +# column is read off the fully built transient instance; columns ``add_log`` +# never sets are nullable with no Python/server default the flush would have +# applied, so an explicit NULL is identical to the old unit-of-work insert. +_REQUEST_LOG_INSERT_COLUMN_KEYS: tuple[str, ...] = tuple( + column.key for column in RequestLog.__table__.columns if column.key != "id" +) + def _naive_utc(value: datetime) -> datetime: """FastAPI parses ISO `Z` query bounds as offset-aware datetimes; @@ -96,6 +106,7 @@ class _DemandCountParams: models: list[str] | None reasoning_efforts: list[str] | None include_success: bool + include_cancelled: bool include_error_other: bool @@ -1107,12 +1118,28 @@ async def add_log( if model_source_id is not None else calculated_cost_from_log(typing_cast(RequestLogLike, log)) ) - self._session.add(log) + # Core insert instead of unit-of-work: the row is fully built + # above, so the ORM flush (relationship cascade scan, + # per-attribute history snapshots) is pure overhead on every + # request's log write. No refresh: every column is set explicitly + # before insert. Once the primary key is known, the instance is + # re-attached as *persistent* (clean, no pending SQL) so callers + # that mutate the returned log and commit through the same + # session get a tracked UPDATE instead of a silent no-op + # (sessions run with ``expire_on_commit=False``, so the attach + # never triggers post-commit lazy loads). + insert_values = {key: getattr(log, key) for key in _REQUEST_LOG_INSERT_COLUMN_KEYS} try: + result = typing_cast( + CursorResult[Any], + await self._session.execute(insert(RequestLog).values(insert_values)), + ) + inserted_primary_key = result.inserted_primary_key + if inserted_primary_key is not None and inserted_primary_key[0] is not None: + log.id = int(inserted_primary_key[0]) + make_transient_to_detached(log) + self._session.add(log) await self._session.commit() - # No refresh: every column is set explicitly before insert and - # expire_on_commit=False, so the round trip was pure overhead - # on every request's log write. return log except sa_exc.ResourceClosedError: return log @@ -1206,6 +1233,7 @@ async def list_recent( models: list[str] | None = None, reasoning_efforts: list[str] | None = None, include_success: bool = True, + include_cancelled: bool = True, include_error_other: bool = True, error_codes_in: list[str] | None = None, error_codes_excluding: list[str] | None = None, @@ -1225,6 +1253,7 @@ async def list_recent( models=models, reasoning_efforts=reasoning_efforts, include_success=include_success, + include_cancelled=include_cancelled, include_error_other=include_error_other, error_codes_in=error_codes_in, error_codes_excluding=error_codes_excluding, @@ -1258,6 +1287,7 @@ async def list_recent( models=models, reasoning_efforts=reasoning_efforts, include_success=include_success, + include_cancelled=include_cancelled, include_error_other=include_error_other, ) @@ -1275,6 +1305,7 @@ async def list_recent( tuple(models or ()), tuple(reasoning_efforts or ()), include_success, + include_cancelled, include_error_other, tuple(sorted(error_codes_in)) if error_codes_in else None, tuple(sorted(error_codes_excluding)) if error_codes_excluding else None, @@ -1363,6 +1394,8 @@ async def _count_recent_from_demand_rollup( statuses = [] if params.include_success: statuses.append("success") + if params.include_cancelled: + statuses.append(CANCELLED_STATUS) if params.include_error_other: statuses.append("error") if statuses: @@ -1427,6 +1460,7 @@ async def list_filter_options( models=models, reasoning_efforts=reasoning_efforts, include_success=True, + include_cancelled=True, include_error_other=True, error_codes_in=None, error_codes_excluding=None, @@ -1441,6 +1475,7 @@ async def list_filter_options( models=models, reasoning_efforts=reasoning_efforts, include_success=True, + include_cancelled=True, include_error_other=True, error_codes_in=None, error_codes_excluding=None, @@ -1569,6 +1604,7 @@ def _build_filters( models: list[str] | None = None, reasoning_efforts: list[str] | None = None, include_success: bool = True, + include_cancelled: bool = True, include_error_other: bool = True, error_codes_in: list[str] | None = None, error_codes_excluding: list[str] | None = None, @@ -1610,6 +1646,8 @@ def _build_filters( status_conditions = [] if include_success: status_conditions.append(RequestLog.status == "success") + if include_cancelled: + status_conditions.append(RequestLog.status == CANCELLED_STATUS) if error_codes_in: status_conditions.append(and_(RequestLog.status == "error", RequestLog.error_code.in_(error_codes_in))) if include_error_other: diff --git a/app/modules/request_logs/schemas.py b/app/modules/request_logs/schemas.py index 261bdca976..d5cbcbe687 100644 --- a/app/modules/request_logs/schemas.py +++ b/app/modules/request_logs/schemas.py @@ -34,6 +34,11 @@ class RequestLogEntry(DashboardModel): client_ip: str | None = None transport: str | None = None upstream_transport: str | None = None + upstream_proxy_route_mode: str | None = None + upstream_proxy_pool_id: str | None = None + upstream_proxy_endpoint_id: str | None = None + upstream_proxy_fallback_used: bool | None = None + upstream_proxy_fail_closed_reason: str | None = None service_tier: str | None = None requested_service_tier: str | None = None actual_service_tier: str | None = None diff --git a/app/modules/request_logs/service.py b/app/modules/request_logs/service.py index 39f33b0ca8..1a5e20a43e 100644 --- a/app/modules/request_logs/service.py +++ b/app/modules/request_logs/service.py @@ -40,6 +40,7 @@ class RequestLogApiKeyOption: @dataclass(frozen=True, slots=True) class RequestLogStatusFilter: include_success: bool + include_cancelled: bool include_error_other: bool error_codes_in: list[str] | None error_codes_excluding: list[str] | None @@ -117,6 +118,7 @@ async def list_recent( models=models, reasoning_efforts=reasoning_efforts, include_success=status_filter.include_success, + include_cancelled=status_filter.include_cancelled, include_error_other=status_filter.include_error_other, error_codes_in=status_filter.error_codes_in, error_codes_excluding=status_filter.error_codes_excluding, @@ -230,6 +232,7 @@ def _map_status_filter(status: list[str] | None) -> RequestLogStatusFilter: if not status: return RequestLogStatusFilter( include_success=True, + include_cancelled=True, include_error_other=True, error_codes_in=None, error_codes_excluding=None, @@ -238,12 +241,14 @@ def _map_status_filter(status: list[str] | None) -> RequestLogStatusFilter: if not normalized or "all" in normalized: return RequestLogStatusFilter( include_success=True, + include_cancelled=True, include_error_other=True, error_codes_in=None, error_codes_excluding=None, ) include_success = "ok" in normalized + include_cancelled = "cancelled" in normalized include_rate_limit = "rate_limit" in normalized include_quota = "quota" in normalized include_error_other = "error" in normalized @@ -256,6 +261,7 @@ def _map_status_filter(status: list[str] | None) -> RequestLogStatusFilter: return RequestLogStatusFilter( include_success=include_success, + include_cancelled=include_cancelled, include_error_other=include_error_other, error_codes_in=sorted(error_codes_in) if error_codes_in else None, error_codes_excluding=sorted(RATE_LIMIT_CODES | QUOTA_CODES) if include_error_other else None, @@ -264,7 +270,7 @@ def _map_status_filter(status: list[str] | None) -> RequestLogStatusFilter: def _normalize_status_values(values: list[tuple[str, str | None]]) -> list[str]: normalized = {normalize_log_status(status, error_code) for status, error_code in values} - ordered = ["ok", "rate_limit", "quota", "error"] + ordered = ["ok", "cancelled", "rate_limit", "quota", "error"] return [status for status in ordered if status in normalized] diff --git a/app/modules/settings/api.py b/app/modules/settings/api.py index 5f0cf941cc..813af868a3 100644 --- a/app/modules/settings/api.py +++ b/app/modules/settings/api.py @@ -20,6 +20,7 @@ validate_dashboard_session, ) from app.core.clients.http import _build_ssl_context +from app.core.config.settings import get_settings as get_app_settings from app.core.config.settings_cache import get_settings_cache from app.core.crypto import TokenEncryptor from app.core.exceptions import DashboardBadRequestError, DashboardSettingsConflictError @@ -412,7 +413,10 @@ async def _validate_proxy_pool_id(context: SettingsContext, pool_id: str | None) async def _get_account_or_error(context: SettingsContext, account_id: str) -> Account: account = await context.session.get(Account, account_id) - if account is None: + # An account marked for background deletion is already deleted from the + # operator's point of view: binding mutations must report not-found, as + # the synchronous delete did once the row was removed. + if account is None or account.delete_requested_at is not None: raise DashboardBadRequestError("Account not found", code="account_not_found") return account @@ -562,6 +566,19 @@ async def update_settings( and payload.upstream_proxy_default_pool_id is not None ): await _validate_proxy_pool_id(context, payload.upstream_proxy_default_pool_id) + if ( + payload.auto_redeem_reset_credits_before_expiry + and not current.auto_redeem_reset_credits_before_expiry + and not get_app_settings().rate_limit_reset_credits_refresh_enabled + ): + # The reset-credit refresh loop is the sole driver of automatic + # redemption; accepting the opt-in while polling is disabled would + # persist a setting that can never run. + raise DashboardBadRequestError( + "autoRedeemResetCreditsBeforeExpiry requires reset-credit polling; " + "set CODEX_LB_RATE_LIMIT_RESET_CREDITS_REFRESH_ENABLED=true first", + code="reset_credit_polling_disabled", + ) try: legacy_threshold_provided = payload.sticky_reallocation_budget_threshold_pct is not None primary_threshold_provided = payload.sticky_reallocation_primary_budget_threshold_pct is not None diff --git a/app/modules/sticky_sessions/cleanup_scheduler.py b/app/modules/sticky_sessions/cleanup_scheduler.py index 3a310e8a67..4f95a48dc6 100644 --- a/app/modules/sticky_sessions/cleanup_scheduler.py +++ b/app/modules/sticky_sessions/cleanup_scheduler.py @@ -21,10 +21,6 @@ missing_durable_bridge_tables, ) from app.modules.proxy.ring_membership import RING_MEMBER_RETENTION_SECONDS, RingMembershipService -from app.modules.proxy.rowless_recovery_repository import ( - ROWLESS_RECOVERY_CAPTURED_RETENTION_SECONDS, - RowlessRecoveryRepository, -) from app.modules.proxy.sticky_repository import StickySessionsRepository from app.modules.settings.repository import SettingsRepository @@ -82,12 +78,15 @@ def _abandoned_bridge_retention_seconds( class StickySessionCleanupScheduler: interval_seconds: int enabled: bool + # Durable bridge transcript retention is a data-safety obligation and must + # continue even when operators disable sticky-session mapping cleanup. + operation_retention_enabled: bool = True _task: asyncio.Task[None] | None = None _stop: asyncio.Event = field(default_factory=asyncio.Event) _lock: asyncio.Lock = field(default_factory=asyncio.Lock) async def start(self) -> None: - if not self.enabled: + if not self.enabled and not self.operation_retention_enabled: return if self._task and not self._task.done(): return @@ -120,60 +119,75 @@ async def _cleanup_as_leader(self) -> None: async with get_background_session() as session: settings_repo = SettingsRepository(session) bridge_repo = DurableBridgeRepository(session) - rowless_repo = RowlessRecoveryRepository(session) sticky_repo = StickySessionsRepository(session) - settings = await settings_repo.get_or_create() - - cutoff = utcnow() - timedelta(seconds=settings.openai_cache_affinity_max_age_seconds) - deleted_count = await sticky_repo.purge_prompt_cache_before(cutoff) - if deleted_count > 0: - logger.info("Purged stale prompt-cache sticky sessions deleted_count=%s", deleted_count) - cleanup_now = utcnow() - stale_hard_codex_session_cutoff = cleanup_now - timedelta( - seconds=_STALE_HARD_CODEX_SESSION_UNAVAILABLE_SECONDS - ) - stale_hard_codex_session_deleted_count = await sticky_repo.purge_stale_hard_codex_session_mappings( - stale_hard_codex_session_cutoff, now=cleanup_now - ) - if stale_hard_codex_session_deleted_count > 0: - logger.info( - "Purged stale hard codex_session sticky mappings pinned to a durably unavailable " - "owner deleted_count=%s", - stale_hard_codex_session_deleted_count, - ) - if startup_module._bridge_durable_schema_ready or not await missing_durable_bridge_tables(session): - bridge_deleted_count = await bridge_repo.purge_closed_before(cutoff) - if bridge_deleted_count > 0: - logger.info("Purged closed HTTP bridge sessions deleted_count=%s", bridge_deleted_count) - abandoned_cutoff = utcnow() - timedelta( - seconds=_abandoned_bridge_retention_seconds(settings, get_settings()) + settings = await settings_repo.get_or_create() if self.enabled else None + + if self.enabled: + assert settings is not None + cutoff = utcnow() - timedelta(seconds=settings.openai_cache_affinity_max_age_seconds) + deleted_count = await sticky_repo.purge_prompt_cache_before(cutoff) + if deleted_count > 0: + logger.info("Purged stale prompt-cache sticky sessions deleted_count=%s", deleted_count) + cleanup_now = utcnow() + stale_hard_codex_session_cutoff = cleanup_now - timedelta( + seconds=_STALE_HARD_CODEX_SESSION_UNAVAILABLE_SECONDS ) - abandoned_deleted_count = await bridge_repo.purge_abandoned_before(abandoned_cutoff) - if abandoned_deleted_count > 0: - logger.info( - "Purged abandoned HTTP bridge sessions deleted_count=%s", abandoned_deleted_count + stale_hard_codex_session_deleted_count = ( + await sticky_repo.purge_stale_hard_codex_session_mappings( + stale_hard_codex_session_cutoff, now=cleanup_now ) - retry_circuit_deleted_count = await bridge_repo.purge_retry_circuits_before( - time.time() - DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS ) - if retry_circuit_deleted_count > 0: + if stale_hard_codex_session_deleted_count > 0: logger.info( - "Purged expired HTTP bridge retry circuits deleted_count=%s", - retry_circuit_deleted_count, + "Purged stale hard codex_session sticky mappings pinned to a durably unavailable " + "owner deleted_count=%s", + stale_hard_codex_session_deleted_count, ) - rowless_deleted = await rowless_repo.purge_expired_audit_rows( - captured_cutoff=cleanup_now - - timedelta(seconds=ROWLESS_RECOVERY_CAPTURED_RETENTION_SECONDS), - ) - if any(rowless_deleted.values()): - logger.info( - "Purged expired unapproved rowless recovery captures captured=%s", - rowless_deleted["captured"], + if startup_module._bridge_durable_schema_ready or not await missing_durable_bridge_tables(session): + if self.enabled: + assert settings is not None + bridge_deleted_count = await bridge_repo.purge_closed_before(cutoff) + if bridge_deleted_count > 0: + logger.info("Purged closed HTTP bridge sessions deleted_count=%s", bridge_deleted_count) + abandoned_cutoff = utcnow() - timedelta( + seconds=_abandoned_bridge_retention_seconds(settings, get_settings()) + ) + abandoned_deleted_count = await bridge_repo.purge_abandoned_before(abandoned_cutoff) + if abandoned_deleted_count > 0: + logger.info( + "Purged abandoned HTTP bridge sessions deleted_count=%s", abandoned_deleted_count + ) + retry_circuit_deleted_count = await bridge_repo.purge_retry_circuits_before( + time.time() - DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS + ) + if retry_circuit_deleted_count > 0: + logger.info( + "Purged expired HTTP bridge retry circuits deleted_count=%s", + retry_circuit_deleted_count, + ) + if self.operation_retention_enabled: + operation_cutoff = utcnow() - timedelta( + seconds=get_settings().http_responses_session_bridge_operation_spool_retention_seconds ) - ring_cutoff = utcnow() - timedelta(seconds=RING_MEMBER_RETENTION_SECONDS) - ring_deleted_count = await RingMembershipService(SessionLocal).purge_stale_before(ring_cutoff) - if ring_deleted_count > 0: - logger.info("Purged stale bridge ring members deleted_count=%s", ring_deleted_count) + operation_deleted_count = 0 + # Drain all eligible batches. A single startup pass is + # bounded to protect latency, but a long-lived process + # must continue pruning old prompt/output transcripts. + while True: + deleted_batch = await bridge_repo.purge_operation_spool(cutoff=operation_cutoff) + operation_deleted_count += deleted_batch + if deleted_batch < 500: + break + if operation_deleted_count > 0: + logger.info( + "Purged expired HTTP bridge operation transcript rows deleted_count=%s", + operation_deleted_count, + ) + if self.enabled: + ring_cutoff = utcnow() - timedelta(seconds=RING_MEMBER_RETENTION_SECONDS) + ring_deleted_count = await RingMembershipService(SessionLocal).purge_stale_before(ring_cutoff) + if ring_deleted_count > 0: + logger.info("Purged stale bridge ring members deleted_count=%s", ring_deleted_count) except Exception: logger.exception("Sticky session cleanup loop failed") diff --git a/app/modules/telemetry/__init__.py b/app/modules/telemetry/__init__.py new file mode 100644 index 0000000000..2c5dcf30e7 --- /dev/null +++ b/app/modules/telemetry/__init__.py @@ -0,0 +1,5 @@ +"""Anonymous, schema-allowlisted telemetry support.""" + +from app.modules.telemetry.snapshot import TelemetrySnapshotBuilder + +__all__ = ["TelemetrySnapshotBuilder"] diff --git a/app/modules/telemetry/api.py b/app/modules/telemetry/api.py new file mode 100644 index 0000000000..7f625d05c4 --- /dev/null +++ b/app/modules/telemetry/api.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import asyncio +import logging +import platform + +from fastapi import APIRouter, Body, Depends, Query +from sqlalchemy.ext.asyncio import AsyncSession + +from app import __version__ +from app.core.auth.dependencies import ( + require_dashboard_write_access, + set_dashboard_error_format, + validate_dashboard_session, +) +from app.db.session import get_session +from app.modules.telemetry.consent import ResolvedConsent, TelemetryConsentStore +from app.modules.telemetry.schemas import ( + TelemetryConsentResponse, + TelemetryConsentUpdate, + TelemetrySnapshotEnvelope, + build_snapshot_envelope, +) +from app.modules.telemetry.sender import TelemetrySender +from app.modules.telemetry.snapshot import TelemetrySnapshotBuilder, deployment_method + +logger = logging.getLogger(__name__) + +_OPT_OUT_TASKS: set[asyncio.Task[None]] = set() + +router = APIRouter( + prefix="/api/settings", + tags=["dashboard"], + dependencies=[Depends(validate_dashboard_session), Depends(set_dashboard_error_format)], +) + + +@router.get("/telemetry", response_model=TelemetryConsentResponse) +async def get_telemetry_consent( + include_preview: bool = Query(default=False), + session: AsyncSession = Depends(get_session), +) -> TelemetryConsentResponse: + store = TelemetryConsentStore(session) + consent = await store.resolve() + return await _response( + session, + store, + consent, + include_preview=include_preview or (consent.state == "undecided" and consent.source == "default"), + ) + + +@router.put("/telemetry", response_model=TelemetryConsentResponse) +async def update_telemetry_consent( + payload: TelemetryConsentUpdate = Body(...), + _write_access=Depends(require_dashboard_write_access), + session: AsyncSession = Depends(get_session), +) -> TelemetryConsentResponse: + store = TelemetryConsentStore(session) + previous = await store.resolve() + consent = await store.set_decision(payload.enabled) + if previous.active and not consent.active: + try: + identity = await store.get_or_create_identity() + task = asyncio.create_task( + TelemetrySender().send_opt_out( + identity, + app_version=__version__, + deployment_mode=deployment_method(), + os_arch=f"{platform.system().lower()}/{platform.machine().lower()}", + ), + name="anonymous-telemetry-opt-out", + ) + _OPT_OUT_TASKS.add(task) + task.add_done_callback(_handle_opt_out_task_done) + except Exception as exc: + logger.debug("Unable to schedule anonymous telemetry opt-out", exc_info=exc) + return await _response(session, store, consent, include_preview=False) + + +async def _response( + session: AsyncSession, + store: TelemetryConsentStore, + consent: ResolvedConsent, + *, + include_preview: bool, +) -> TelemetryConsentResponse: + preview: TelemetrySnapshotEnvelope | None = None + if include_preview: + identity = await store.get_or_create_identity() + snapshot_consent = "enabled" if consent.state == "disabled" else consent.state + snapshot = await TelemetrySnapshotBuilder(session).build( + identity.instance_id, + consent=snapshot_consent, + ) + preview = build_snapshot_envelope(snapshot) + return TelemetryConsentResponse( + state=consent.state, + source=consent.source, + active=consent.active, + preview=preview, + ) + + +def _handle_opt_out_task_done(task: asyncio.Task[None]) -> None: + try: + if task.cancelled(): + return + if exc := task.exception(): + logger.debug( + "Anonymous telemetry opt-out background task failed", + exc_info=(type(exc), exc, exc.__traceback__), + ) + finally: + _OPT_OUT_TASKS.discard(task) diff --git a/app/modules/telemetry/clients.py b/app/modules/telemetry/clients.py new file mode 100644 index 0000000000..4cf29496e2 --- /dev/null +++ b/app/modules/telemetry/clients.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass + +CLIENT_FAMILY_BY_RAW_GROUP: dict[str, str] = { + "codex_exec": "codex-cli", + "codex-tui": "codex-cli", + "Codex Desktop": "codex-desktop", + "codex_vscode": "codex-vscode", + "AsyncOpenAI": "openai-sdk-python", + "OpenAI": "openai-sdk-js", + "ai": "vercel-ai-sdk", + "ai-sdk": "vercel-ai-sdk", + "opencode": "opencode", + "Mozilla": "browser", + "curl": "script", + "undici": "script", + "node": "script", + "Python-urllib": "script", + "python-requests": "script", + "aiohttp": "script", +} + +CANONICAL_CLIENT_FAMILIES = frozenset({*CLIENT_FAMILY_BY_RAW_GROUP.values(), "other"}) + + +@dataclass(frozen=True, slots=True) +class ClientCount: + raw_group: str | None + requests: int + + +def client_family(raw_group: str | None) -> str: + return CLIENT_FAMILY_BY_RAW_GROUP.get(raw_group or "", "other") + + +def client_shares(rows: Iterable[ClientCount]) -> tuple[dict[str, float], float]: + counts: defaultdict[str, int] = defaultdict(int) + total = 0 + for row in rows: + requests = max(0, row.requests) + family = client_family(row.raw_group) + if family not in CANONICAL_CLIENT_FAMILIES: + raise ValueError(f"non-canonical telemetry client family: {family}") + counts[family] += requests + total += requests + if total == 0: + return {}, 0.0 + shares = {family: _ratio(count, total) for family, count in sorted(counts.items()) if count > 0} + return shares, shares.get("other", 0.0) + + +def catalog_model_name(model: str | None, catalog: frozenset[str]) -> str: + normalized = (model or "").strip() + return normalized if normalized in catalog else "other" + + +def _ratio(numerator: int | float, denominator: int | float) -> float: + return round(float(numerator) / float(denominator), 6) if denominator else 0.0 diff --git a/app/modules/telemetry/consent.py b/app/modules/telemetry/consent.py new file mode 100644 index 0000000000..4c3c9d2f2a --- /dev/null +++ b/app/modules/telemetry/consent.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import base64 +from dataclasses import dataclass +from typing import Literal, cast +from uuid import uuid4 + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, PublicFormat +from sqlalchemy import or_, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config.settings import Settings, get_settings +from app.core.crypto import TokenEncryptor +from app.db.models import DashboardSettings +from app.modules.settings.repository import SettingsRepository + +ConsentState = Literal["undecided", "enabled", "disabled"] +ConsentSource = Literal["env", "persisted", "default"] +_VALID_STATES = frozenset({"undecided", "enabled", "disabled"}) + + +@dataclass(frozen=True, slots=True) +class ResolvedConsent: + state: ConsentState + source: ConsentSource + active: bool + + +@dataclass(frozen=True, slots=True) +class TelemetryIdentity: + instance_id: str + private_key: Ed25519PrivateKey + + @property + def public_key_hex(self) -> str: + public_bytes = self.private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw) + return public_bytes.hex() + + +def resolve_consent(telemetry_enabled: bool | None, persisted_state: str) -> ResolvedConsent: + if telemetry_enabled is not None: + state: ConsentState = "enabled" if telemetry_enabled else "disabled" + return ResolvedConsent(state=state, source="env", active=telemetry_enabled) + if persisted_state not in _VALID_STATES: + raise ValueError(f"invalid telemetry consent state: {persisted_state}") + state = cast("ConsentState", persisted_state) + if state == "undecided": + return ResolvedConsent(state="undecided", source="default", active=True) + return ResolvedConsent(state=state, source="persisted", active=state == "enabled") + + +class TelemetryConsentStore: + def __init__( + self, + session: AsyncSession, + *, + settings: Settings | None = None, + encryptor: TokenEncryptor | None = None, + ) -> None: + self._session = session + self._settings = settings or get_settings() + self._encryptor = encryptor or TokenEncryptor() + self._repository = SettingsRepository(session) + + async def resolve(self) -> ResolvedConsent: + row = await self._repository.get_or_create() + return resolve_consent(self._settings.telemetry_enabled, row.telemetry_consent) + + async def set_decision(self, enabled: bool) -> ResolvedConsent: + row = await self._repository.get_or_create() + row.telemetry_consent = "enabled" if enabled else "disabled" + await self._repository.commit_refresh(row) + return resolve_consent(self._settings.telemetry_enabled, row.telemetry_consent) + + async def get_or_create_identity(self) -> TelemetryIdentity: + row = await self._repository.get_or_create() + if row.telemetry_instance_id is None or row.telemetry_private_key_encrypted is None: + await self._mint_identity_if_missing() + self._session.expire_all() + row = await self._repository.get_or_create() + if row.telemetry_instance_id is None or row.telemetry_private_key_encrypted is None: + raise RuntimeError("telemetry identity could not be persisted") + raw_private_key = base64.b64decode(self._encryptor.decrypt(row.telemetry_private_key_encrypted)) + private_key = Ed25519PrivateKey.from_private_bytes(raw_private_key) + return TelemetryIdentity(instance_id=row.telemetry_instance_id, private_key=private_key) + + async def _mint_identity_if_missing(self) -> None: + private_key = Ed25519PrivateKey.generate() + raw_private_key = private_key.private_bytes(Encoding.Raw, PrivateFormat.Raw, NoEncryption()) + encrypted = self._encryptor.encrypt(base64.b64encode(raw_private_key).decode("ascii")) + await self._session.execute( + update(DashboardSettings) + .where( + DashboardSettings.id == 1, + or_( + DashboardSettings.telemetry_instance_id.is_(None), + DashboardSettings.telemetry_private_key_encrypted.is_(None), + ), + ) + .values( + telemetry_instance_id=str(uuid4()), + telemetry_private_key_encrypted=encrypted, + version=DashboardSettings.version + 1, + ) + .execution_options(synchronize_session=False) + ) + await self._session.commit() diff --git a/app/modules/telemetry/scheduler.py b/app/modules/telemetry/scheduler.py new file mode 100644 index 0000000000..ea1ee2fc9d --- /dev/null +++ b/app/modules/telemetry/scheduler.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import asyncio +import contextlib +import importlib +import logging +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from functools import partial +from typing import Protocol, TypeVar, cast + +from app.db.session import get_background_session +from app.modules.telemetry.consent import TelemetryConsentStore +from app.modules.telemetry.sender import TelemetrySender +from app.modules.telemetry.snapshot import TelemetrySnapshotBuilder + +logger = logging.getLogger(__name__) + +TELEMETRY_INTERVAL_SECONDS = 24 * 60 * 60 +TELEMETRY_FIELDS_DOCUMENTATION = "https://soju06.github.io/codex-lb/telemetry/" + +_T = TypeVar("_T") + + +class _LeaderElectionLike(Protocol): + async def run_if_leader(self, fn: Callable[[], Awaitable[_T]]) -> _T | None: ... + + +def _get_leader_election() -> _LeaderElectionLike: + module = importlib.import_module("app.core.scheduling.leader_election") + return cast(_LeaderElectionLike, module.get_leader_election()) + + +@dataclass(slots=True) +class TelemetryScheduler: + sender: TelemetrySender = field(default_factory=TelemetrySender) + interval_seconds: float = TELEMETRY_INTERVAL_SECONDS + _task: asyncio.Task[None] | None = None + _stop: asyncio.Event = field(default_factory=asyncio.Event) + _lock: asyncio.Lock = field(default_factory=asyncio.Lock) + + async def start(self) -> None: + if self._task and not self._task.done(): + return + self._stop.clear() + self._task = asyncio.create_task(self._run_loop(), name="anonymous-telemetry-scheduler") + + async def stop(self) -> None: + self._stop.set() + if self._task is None: + return + self._task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._task + self._task = None + + async def _run_loop(self) -> None: + first_tick = True + while not self._stop.is_set(): + await self._tick(log_undecided_notice=first_tick) + first_tick = False + try: + await asyncio.wait_for(self._stop.wait(), timeout=self.interval_seconds) + except asyncio.TimeoutError: + continue + + async def _tick(self, *, log_undecided_notice: bool = False) -> None: + await _get_leader_election().run_if_leader( + partial(self._tick_as_leader, log_undecided_notice=log_undecided_notice) + ) + + async def _tick_as_leader(self, *, log_undecided_notice: bool = False) -> None: + async with self._lock: + try: + async with get_background_session() as session: + store = TelemetryConsentStore(session) + consent = await store.resolve() + if log_undecided_notice and consent.state == "undecided" and consent.source == "default": + logger.info( + "Anonymous telemetry is active; collected fields: %s; disable with " + "CODEX_LB_TELEMETRY_ENABLED=false", + TELEMETRY_FIELDS_DOCUMENTATION, + ) + if not consent.active: + return + assert consent.state != "disabled" + identity = await store.get_or_create_identity() + snapshot = await TelemetrySnapshotBuilder(session).build( + identity.instance_id, + consent=consent.state, + ) + await self.sender.send_snapshot(snapshot) + except Exception as exc: + logger.debug("Anonymous telemetry scheduler tick failed", exc_info=exc) + + +def build_telemetry_scheduler() -> TelemetryScheduler: + return TelemetryScheduler() diff --git a/app/modules/telemetry/schemas.py b/app/modules/telemetry/schemas.py new file mode 100644 index 0000000000..d8d4947f70 --- /dev/null +++ b/app/modules/telemetry/schemas.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +DeploymentMethod = Literal["docker", "k8s", "pip", "bare"] +ActiveConsentState = Literal["undecided", "enabled"] + + +class TelemetryModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class DeploymentSnapshot(TelemetryModel): + method: DeploymentMethod + db_backend: Literal["sqlite", "postgres"] + db_size_bucket: Literal["unknown", "<100MB", "100MB-1GB", "1-5GB", "5-10GB", "10-50GB", "50GB+"] + replicas: int = Field(ge=1) + reverse_proxy: bool + + +class PlanMixSnapshot(TelemetryModel): + plus: str + pro: str + team: str + free: str + + +class AccountsSnapshot(TelemetryModel): + pool_bucket: str + plan_mix: PlanMixSnapshot + workspace_accounts: bool + routing_policy: str + limit_warmup_enabled: bool + egress_proxy_used: bool + + +class RequestKindsSnapshot(TelemetryModel): + responses: float + chat: float + images: float + unknown: float + + +class TransportMixSnapshot(TelemetryModel): + ws: float + http_bridge: float + + +class ServiceTierMixSnapshot(TelemetryModel): + default: float + flex: float + priority: float + + +class ModelUsageSnapshot(TelemetryModel): + name: str + share: float + reasoning: dict[str, float] + avg_output_tokens_bucket: str + + +class UsageSnapshot(TelemetryModel): + requests: int = Field(ge=0) + success_rate: float = Field(ge=0.0, le=1.0) + tokens_input: int = Field(ge=0) + tokens_output: int = Field(ge=0) + tokens_cached_ratio: float = Field(ge=0.0, le=1.0) + cost_usd_bucket: str + request_kinds: RequestKindsSnapshot + transport_mix: TransportMixSnapshot + service_tier_mix: ServiceTierMixSnapshot + clients: dict[str, float] + clients_other_ratio: float = Field(ge=0.0, le=1.0) + models: list[ModelUsageSnapshot] + latency_ms_p50: int = Field(ge=0) + ttft_ms_p50: int = Field(ge=0) + ttft_ms_p95: int = Field(ge=0) + rate_limit_429_ratio: float = Field(ge=0.0, le=1.0) + top_upstream_errors: list[str] = Field(max_length=5) + + +class FeaturesSnapshot(TelemetryModel): + api_firewall: bool + quota_planner: bool + sticky_sessions: bool + conversation_archive: bool + automations: bool + fleet: bool + model_sources_count: int = Field(ge=0) + api_keys_bucket: str + prometheus: bool + otel: bool + dashboard_auth: bool + reset_credits: bool + image_api_used: bool + + +class TelemetrySnapshot(TelemetryModel): + schema_version: Literal[1] = 1 + consent: ActiveConsentState + instance_id: str + version: str + python: str + os: str + arch: str + uptime_hours: int = Field(ge=0) + deploy: DeploymentSnapshot + accounts: AccountsSnapshot + usage_7d: UsageSnapshot + features: FeaturesSnapshot + + +class TelemetryRegistration(TelemetryModel): + app_name: Literal["codex-lb"] = "codex-lb" + app_version: str + deployment_mode: DeploymentMethod + environment: str = "" + instance_id: str + os_arch: str + public_key: str + + +class TelemetryActivation(TelemetryModel): + action: Literal["activate"] = "activate" + + +class TelemetryOptOut(TelemetryModel): + app_version: str + event: Literal["optout"] = "optout" + instance_id: str + occurred_at: str + + +class TelemetrySnapshotEnvelope(TelemetryModel): + instance_id: str + metrics: TelemetrySnapshot + timestamp: datetime + + +def build_snapshot_envelope( + snapshot: TelemetrySnapshot, + *, + timestamp: datetime | None = None, +) -> TelemetrySnapshotEnvelope: + return TelemetrySnapshotEnvelope( + instance_id=snapshot.instance_id, + metrics=snapshot, + timestamp=timestamp or datetime.now(UTC), + ) + + +class TelemetryConsentUpdate(TelemetryModel): + enabled: bool + + +class TelemetryConsentResponse(TelemetryModel): + state: Literal["undecided", "enabled", "disabled"] + source: Literal["env", "persisted", "default"] + active: bool + preview: TelemetrySnapshotEnvelope | None diff --git a/app/modules/telemetry/sender.py b/app/modules/telemetry/sender.py new file mode 100644 index 0000000000..6b303d17b4 --- /dev/null +++ b/app/modules/telemetry/sender.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import asyncio +import json +import logging +from collections.abc import Awaitable, Callable + +import aiohttp + +from app.core.config.settings import get_settings +from app.core.utils.time import utcnow +from app.db.session import get_background_session +from app.modules.telemetry.consent import TelemetryConsentStore, TelemetryIdentity +from app.modules.telemetry.schemas import ( + DeploymentMethod, + TelemetryActivation, + TelemetryModel, + TelemetryOptOut, + TelemetryRegistration, + TelemetrySnapshot, + build_snapshot_envelope, +) + +logger = logging.getLogger(__name__) + +_TIMEOUT_SECONDS = 5.0 +_MAX_ATTEMPTS = 2 +SenderContextProvider = Callable[[], Awaitable[tuple[bool, TelemetryIdentity | None]]] + + +class TelemetryProtocolError(RuntimeError): + pass + + +class TelemetrySender: + def __init__( + self, + endpoint: str | None = None, + *, + context_provider: SenderContextProvider | None = None, + ) -> None: + self._endpoint = (endpoint or get_settings().telemetry_endpoint).rstrip("/") + self._context_provider = context_provider or _load_sender_context + self._activated_instance_id: str | None = None + + async def send_snapshot(self, snapshot: TelemetrySnapshot) -> None: + try: + active, identity = await self._context_provider() + if not active: + return + if identity is None: + raise TelemetryProtocolError("active telemetry has no identity") + if snapshot.instance_id != identity.instance_id: + raise TelemetryProtocolError("snapshot identity does not match persisted telemetry identity") + async with asyncio.timeout(_TIMEOUT_SECONDS): + timeout = aiohttp.ClientTimeout(total=_TIMEOUT_SECONDS) + async with aiohttp.ClientSession(timeout=timeout, trust_env=False) as session: + await self._send_with_retry(lambda: self._transmit_once(session, snapshot, identity)) + except Exception as exc: + logger.debug("Anonymous telemetry transmission failed", exc_info=exc) + + async def send_opt_out( + self, + identity: TelemetryIdentity, + *, + app_version: str, + deployment_mode: DeploymentMethod, + os_arch: str, + ) -> None: + try: + event = TelemetryOptOut( + app_version=app_version, + instance_id=identity.instance_id, + occurred_at=f"{utcnow().isoformat()}Z", + ) + async with asyncio.timeout(_TIMEOUT_SECONDS): + timeout = aiohttp.ClientTimeout(total=_TIMEOUT_SECONDS) + async with aiohttp.ClientSession(timeout=timeout, trust_env=False) as session: + await self._send_with_retry( + lambda: self._transmit_opt_out_once( + session, + event, + identity, + deployment_mode=deployment_mode, + os_arch=os_arch, + ) + ) + except Exception as exc: + logger.debug("Anonymous telemetry opt-out transmission failed", exc_info=exc) + + async def _send_with_retry(self, operation: Callable[[], Awaitable[None]]) -> None: + last_error: Exception | None = None + for attempt in range(_MAX_ATTEMPTS): + try: + await operation() + return + except Exception as exc: + last_error = exc + logger.debug("Anonymous telemetry attempt %d failed", attempt + 1, exc_info=exc) + if last_error is not None: + raise last_error + + async def _transmit_once( + self, + session: aiohttp.ClientSession, + snapshot: TelemetrySnapshot, + identity: TelemetryIdentity, + ) -> None: + await self._ensure_activated( + session, + identity, + app_version=snapshot.version, + deployment_mode=snapshot.deploy.method, + os_arch=f"{snapshot.os}/{snapshot.arch}", + ) + + envelope = build_snapshot_envelope(snapshot) + try: + active, current_identity = await self._context_provider() + identity_matches = ( + current_identity is not None + and current_identity.instance_id == identity.instance_id + and current_identity.public_key_hex == identity.public_key_hex + ) + except Exception as exc: + logger.debug("Anonymous telemetry consent re-check failed", exc_info=exc) + return + if not active or not identity_matches: + return + + await self._post_signed(session, "/v1/snapshot", _json_bytes(envelope), identity, accepted={200, 202}) + + async def _transmit_opt_out_once( + self, + session: aiohttp.ClientSession, + event: TelemetryOptOut, + identity: TelemetryIdentity, + *, + deployment_mode: DeploymentMethod, + os_arch: str, + ) -> None: + await self._ensure_activated( + session, + identity, + app_version=event.app_version, + deployment_mode=deployment_mode, + os_arch=os_arch, + ) + await self._post_signed(session, "/v1/optout", _json_bytes(event), identity, accepted={200}) + + async def _ensure_activated( + self, + session: aiohttp.ClientSession, + identity: TelemetryIdentity, + *, + app_version: str, + deployment_mode: DeploymentMethod, + os_arch: str, + ) -> None: + if self._activated_instance_id == identity.instance_id: + return + registration = TelemetryRegistration( + app_version=app_version, + deployment_mode=deployment_mode, + instance_id=identity.instance_id, + os_arch=os_arch, + public_key=identity.public_key_hex, + ) + await self._post(session, "/v1/register", _json_bytes(registration), accepted={200, 201}) + + activation = TelemetryActivation() + await self._post_signed(session, "/v1/activate", _json_bytes(activation), identity, accepted={200}) + self._activated_instance_id = identity.instance_id + + async def _post_signed( + self, + session: aiohttp.ClientSession, + path: str, + body: bytes, + identity: TelemetryIdentity, + *, + accepted: set[int], + ) -> None: + await self._post( + session, + path, + body, + accepted=accepted, + headers={ + "X-Instance-ID": identity.instance_id, + "X-Signature": identity.private_key.sign(body).hex(), + }, + ) + + async def _post( + self, + session: aiohttp.ClientSession, + path: str, + body: bytes, + *, + accepted: set[int], + headers: dict[str, str] | None = None, + ) -> None: + request_headers = {"Content-Type": "application/json", **(headers or {})} + async with session.post(f"{self._endpoint}{path}", data=body, headers=request_headers) as response: + await response.read() + if response.status not in accepted: + raise TelemetryProtocolError(f"SHM {path} returned HTTP {response.status}") + + +async def _load_sender_context() -> tuple[bool, TelemetryIdentity | None]: + async with get_background_session() as session: + store = TelemetryConsentStore(session) + consent = await store.resolve() + if not consent.active: + return False, None + return True, await store.get_or_create_identity() + + +def _json_bytes(value: TelemetryModel) -> bytes: + return json.dumps(value.model_dump(mode="json"), separators=(",", ":"), sort_keys=True).encode("utf-8") diff --git a/app/modules/telemetry/snapshot.py b/app/modules/telemetry/snapshot.py new file mode 100644 index 0000000000..88d9e5cc4f --- /dev/null +++ b/app/modules/telemetry/snapshot.py @@ -0,0 +1,482 @@ +from __future__ import annotations + +import importlib.metadata +import logging +import os +import platform +import time +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import timedelta +from pathlib import Path +from typing import Literal, get_args + +from sqlalchemy import and_, case, func, select, text +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import InstrumentedAttribute +from sqlalchemy.sql.elements import ColumnElement + +from app import __version__ +from app.core.auth.dashboard_mode import DashboardAuthMode +from app.core.balancer.logic import RoutingStrategy +from app.core.config.settings import Settings, get_settings +from app.core.openai.model_registry import get_model_registry +from app.core.usage.logs import NON_ERROR_STATUSES +from app.core.utils.time import utcnow +from app.db.models import ( + Account, + ApiFirewallAllowlist, + ApiKey, + AutomationJob, + ModelSource, + ProxyEndpoint, + QuotaPlannerSettings, + RequestLog, +) +from app.db.sqlite_utils import sqlite_db_path_from_url +from app.modules.reports.repository import ReportsRepository, _report_conditions +from app.modules.settings.repository import SettingsRepository +from app.modules.telemetry.clients import ClientCount, catalog_model_name, client_shares +from app.modules.telemetry.schemas import ( + AccountsSnapshot, + ActiveConsentState, + DeploymentMethod, + DeploymentSnapshot, + FeaturesSnapshot, + ModelUsageSnapshot, + PlanMixSnapshot, + RequestKindsSnapshot, + ServiceTierMixSnapshot, + TelemetrySnapshot, + TransportMixSnapshot, + UsageSnapshot, +) + +_PROCESS_STARTED = time.monotonic() +_MIB = 1024**2 +_GIB = 1024**3 +_REASONING_EFFORTS = frozenset({"minimal", "low", "medium", "high", "xhigh", "max", "ultra"}) +_ROUTING_POLICIES: frozenset[str] = frozenset(get_args(RoutingStrategy)) +_SAFE_UPSTREAM_ERROR_CODES = frozenset( + { + "authentication_error", + "billing_not_active", + "context_length_exceeded", + "insufficient_quota", + "invalid_api_key", + "invalid_request_error", + "model_not_found", + "rate_limit_exceeded", + "server_error", + "server_overloaded", + "service_unavailable", + "usage_limit_reached", + "upstream_unavailable", + } +) + +logger = logging.getLogger(__name__) + +type Predicate = ColumnElement[bool] +type NullableIntegerColumn = InstrumentedAttribute[int | None] + + +def count_bucket(value: int) -> str: + if value <= 0: + return "0" + if value == 1: + return "1" + if value <= 5: + return "2-5" + if value <= 20: + return "6-20" + if value <= 100: + return "21-100" + return "100+" + + +def db_size_bucket( + size_bytes: int | None, +) -> Literal["unknown", "<100MB", "100MB-1GB", "1-5GB", "5-10GB", "10-50GB", "50GB+"]: + if size_bytes is None: + return "unknown" + if size_bytes < 100 * _MIB: + return "<100MB" + if size_bytes < _GIB: + return "100MB-1GB" + if size_bytes < 5 * _GIB: + return "1-5GB" + if size_bytes < 10 * _GIB: + return "5-10GB" + if size_bytes < 50 * _GIB: + return "10-50GB" + return "50GB+" + + +def cost_bucket(cost_usd: float) -> str: + if cost_usd < 10: + return "<10" + if cost_usd < 100: + return "10-100" + if cost_usd < 1_000: + return "100-1k" + if cost_usd < 10_000: + return "1k-10k" + if cost_usd < 50_000: + return "10k-50k" + return "50k+" + + +def output_tokens_bucket(tokens: float) -> str: + if tokens < 250: + return "<250" + if tokens < 1_000: + return "250-1k" + if tokens < 4_000: + return "1k-4k" + if tokens < 16_000: + return "4k-16k" + return "16k+" + + +def _ratio(numerator: int | float, denominator: int | float) -> float: + if not denominator: + return 0.0 + return round(min(1.0, max(0.0, float(numerator) / float(denominator))), 6) + + +@dataclass(slots=True) +class _ModelAccumulator: + requests: int = 0 + output_tokens: int = 0 + reasoning_counts: dict[str, int] = field(default_factory=lambda: defaultdict(int)) + + +class TelemetrySnapshotBuilder: + def __init__(self, session: AsyncSession, *, settings: Settings | None = None) -> None: + self._session = session + self._settings = settings or get_settings() + + async def build( + self, + instance_id: str, + *, + consent: ActiveConsentState, + ) -> TelemetrySnapshot: + now = utcnow() + start = now - timedelta(days=7) + reports = ReportsRepository(self._session) + summary = await reports.aggregate_summary(start, now) + ua_rows = await reports.aggregate_by_useragent(start, now) + clients, clients_other_ratio = client_shares( + ClientCount(raw_group=row.useragent_group, requests=row.request_count) for row in ua_rows + ) + conditions = _report_conditions(start, now, None, None, None) + dashboard_settings = await SettingsRepository(self._session).get_or_create() + + account_count, workspace_accounts, plan_counts = await self._account_aggregates() + database_size = await self._database_size_bytes() + models = await self._model_usage(conditions, summary.total_requests) + request_kinds = self._request_kind_mix(summary.total_requests) + transport_mix = await self._transport_mix(conditions, summary.total_requests) + service_tier_mix = await self._service_tier_mix(conditions, summary.total_requests) + latency_p50 = await self._percentile(RequestLog.latency_ms, conditions, 0.50) + ttft_p50 = await self._percentile(RequestLog.latency_first_token_ms, conditions, 0.50) + ttft_p95 = await self._percentile(RequestLog.latency_first_token_ms, conditions, 0.95) + rate_limit_429_count = await self._count_where(conditions, RequestLog.upstream_status_code == 429) + top_errors = await self._top_upstream_errors(conditions) + feature_counts = await self._feature_counts(conditions) + + method = deployment_method() + db_backend = "postgres" if self._session.get_bind().dialect.name == "postgresql" else "sqlite" + plan_mix = PlanMixSnapshot( + plus=count_bucket(plan_counts.get("plus", 0)), + pro=count_bucket(plan_counts.get("pro", 0)), + team=count_bucket(plan_counts.get("team", 0)), + free=count_bucket(plan_counts.get("free", 0)), + ) + return TelemetrySnapshot( + consent=consent, + instance_id=instance_id, + version=__version__, + python=f"{platform.python_version_tuple()[0]}.{platform.python_version_tuple()[1]}", + os=platform.system().lower(), + arch=platform.machine().lower(), + uptime_hours=max(0, int((time.monotonic() - _PROCESS_STARTED) // 3600)), + deploy=DeploymentSnapshot( + method=method, + db_backend=db_backend, + db_size_bucket=db_size_bucket(database_size), + replicas=max(1, len(self._settings.http_responses_session_bridge_instance_ring)), + reverse_proxy=self._settings.firewall_trust_proxy_headers, + ), + accounts=AccountsSnapshot( + pool_bucket=count_bucket(account_count), + plan_mix=plan_mix, + workspace_accounts=workspace_accounts, + routing_policy=_canonical_routing_policy(dashboard_settings.routing_strategy), + limit_warmup_enabled=dashboard_settings.limit_warmup_enabled, + egress_proxy_used=( + dashboard_settings.upstream_proxy_routing_enabled or feature_counts.active_proxy_endpoints > 0 + ), + ), + usage_7d=UsageSnapshot( + requests=summary.total_requests, + # Cancelled terminals are neither errors nor successes + # (NON_ERROR_STATUSES); counting them as successes would + # inflate the rate on disconnect-heavy workloads. + success_rate=_ratio( + summary.total_requests - summary.total_errors - summary.total_cancelled, + summary.total_requests, + ), + tokens_input=summary.total_input_tokens, + tokens_output=summary.total_output_tokens, + tokens_cached_ratio=_ratio(summary.total_cached_tokens, summary.total_input_tokens), + cost_usd_bucket=cost_bucket(max(0.0, summary.total_cost_usd)), + request_kinds=request_kinds, + transport_mix=transport_mix, + service_tier_mix=service_tier_mix, + clients=clients, + clients_other_ratio=clients_other_ratio, + models=models, + latency_ms_p50=latency_p50, + ttft_ms_p50=ttft_p50, + ttft_ms_p95=ttft_p95, + rate_limit_429_ratio=_ratio(rate_limit_429_count, summary.total_requests), + top_upstream_errors=top_errors, + ), + features=FeaturesSnapshot( + api_firewall=feature_counts.firewall_entries > 0, + quota_planner=( + self._settings.quota_planner_scheduler_enabled and feature_counts.quota_planner_mode != "off" + ), + sticky_sessions=dashboard_settings.sticky_threads_enabled, + conversation_archive=self._settings.conversation_archive_enabled, + automations=(self._settings.automations_scheduler_enabled and feature_counts.enabled_automations > 0), + fleet=True, + model_sources_count=feature_counts.model_sources, + api_keys_bucket=count_bucket(feature_counts.api_keys), + prometheus=self._settings.metrics_enabled, + otel=self._settings.otel_enabled, + dashboard_auth=self._settings.dashboard_auth_mode != DashboardAuthMode.DISABLED, + reset_credits=( + dashboard_settings.show_reset_credit_badges + or dashboard_settings.auto_redeem_reset_credits_before_expiry + ), + image_api_used=feature_counts.image_requests > 0, + ), + ) + + async def _account_aggregates(self) -> tuple[int, bool, dict[str, int]]: + result = await self._session.execute( + select( + func.count().label("accounts"), + func.coalesce(func.sum(case((Account.workspace_id.is_not(None), 1), else_=0)), 0).label("workspace"), + ) + ) + row = result.one() + plan_result = await self._session.execute(select(Account.plan_type, func.count()).group_by(Account.plan_type)) + plan_counts: defaultdict[str, int] = defaultdict(int) + for raw_plan, raw_count in plan_result.all(): + plan_counts[_canonical_plan(raw_plan)] += int(raw_count) + return int(row.accounts), bool(row.workspace), dict(plan_counts) + + async def _model_usage(self, conditions: list[Predicate], total_requests: int) -> list[ModelUsageSnapshot]: + result = await self._session.execute( + select( + RequestLog.model, + RequestLog.reasoning_effort, + func.count().label("requests"), + func.coalesce(func.sum(RequestLog.output_tokens), 0).label("output_tokens"), + ) + .where(and_(*conditions)) + .group_by(RequestLog.model, RequestLog.reasoning_effort) + ) + catalog = frozenset(get_model_registry().get_models_with_fallback()) + grouped: defaultdict[str, _ModelAccumulator] = defaultdict(_ModelAccumulator) + for row in result.all(): + name = catalog_model_name(row.model, catalog) + accumulator = grouped[name] + count = int(row.requests) + accumulator.requests += count + accumulator.output_tokens += int(row.output_tokens) + reasoning = _canonical_reasoning(row.reasoning_effort) + accumulator.reasoning_counts[reasoning] += count + return [ + ModelUsageSnapshot( + name=name, + share=_ratio(values.requests, total_requests), + reasoning={ + reasoning: _ratio(count, values.requests) + for reasoning, count in sorted(values.reasoning_counts.items()) + }, + avg_output_tokens_bucket=output_tokens_bucket( + values.output_tokens / values.requests if values.requests else 0 + ), + ) + for name, values in sorted(grouped.items()) + ] + + def _request_kind_mix(self, total: int) -> RequestKindsSnapshot: + # ``request_logs.request_kind`` records workload classes such as + # normal/warmup/compaction, not the ingress route family. Chat, + # Responses, images, and audio can therefore be indistinguishable in + # persisted rows. Report that limitation instead of inferring a route + # from the upstream source or model name. + return RequestKindsSnapshot( + responses=0.0, + chat=0.0, + images=0.0, + unknown=1.0 if total else 0.0, + ) + + async def _transport_mix(self, conditions: list[Predicate], total: int) -> TransportMixSnapshot: + websocket = await self._count_where(conditions, RequestLog.transport == "websocket") + return TransportMixSnapshot(ws=_ratio(websocket, total), http_bridge=_ratio(total - websocket, total)) + + async def _service_tier_mix(self, conditions: list[Predicate], total: int) -> ServiceTierMixSnapshot: + # "fast" is normalized to "priority" at write time + # (_normalize_service_tier_value), so the persisted vocabulary here is + # default/flex/priority; lumping priority into default would hide fast + # mode traffic from the mix. + tier = func.coalesce(RequestLog.actual_service_tier, RequestLog.service_tier, "default") + flex = await self._count_where(conditions, tier == "flex") + priority = await self._count_where(conditions, tier == "priority") + return ServiceTierMixSnapshot( + default=_ratio(total - flex - priority, total), + flex=_ratio(flex, total), + priority=_ratio(priority, total), + ) + + async def _percentile( + self, + column: NullableIntegerColumn, + conditions: list[Predicate], + quantile: float, + ) -> int: + count_result = await self._session.execute( + select(func.count()).where(and_(*conditions, column.is_not(None), column >= 0)) + ) + count = int(count_result.scalar_one()) + if count == 0: + return 0 + rank = min(count - 1, max(0, int((count - 1) * quantile + 0.5))) + result = await self._session.execute( + select(column) + .where(and_(*conditions, column.is_not(None), column >= 0)) + .order_by(column) + .offset(rank) + .limit(1) + ) + value = result.scalar_one() + if value is None: + raise RuntimeError("percentile query returned a null value after a non-null filter") + return int(value) + + async def _count_where(self, conditions: list[Predicate], *extra_conditions: Predicate) -> int: + result = await self._session.execute(select(func.count()).where(and_(*conditions, *extra_conditions))) + return int(result.scalar_one()) + + async def _top_upstream_errors(self, conditions: list[Predicate]) -> list[str]: + # Cancelled rows keep upstream_error_code='client_disconnected', so + # filtering on the code alone would let routine disconnects displace + # genuine upstream failures; restrict to actual error statuses like + # the other error-metric surfaces. + result = await self._session.execute( + select(RequestLog.upstream_error_code, func.count().label("requests")) + .where( + and_( + *conditions, + RequestLog.upstream_error_code.is_not(None), + RequestLog.status.not_in(NON_ERROR_STATUSES), + ) + ) + .group_by(RequestLog.upstream_error_code) + ) + counts: defaultdict[str, int] = defaultdict(int) + for raw_code, count in result.all(): + code = raw_code if raw_code in _SAFE_UPSTREAM_ERROR_CODES else "other" + counts[code] += int(count) + return [code for code, _ in sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:5]] + + async def _feature_counts(self, conditions: list[Predicate]) -> _FeatureCounts: + scalar_queries = ( + select(func.count()).select_from(ApiFirewallAllowlist), + select(func.count()).select_from(ApiKey), + select(func.count()).select_from(ModelSource).where(ModelSource.is_enabled.is_(True)), + select(func.count()).select_from(ProxyEndpoint).where(ProxyEndpoint.is_active.is_(True)), + select(func.count()).select_from(AutomationJob).where(AutomationJob.enabled.is_(True)), + select(func.count()).select_from(RequestLog).where(and_(*conditions, RequestLog.model.like("gpt-image-%"))), + select(QuotaPlannerSettings.mode).where(QuotaPlannerSettings.id == 1), + ) + values = [] + for query in scalar_queries: + values.append((await self._session.execute(query)).scalar_one_or_none()) + return _FeatureCounts( + firewall_entries=int(values[0] or 0), + api_keys=int(values[1] or 0), + model_sources=int(values[2] or 0), + active_proxy_endpoints=int(values[3] or 0), + enabled_automations=int(values[4] or 0), + image_requests=int(values[5] or 0), + quota_planner_mode=str(values[6] or "shadow"), + ) + + async def _database_size_bytes(self) -> int | None: + if self._session.get_bind().dialect.name == "postgresql": + result = await self._session.execute(text("SELECT pg_database_size(current_database())")) + return int(result.scalar_one()) + path = sqlite_db_path_from_url(self._settings.database_url) + if path is None: + return None + try: + return Path(path).stat().st_size + except OSError as exc: + logger.debug("Unable to measure SQLite database size path=%s", path, exc_info=exc) + return None + + +@dataclass(frozen=True, slots=True) +class _FeatureCounts: + firewall_entries: int + api_keys: int + model_sources: int + active_proxy_endpoints: int + enabled_automations: int + image_requests: int + quota_planner_mode: str + + +def _canonical_plan(raw_plan: str | None) -> str: + normalized = (raw_plan or "").strip().lower() + if normalized in {"pro", "prolite"}: + return "pro" + if normalized in {"team", "business", "enterprise", "edu", "education"}: + return "team" + if normalized == "plus": + return "plus" + return "free" + + +def _canonical_reasoning(raw_effort: str | None) -> str: + normalized = (raw_effort or "").strip().lower() + if not normalized: + return "unspecified" + return normalized if normalized in _REASONING_EFFORTS else "other" + + +def _canonical_routing_policy(raw_policy: str | None) -> str: + normalized = (raw_policy or "").strip().lower() + return normalized if normalized in _ROUTING_POLICIES else "other" + + +def deployment_method() -> DeploymentMethod: + if os.environ.get("KUBERNETES_SERVICE_HOST") or Path("/var/run/secrets/kubernetes.io/serviceaccount").exists(): + return "k8s" + if Path("/.dockerenv").exists() or Path("/run/.containerenv").exists(): + return "docker" + try: + importlib.metadata.distribution("codex-lb") + except importlib.metadata.PackageNotFoundError: + return "bare" + return "pip" diff --git a/app/modules/usage/live_ingest.py b/app/modules/usage/live_ingest.py index d26c2e7e2c..af4ecba115 100644 --- a/app/modules/usage/live_ingest.py +++ b/app/modules/usage/live_ingest.py @@ -3,24 +3,20 @@ import asyncio import logging import time +import weakref from dataclasses import dataclass -from sqlalchemy import select - from app.core import usage as usage_core from app.core.config.settings import get_settings from app.core.usage.live_hub import register_live_usage_publisher from app.core.usage.live_snapshots import LiveRateLimitSnapshot, LiveUsageWindow -from app.db.models import Account from app.db.session import get_background_session from app.modules.proxy.account_cache import get_account_selection_cache from app.modules.proxy.rate_limit_cache import get_rate_limit_headers_cache -from app.modules.usage.repository import UsageRepository +from app.modules.usage.repository import UsageRepository, UsageWindowWrite logger = logging.getLogger(__name__) -_RESOLUTION_TTL_SECONDS = 300.0 - # Write-coalescing tuning (fixed; issue #1340 / PRINCIPLES.md P2). The # ingestor keeps both as constructor fields so tests can exercise queue # overflow and coalescing with small values. @@ -28,6 +24,50 @@ _WRITE_MIN_INTERVAL_SECONDS = 5.0 _CACHE_INVALIDATION_MIN_INTERVAL_SECONDS = 5.0 +# Ownership accounting for every task any ingestor instance creates (consumer +# and trailing cache invalidation), so a task an owner lost track of (a stop +# cancelled mid-await) can never end in a silently dropped exception: +# +# - `_owned_tasks` holds weak references (they never extend task lifetime) so +# the test suite's leak fence can cancel pending tasks and settle completed +# ones that some reference chain kept alive across a test boundary. +# - `_record_owned_task_result` runs as each task's done callback: it +# retrieves the exception (so the loop's unobserved-task warning can never +# fire at garbage-collection time), logs it, and records it in the bounded +# `_owned_task_failures` strong handoff for the fence to drain (#1755). +# +# `_settled_owned_tasks` (also weak) marks tasks whose result was already +# recorded, so the done callback and the fence's sweep of completed tasks +# settle each task exactly once even when both observe it. +_owned_tasks: weakref.WeakSet[asyncio.Task[None]] = weakref.WeakSet() +_settled_owned_tasks: weakref.WeakSet[asyncio.Task[None]] = weakref.WeakSet() +# (task name, exception repr) pairs. Reprs, not exception objects: a stored +# exception's traceback would keep the failed ingestor's whole object graph +# (task, queue, cached state) alive for the process lifetime, since production +# never drains this record. +_owned_task_failures: list[tuple[str, str]] = [] +_MAX_OWNED_TASK_FAILURES = 16 + + +def _record_owned_task_result(task: asyncio.Task[None]) -> None: + if task in _settled_owned_tasks: + return + _settled_owned_tasks.add(task) + _owned_tasks.discard(task) + if task.cancelled(): + return + exc = task.exception() + if exc is None: + return + logger.error("Live usage ingestor task %r died unexpectedly", task.get_name(), exc_info=exc) + if len(_owned_task_failures) < _MAX_OWNED_TASK_FAILURES: + _owned_task_failures.append((task.get_name(), repr(exc))) + + +def _enroll_owned_task(task: asyncio.Task[None]) -> None: + _owned_tasks.add(task) + task.add_done_callback(_record_owned_task_result) + @dataclass(frozen=True, slots=True) class _QueuedSnapshot: @@ -68,7 +108,6 @@ def __init__( self._queue: asyncio.Queue[_QueuedSnapshot] = asyncio.Queue(maxsize=max(1, queue_size)) self._write_min_interval_seconds = write_min_interval_seconds self._last_write: dict[str, tuple[tuple[object, ...], float]] = {} - self._resolution_cache: dict[str, tuple[str | None, float]] = {} self._consumer: asyncio.Task[None] | None = None self._dropped = 0 self._last_cache_invalidation = 0.0 @@ -102,6 +141,11 @@ def publish( def start(self) -> None: if self._consumer is None or self._consumer.done(): self._consumer = asyncio.create_task(self._run(), name="live-usage-ingestor") + _enroll_owned_task(self._consumer) + + def is_running(self) -> bool: + consumer = self._consumer + return consumer is not None and not consumer.done() async def stop(self) -> None: consumer = self._consumer @@ -141,14 +185,6 @@ async def _run(self) -> None: ) async def _ingest(self, item: _QueuedSnapshot) -> None: - account_id = item.account_id - if account_id is None: - account_id = await self._resolve_account_id(item.chatgpt_account_id) - if account_id is None: - return - if self._should_skip(account_id, item.snapshot): - return - snapshot = item.snapshot primary = snapshot.primary secondary = snapshot.secondary @@ -162,51 +198,56 @@ async def _ingest(self, item: _QueuedSnapshot) -> None: and primary.window_minutes == usage_core.DEFAULT_WINDOW_MINUTES_MONTHLY ): monthly, primary = primary, None - async with get_background_session() as session: - repo = UsageRepository(session) - if primary is not None: - await repo.add_entry( - account_id=account_id, - used_percent=float(primary.used_percent), - input_tokens=None, - output_tokens=None, + windows: list[UsageWindowWrite] = [] + if primary is not None: + windows.append( + UsageWindowWrite( window="primary", + used_percent=float(primary.used_percent), reset_at=primary.reset_at, window_minutes=primary.window_minutes, credits_has=snapshot.credits_has, credits_unlimited=snapshot.credits_unlimited, credits_balance=snapshot.credits_balance, ) - if secondary is not None: - # Mirror the poller: credits normally ride the primary row. - # A secondary-only snapshot (e.g. the short window is not - # being reported) must still carry the fresh credit state. - secondary_carries_credits = primary is None - await repo.add_entry( - account_id=account_id, - used_percent=float(secondary.used_percent), - input_tokens=None, - output_tokens=None, + ) + if secondary is not None: + # Mirror the poller: credits normally ride the primary row. A + # secondary-only snapshot must still carry fresh credit state. + secondary_carries_credits = primary is None + windows.append( + UsageWindowWrite( window="secondary", + used_percent=float(secondary.used_percent), reset_at=secondary.reset_at, window_minutes=secondary.window_minutes, credits_has=snapshot.credits_has if secondary_carries_credits else None, credits_unlimited=snapshot.credits_unlimited if secondary_carries_credits else None, credits_balance=snapshot.credits_balance if secondary_carries_credits else None, ) - if monthly is not None: - await repo.add_entry( - account_id=account_id, - used_percent=float(monthly.used_percent), - input_tokens=None, - output_tokens=None, + ) + if monthly is not None: + windows.append( + UsageWindowWrite( window="monthly", + used_percent=float(monthly.used_percent), reset_at=monthly.reset_at, window_minutes=monthly.window_minutes, credits_has=snapshot.credits_has, credits_unlimited=snapshot.credits_unlimited, credits_balance=snapshot.credits_balance, ) + ) + + async with get_background_session() as session: + account_id = await UsageRepository(session).settle_live_account_snapshot( + account_id=item.account_id, + chatgpt_account_id=item.chatgpt_account_id, + windows=windows, + should_skip=lambda resolved: self._should_skip(resolved, snapshot), + ) + if account_id is None: + return self._last_write[account_id] = (_fingerprint(snapshot), time.monotonic()) await self._invalidate_caches_throttled() @@ -222,7 +263,11 @@ async def _invalidate_caches_throttled(self) -> None: await self._invalidate_caches_now() return if self._trailing_invalidation is None or self._trailing_invalidation.done(): - self._trailing_invalidation = asyncio.create_task(self._trailing_invalidate(remaining)) + self._trailing_invalidation = asyncio.create_task( + self._trailing_invalidate(remaining), + name="live-usage-trailing-invalidation", + ) + _enroll_owned_task(self._trailing_invalidation) async def _trailing_invalidate(self, delay_seconds: float) -> None: await asyncio.sleep(delay_seconds) @@ -236,30 +281,30 @@ async def _invalidate_caches_now(self) -> None: # values before the TTL expires. await get_rate_limit_headers_cache().invalidate() - async def _resolve_account_id(self, chatgpt_account_id: str | None) -> str | None: - if not chatgpt_account_id: - return None - cached = self._resolution_cache.get(chatgpt_account_id) - now = time.monotonic() - if cached is not None and now - cached[1] < _RESOLUTION_TTL_SECONDS: - return cached[0] - async with get_background_session() as session: - rows = ( - (await session.execute(select(Account.id).where(Account.chatgpt_account_id == chatgpt_account_id))) - .scalars() - .all() - ) - # Ambiguous identities (multiple workspace slots) are dropped rather - # than guessed; the poller stays authoritative for them. - resolved = rows[0] if len(rows) == 1 else None - self._resolution_cache[chatgpt_account_id] = (resolved, now) - return resolved - _ingestor: LiveUsageIngestor | None = None +# Registrations a nested startup displaced, innermost-last. A stack rather +# than a single prior slot: lifespans can nest more than one level deep (each +# portal-loop ``TestClient`` adds one), and a stack restores each displaced +# outer lifespan in LIFO order while an out-of-order stop simply removes its +# instance from wherever it sits — a single slot would forget everything below +# the most recent displacement. +_displaced_ingestors: list[LiveUsageIngestor] = [] def start_live_usage_ingestor() -> LiveUsageIngestor | None: + """Create, start, and register a fresh ingestor as the current singleton. + + The caller (the app lifespan) MUST hold the returned instance and pass it + back to ``stop_live_usage_ingestor`` at shutdown. Two lifespans can be + live in one process (the test suite nests a portal-loop ``TestClient`` + inside an app already running on the session loop); each owns its own + instance, and the module global only tracks whichever registered last. A + started ingestor whose only strong root is the module global would become + an unreferenced reference cycle (task -> coroutine frame -> ingestor -> + queue -> getter future -> task) the moment a nested startup overwrites the + global, and the cyclic GC would then destroy its consumer task mid-await. + """ global _ingestor settings = get_settings() if not getattr(settings, "live_usage_ingestion_enabled", True): @@ -270,15 +315,47 @@ def start_live_usage_ingestor() -> LiveUsageIngestor | None: write_min_interval_seconds=_WRITE_MIN_INTERVAL_SECONDS, ) ingestor.start() + if _ingestor is not None and _ingestor.is_running(): + # A nested startup displaces a still-running outer registration; + # remember it so the nested shutdown can restore it (a dead instance + # is never worth remembering). + _displaced_ingestors.append(_ingestor) register_live_usage_publisher(ingestor.publish) _ingestor = ingestor return ingestor -async def stop_live_usage_ingestor() -> None: +async def stop_live_usage_ingestor(ingestor: LiveUsageIngestor | None = None) -> None: + """Stop ``ingestor``, or the current singleton when omitted. + + The module global and the publisher registration are touched only when + the stopped instance still owns them, so a lifespan shutting down cannot + orphan or unregister a nested lifespan's newer instance — and a nested + lifespan's shutdown cannot leave the outer instance dangling with no + stop path (the leak behind issue #1755's cross-test poisoning). When the + stopped instance is the current registration, the most recent displaced + ingestor that is still running is restored (registration and publisher + wiring), so a still-live outer lifespan resumes receiving publications + instead of going silently deaf after a nested shutdown. + """ global _ingestor - ingestor = _ingestor - _ingestor = None - register_live_usage_publisher(None) + if ingestor is None: + ingestor = _ingestor + if ingestor is not None: + # Whatever happens next, a stopped instance must never be restorable. + try: + _displaced_ingestors.remove(ingestor) + except ValueError: + pass + if ingestor is None or _ingestor is ingestor: + restored: LiveUsageIngestor | None = None + while _displaced_ingestors: + candidate = _displaced_ingestors.pop() + if candidate.is_running(): + restored = candidate + break + # Stopped or dead in the meantime — never restore a dead instance. + _ingestor = restored + register_live_usage_publisher(restored.publish if restored is not None else None) if ingestor is not None: await ingestor.stop() diff --git a/app/modules/usage/repository.py b/app/modules/usage/repository.py index 1064e56a22..65727688ca 100644 --- a/app/modules/usage/repository.py +++ b/app/modules/usage/repository.py @@ -7,16 +7,32 @@ from datetime import datetime from hashlib import sha256 from threading import RLock -from typing import Any, cast +from typing import Any, Callable, cast from anyio import to_thread -from sqlalchemy import Integer, and_, delete, func, literal_column, or_, select, true, tuple_ +from sqlalchemy import ( + Integer, + String, + and_, + column, + delete, + func, + literal_column, + or_, + select, + text, + true, + tuple_, + union_all, + values, +) from sqlalchemy import cast as sqlalchemy_cast from sqlalchemy.ext.asyncio import AsyncSession from app.core.config.settings import get_settings from app.core.usage.types import UsageAggregateRow, UsageTrendBucket from app.core.utils.time import utcnow +from app.db.account_identity_lock import lock_postgresql_account_identities from app.db.models import Account, AdditionalUsageHistory, UsageHistory from app.db.session import relax_commit_durability, sqlite_writer_section from app.db.sqlite_utils import sqlite_db_path_from_url @@ -50,6 +66,35 @@ class UsageWindowWrite: credits_balance: float | None = None +class LiveSnapshotOwnerIdentityRelockError(RuntimeError): + """The selected live-snapshot owner's identity changed twice.""" + + +def _account_snapshot_entries( + account_id: str, + windows: Collection[UsageWindowWrite], + *, + recorded_at: datetime | None = None, +) -> list[UsageHistory]: + captured_at = recorded_at or utcnow() + return [ + UsageHistory( + account_id=account_id, + used_percent=window.used_percent, + input_tokens=None, + output_tokens=None, + window=window.window, + reset_at=window.reset_at, + window_minutes=window.window_minutes, + credits_has=window.credits_has, + credits_unlimited=window.credits_unlimited, + credits_balance=window.credits_balance, + recorded_at=captured_at, + ) + for window in windows + ] + + @dataclass(frozen=True, slots=True) class _BulkHistoryCacheMetadata: row_count: int @@ -635,23 +680,7 @@ async def add_account_snapshot( """Persist one account's standard usage windows atomically.""" if not windows: return [] - captured_at = recorded_at or utcnow() - entries = [ - UsageHistory( - account_id=account_id, - used_percent=window.used_percent, - input_tokens=None, - output_tokens=None, - window=window.window, - reset_at=window.reset_at, - window_minutes=window.window_minutes, - credits_has=window.credits_has, - credits_unlimited=window.credits_unlimited, - credits_balance=window.credits_balance, - recorded_at=captured_at, - ) - for window in windows - ] + entries = _account_snapshot_entries(account_id, windows, recorded_at=recorded_at) try: async with sqlite_writer_section(): # Telemetry write: this transaction only appends usage-history @@ -664,6 +693,133 @@ async def add_account_snapshot( raise return entries + async def _resolve_postgresql_live_snapshot_owner( + self, + account_id: str | None, + chatgpt_account_id: str | None, + ) -> str | None: + locked_identities = (chatgpt_account_id,) + fallback_identity = chatgpt_account_id + relocked = False + + while True: + await lock_postgresql_account_identities(self._session, locked_identities) + locked_identity_values = frozenset(identity for identity in locked_identities if identity) + identity_to_relock: str | None = None + + if account_id is not None: + # Read before taking the row lock so MVCC preserves the + # current recovery identity even when its writer has already + # deleted the local row but not committed yet. + observed = ( + await self._session.execute( + select(Account.id, Account.chatgpt_account_id).where(Account.id == account_id) + ) + ).one_or_none() + if observed is not None: + observed_identity = observed.chatgpt_account_id + if observed_identity and observed_identity not in locked_identity_values: + identity_to_relock = observed_identity + else: + locked = ( + await self._session.execute( + select(Account.id, Account.chatgpt_account_id) + .where(Account.id == account_id) + .with_for_update(key_share=True) + ) + ).one_or_none() + if locked is not None: + if locked.chatgpt_account_id and locked.chatgpt_account_id not in locked_identity_values: + identity_to_relock = locked.chatgpt_account_id + else: + return locked.id + + if identity_to_relock is not None: + if relocked: + raise LiveSnapshotOwnerIdentityRelockError( + "Live snapshot owner identity changed during PostgreSQL relock" + ) + # Release the first lock before adding another identity; the + # shared helper can then reacquire the full set in canonical + # order without inverting an account writer's lock order. + await self._session.rollback() + fallback_identity = identity_to_relock + locked_identities = (chatgpt_account_id, identity_to_relock) + relocked = True + continue + + if fallback_identity: + upstream_stmt = ( + select(Account.id) + .where(Account.chatgpt_account_id == fallback_identity) + .with_for_update(key_share=True) + ) + matches = list((await self._session.execute(upstream_stmt)).scalars().all()) + if len(matches) == 1: + return matches[0] + return None + + async def settle_live_account_snapshot( + self, + *, + account_id: str | None, + chatgpt_account_id: str | None, + windows: Collection[UsageWindowWrite], + should_skip: Callable[[str], bool], + ) -> str | None: + """Resolve a live snapshot owner and atomically persist its windows.""" + if not windows: + return None + + try: + async with sqlite_writer_section(): + bind = self._session.get_bind() + dialect_name = bind.dialect.name if bind is not None else "sqlite" + if dialect_name == "sqlite": + # Acquire SQLite's database-wide writer slot before owner + # lookup. Consolidation then commits before this lookup or + # waits until the snapshot commit, so the chosen FK owner + # cannot disappear between SELECT and INSERT. + await self._session.execute(text("BEGIN IMMEDIATE")) + resolved_account_id = None + if account_id is not None: + resolved_account_id = await self._session.scalar( + select(Account.id).where(Account.id == account_id) + ) + if resolved_account_id is None and chatgpt_account_id: + matches = list( + ( + await self._session.execute( + select(Account.id).where(Account.chatgpt_account_id == chatgpt_account_id) + ) + ) + .scalars() + .all() + ) + if len(matches) == 1: + resolved_account_id = matches[0] + else: + resolved_account_id = await self._resolve_postgresql_live_snapshot_owner( + account_id, + chatgpt_account_id, + ) + + if resolved_account_id is None or should_skip(resolved_account_id): + await self._session.rollback() + return None + + entries = _account_snapshot_entries(resolved_account_id, windows) + # Telemetry write: this transaction only locks the owner and + # appends usage-history rows, so it may skip synchronous WAL + # flush just like add_account_snapshot(). + await relax_commit_durability(self._session) + self._session.add_all(entries) + await self._session.commit() + except BaseException: + await self._session.rollback() + raise + return resolved_account_id + async def aggregate_since( self, since: datetime, @@ -779,7 +935,7 @@ async def history_since( _window_clause(window), UsageHistory.recorded_at >= since, ) - .order_by(UsageHistory.recorded_at.asc()) + .order_by(UsageHistory.recorded_at.asc(), UsageHistory.id.asc()) ) result = await self._session.execute(stmt) return list(result.scalars().all()) @@ -791,6 +947,8 @@ async def bulk_history_since( since: datetime, *, cutoffs: dict[str, datetime] | None = None, + per_account_row_cap: int | None = None, + uncapped_recent_floor: datetime | None = None, ) -> dict[str, list[UsageHistorySnapshot]]: """Fetch minimal usage history fields for multiple accounts in a single query. @@ -801,6 +959,24 @@ async def bulk_history_since( ignores ``cutoffs`` (its snapshot cache is keyed on the shared floor); callers keep their own per-account trimming, so honoring the bound here only changes how many rows are read, never the result. + + ``per_account_row_cap`` additionally bounds each account's slice to + its newest rows inside the cutoff (PostgreSQL only). Live snapshot + ingestion appends usage rows per proxied request, so a busy account's + 7-day window can hold tens of thousands of rows while the projection + consumers (EWMA depletion, weekly-pace burn/smoothing) only read the + recent tail. Each capped slice keeps oldest-first ordering. The + SQLite snapshot-cache path ignores the cap the same way it ignores + ``cutoffs``. + + ``uncapped_recent_floor`` exempts rows at or after the given time + from the row cap: every in-cutoff row newer than the floor is always + returned, and the cap bounds only the older remainder. Consumers + whose math weighs every sample in a fixed time window equally (the + weekly-pace smoothing mean) pass their window start here so a + write-rate burst can never silently truncate that window, while + tail-weighted consumers (EWMA) stay covered by the cap alone. + Ignored unless ``per_account_row_cap`` is set on PostgreSQL. """ if not account_ids: return {} @@ -816,6 +992,16 @@ async def bulk_history_since( since, ) + if per_account_row_cap is not None and dialect == "postgresql": + return await self._bulk_history_since_capped_postgresql( + account_ids, + window, + since, + cutoffs=cutoffs, + per_account_row_cap=per_account_row_cap, + uncapped_recent_floor=uncapped_recent_floor, + ) + if cutoffs: recency_clause = or_( *( @@ -860,6 +1046,103 @@ async def bulk_history_since( grouped.setdefault(snapshot.account_id, []).append(snapshot) return grouped + async def _bulk_history_since_capped_postgresql( + self, + account_ids: list[str], + window: str, + since: datetime, + *, + cutoffs: dict[str, datetime] | None, + per_account_row_cap: int, + uncapped_recent_floor: datetime | None, + ) -> dict[str, list[UsageHistorySnapshot]]: + """Per-account newest-first capped fetch (PostgreSQL). + + One lateral top-N probe per account instead of one shared range scan: + the probe descends idx_usage_window_account_time_covering (or its + raw-window twin) backward and stops at the cap or the account's + cutoff, whichever comes first, so the read never touches the bulk of + a dense account's window. The OR-of-cutoffs shape this replaces + returned every in-window row (hundreds of thousands on dense + deployments) to Python only for the projection consumers to use the + recent tail. + + With ``uncapped_recent_floor`` the probe splits into two disjoint + branches over the same covering index: rows at or after the floor are + returned in full (time-bounded, so still cheap), and the top-N cap + applies only to rows between the cutoff and the floor. Snapshot + ingestion writes per proxied request whenever the usage fingerprint + moves, so a fixed row cap alone cannot guarantee it out-lasts a + burst inside an equal-weight consumer window. + """ + value_columns = [ + column("account_id", String()), + column("cutoff", UsageHistory.recorded_at.type), + ] + if uncapped_recent_floor is not None: + value_columns.append(column("uncapped_floor", UsageHistory.recorded_at.type)) + value_rows: list[tuple] = [] + for account_id in account_ids: + cutoff = max(cutoffs.get(account_id, since), since) if cutoffs else since + if uncapped_recent_floor is None: + value_rows.append((account_id, cutoff)) + else: + value_rows.append((account_id, cutoff, max(cutoff, uncapped_recent_floor))) + account_cutoffs = values(*value_columns, name="account_cutoffs").data(value_rows) + snapshot_columns = ( + UsageHistory.id, + UsageHistory.account_id, + UsageHistory.used_percent, + UsageHistory.recorded_at, + UsageHistory.reset_at, + UsageHistory.window_minutes, + ) + capped_tail = ( + select(*snapshot_columns) + .where( + UsageHistory.account_id == account_cutoffs.c.account_id, + UsageHistory.recorded_at >= account_cutoffs.c.cutoff, + *( + (UsageHistory.recorded_at < account_cutoffs.c.uncapped_floor,) + if uncapped_recent_floor is not None + else () + ), + _window_clause(window), + ) + .order_by(UsageHistory.recorded_at.desc(), UsageHistory.id.desc()) + .limit(per_account_row_cap) + .correlate(account_cutoffs) + ) + if uncapped_recent_floor is not None: + uncapped_recent = ( + select(*snapshot_columns) + .where( + UsageHistory.account_id == account_cutoffs.c.account_id, + UsageHistory.recorded_at >= account_cutoffs.c.uncapped_floor, + _window_clause(window), + ) + .correlate(account_cutoffs) + ) + recent = union_all(uncapped_recent, capped_tail).lateral("recent") + else: + recent = capped_tail.lateral("recent") + stmt = select(recent).select_from(account_cutoffs.join(recent, true())) + result = await self._session.execute(stmt) + grouped: dict[str, list[UsageHistorySnapshot]] = {} + for row in result.all(): + snapshot = UsageHistorySnapshot( + id=int(row.id), + account_id=row.account_id, + used_percent=float(row.used_percent), + recorded_at=row.recorded_at, + reset_at=float(row.reset_at) if row.reset_at is not None else None, + window_minutes=int(row.window_minutes) if row.window_minutes is not None else None, + ) + grouped.setdefault(snapshot.account_id, []).append(snapshot) + for snapshots in grouped.values(): + snapshots.sort(key=lambda snapshot: (snapshot.recorded_at, snapshot.id)) + return grouped + async def trends_by_bucket( self, since: datetime, diff --git a/app/modules/usage/updater.py b/app/modules/usage/updater.py index 2327ecba14..66af2e2c98 100644 --- a/app/modules/usage/updater.py +++ b/app/modules/usage/updater.py @@ -26,6 +26,7 @@ from app.core.upstream_proxy import ResolvedUpstreamRoute, UpstreamProxyRouteError, resolve_upstream_route from app.core.usage.models import AdditionalRateLimitPayload, UsagePayload, UsageWindow from app.core.utils.request_id import get_request_id +from app.core.utils.shared_future import wait_on_shared_future from app.core.utils.time import utcnow from app.db.models import Account, AccountStatus, UsageHistory from app.db.session import get_background_session @@ -199,14 +200,14 @@ async def run( if wait_for_existing is None: break try: - await asyncio.shield(wait_for_existing) + await wait_on_shared_future(wait_for_existing) except asyncio.CancelledError: current_task = asyncio.current_task() if current_task is not None and current_task.cancelling(): raise except Exception: pass - return await asyncio.shield(task) + return await wait_on_shared_future(task) async def _run_factory( self, diff --git a/deploy/helm/codex-lb/README.md b/deploy/helm/codex-lb/README.md index b39326bec0..ed6e72295b 100644 --- a/deploy/helm/codex-lb/README.md +++ b/deploy/helm/codex-lb/README.md @@ -425,6 +425,13 @@ externalSecrets: enabled: true # Use External Secrets Operator ``` +The Grafana sidecar imports the dashboard JSON but does not provision +datasources or database credentials. In the **codex-lb TTFT Breakdown** +dashboard, select the Grafana PostgreSQL datasource that points to the +codex-lb database from the visible **PostgreSQL** (`DS_SQL`) dropdown. All +four SQL panels follow that one runtime selection. The owning contract is in +[proxy runtime observability](../../../openspec/specs/proxy-runtime-observability/). + Install with: ```bash diff --git a/deploy/helm/codex-lb/dashboards/ttft-breakdown.json b/deploy/helm/codex-lb/dashboards/ttft-breakdown.json index 7ab2751bfd..57c830dea2 100644 --- a/deploy/helm/codex-lb/dashboards/ttft-breakdown.json +++ b/deploy/helm/codex-lb/dashboards/ttft-breakdown.json @@ -5,7 +5,10 @@ "editable": true, "panels": [ { - "datasource": "${DS_SQL}", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}" + }, "fieldConfig": { "defaults": { "unit": "ms" @@ -29,7 +32,10 @@ "type": "table" }, { - "datasource": "${DS_SQL}", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}" + }, "fieldConfig": { "defaults": {}, "overrides": [] @@ -51,7 +57,10 @@ "type": "table" }, { - "datasource": "${DS_SQL}", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}" + }, "fieldConfig": { "defaults": { "unit": "ms" @@ -75,7 +84,10 @@ "type": "table" }, { - "datasource": "${DS_SQL}", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}" + }, "fieldConfig": { "defaults": { "unit": "ms" @@ -106,7 +118,20 @@ "ttft" ], "templating": { - "list": [] + "list": [ + { + "hide": 0, + "includeAll": false, + "label": "PostgreSQL", + "multi": false, + "name": "DS_SQL", + "options": [], + "query": "grafana-postgresql-datasource", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] }, "time": { "from": "now-24h", @@ -116,4 +141,4 @@ "title": "codex-lb TTFT Breakdown", "uid": "codex-lb-ttft-breakdown", "version": 1 -} \ No newline at end of file +} diff --git a/docker-compose.yml b/docker-compose.yml index 3c1d518035..b9c730521c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,6 +92,13 @@ services: fi exec docker-entrypoint.sh "$@" - codex-lb-postgres-entrypoint-guard + # Docker's default /dev/shm is 64MB. PostgreSQL parallel workers exchange + # tuples through dynamic shared memory under /dev/shm, so hash joins that + # spill past 64MB abort with "could not resize shared memory segment ... + # No space left on device". 1GB gives parallel query realistic headroom. + # (The Helm chart needs no equivalent: the bundled Bitnami PostgreSQL + # sub-chart mounts a memory-backed /dev/shm by default via shmVolume.) + shm_size: 1gb environment: POSTGRES_USER: codex_lb POSTGRES_PASSWORD: codex_lb diff --git a/docs/api-keys.md b/docs/api-keys.md index 3cd9c13c3a..08ca388a15 100644 --- a/docs/api-keys.md +++ b/docs/api-keys.md @@ -27,6 +27,19 @@ Keys can also be scoped to specific accounts, so a key draws quota only from the ![API keys with assigned accounts](screenshots/apis-assigned-accounts.jpg) +## Reasoning effort policies + +A key can either enforce one reasoning effort or allow a selected non-empty set of client-requested efforts. +Leave the allowed-efforts selection empty to keep the existing unrestricted behavior. A request that explicitly +sets an effort outside its key's allowlist receives a `403 reasoning_effort_not_allowed` response. Requests that +omit a reasoning effort continue to use the model or upstream default. + +The policy evaluates the effort selected by the client, including supported model aliases such as `-xhigh`. +Each configured effort is distinct: allowing `high` does not allow `xhigh`, and allowing `max` does not allow +`ultra`. The proxy still rewrites an allowed `ultra` request to the upstream wire value `max`. + +![API key reasoning-effort policy](screenshots/apis-reasoning-efforts.jpg) + For wiring keys into each client, see [Client Setup](client-setup.md). --- diff --git a/docs/client-setup.md b/docs/client-setup.md index 21f37e6ef4..2ee547810e 100644 --- a/docs/client-setup.md +++ b/docs/client-setup.md @@ -4,7 +4,7 @@ Point any OpenAI-compatible client at codex-lb. If [API key auth](api-keys.md) i Model availability is discovered from the upstream Codex model catalog and can vary by account plan, workspace, rollout, and upstream deprecation state. Prefer the live `GET /v1/models` or `GET /backend-api/codex/models` response over a copied static table when configuring clients or API-key model allowlists. -The examples below use the current frontier lineup: **`gpt-5.6-sol`** (strongest), **`gpt-5.6-terra`** (balanced), and **`gpt-5.6-luna`** (fast) — all 372k context. `gpt-5.5` and `gpt-5.4` are still served for older pinned clients; retired slugs such as `gpt-5.3-codex`, `gpt-5.3-codex-spark`, and `gpt-5.1-codex-mini` were dropped from the upstream bundled catalog and should no longer be used in new configs. +The examples below use the current frontier lineup: **`gpt-5.6-sol`** (strongest), **`gpt-5.6-terra`** (balanced), and **`gpt-5.6-luna`** (fast) — all with a 272k default input budget and an 872k upstream maximum ([opt-in, Codex CLI only](#opting-into-the-872k-context-window)). `gpt-5.5` and `gpt-5.4` are still served for older pinned clients; retired slugs such as `gpt-5.3-codex`, `gpt-5.3-codex-spark`, and `gpt-5.1-codex-mini` were dropped from the upstream bundled catalog and should no longer be used in new configs. | Client | Endpoint | Config | |--------|----------|--------| @@ -31,7 +31,115 @@ supports_websockets = true requires_openai_auth = true # required for codex app ``` -This documented `requires_openai_auth = true` setup uses Codex-backed authentication and does not need an `x-openai-actor-authorization` marker to be eligible for Codex's built-in `$imagegen` tool. Provider configurations that intentionally skip OpenAI login have a different eligibility path; see the [Images compatibility context](https://github.com/Soju06/codex-lb/blob/main/openspec/specs/images-api-compat/context.md#codex-provider-eligibility). +### Opting into the 872k context window + +GPT-5.6 ships a 272,000-token default input budget with an 872,000-token +maximum. codex-lb advertises both — `context_window` and `max_context_window` +on `GET /backend-api/codex/models` — and the Codex CLI stays on the default +until you raise it in `~/.codex/config.toml` (top level, before any +`[section]` header): + +```toml +model_context_window = 872000 +``` + +- Values above `max_context_window` are clamped to it: `model_context_window = + 1000000` resolves to 872,000 and does not unlock a 1M window. +- Leave `model_auto_compact_token_limit` unset. Codex auto-compacts at 90% of + the resolved window — 784,800 tokens here — and clamps any larger configured + value down to that, so setting `900000` is a no-op. Set it only to compact + *earlier*. +- Cost: input beyond the 272,000-token threshold is metered at the upstream + long-context rate. That threshold is why 272,000 stays the default. + +These keys are Codex-CLI-only. The OpenCode / OpenClaw / SDK examples below +stay at 272000 because `/v1/models` reports the default input budget, not the +ceiling. + +### Daybreak Blue profile (Trusted Access) + +Use a separate provider for authorized defensive cybersecurity work. The +ordinary `codex-lb` provider above must remain free of the capability header; +adding it there would classify every request as requiring the restricted pool. + +First add this opt-in provider to the same machine-local +`~/.codex/config.toml`: + +```toml +[model_providers.codex-lb-daybreak-blue] +name = "openai" +base_url = "http://127.0.0.1:2455/backend-api/codex" +wire_api = "responses" +env_key = "CODEX_LB_API_KEY" +supports_websockets = true +requires_openai_auth = true +http_headers = { "X-Codex-LB-Required-Capability" = "trusted_cyber" } +``` + +Then create `~/.codex/daybreak-blue.config.toml`: + +```toml +model = "gpt-5.6-sol" +model_provider = "codex-lb-daybreak-blue" +``` + +Activate it explicitly for the task or orchestration root that needs the +restricted route: + +```bash +export CODEX_LB_API_KEY="sk-clb-..." # key from the dashboard +codex --profile daybreak-blue +codex exec --profile daybreak-blue "" +``` + +Current Codex versions load named profiles from sibling +`.config.toml` files; legacy `[profiles.]` tables are no longer +selected. Provider and profile keys are machine-local, so a project +`.codex/config.toml` cannot activate this route. See the official +[Codex profile documentation](https://learn.chatgpt.com/docs/config-file/config-advanced#profiles). + +The static header is an authenticated routing requirement, not a grant. Use +this profile only when the selected identity and ChatGPT workspace or API +organization/project are already approved for the intended Codex product +surface. The dedicated provider always supplies a Codex LB API key because +unauthenticated capability carriers are rejected even on a local deployment. +When the capability header is present, Codex LB validates that key for the +request even if global API-key auth is disabled; ordinary requests without the +header keep the deployment's normal auth behavior. Current Codex clients may +fall back from WebSocket to HTTP even when `supports_websockets = true`, and +static provider headers also accompany control and Images requests. Codex LB +authenticates capability-bearing HTTP and non-Responses WebSocket requests and +then rejects them with `required_capability_transport_unsupported` before +account selection or upstream dispatch. This includes Responses/compact HTTP +fallback, Codex control, admission, warmup, files, transcription, Chat +Completions, Images, reset-credit consume, and Live WebSockets; Chat Completions +is guarded defensively if a provider client reaches that equivalent routing +sink. Authenticated `/models` initialization and local API-key usage or +reset-credit listings remain available because they do not route an upstream +account. Restore direct Responses WebSocket availability +instead of removing the carrier or retrying through ordinary HTTP. Codex LB +narrows a direct WebSocket turn's first and later account selections to eligible accounts already marked +`security_work_authorized`; if none are available, it fails closed without +ordinary fallback. Selecting `gpt-5.6-sol` by itself does not activate this +path, and a Daybreak alias may resolve to that same underlying model. See +OpenAI's +[Trusted Access guidance](https://developers.openai.com/api/docs/guides/safety-checks/cybersecurity#authorized-access-and-agentic-workflows). + +Complete inert examples are available as +[`config.toml`](examples/codex/config.toml) and +[`daybreak-blue.config.toml`](examples/codex/daybreak-blue.config.toml). To +roll back, stop using `--profile daybreak-blue`, remove the profile file, and +optionally remove only the `codex-lb-daybreak-blue` provider block. No server or +database change is required. + +This documented `requires_openai_auth = true` setup makes the provider eligible +for Codex's built-in `$imagegen` tool, but the Daybreak carrier is intentionally +rejected on the Images HTTP routes before their ordinary account-routing +pipeline. Consequently `$imagegen` fails closed inside the Daybreak profile; +do not remove the carrier to make it work during a restricted task. Use the +ordinary provider only for separate work that does not require Daybreak +routing. Provider configurations that intentionally skip OpenAI login have a +different eligibility path; see the [Images compatibility context](https://github.com/Soju06/codex-lb/blob/main/openspec/specs/images-api-compat/context.md#codex-provider-eligibility). ### WebSocket transport @@ -44,17 +152,11 @@ export CODEX_LB_UPSTREAM_STREAM_TRANSPORT=websocket `auto` is the default and uses native WebSockets for native Codex headers or models that prefer them. You can also switch this in the dashboard under Settings → Routing → Upstream stream transport. -Note: Codex itself does not currently expose a stable documented `wire_api = "websocket"` provider mode. -If you want to experiment on the Codex side, the current CLI exposes under-development feature flags: - -```toml -[features] -responses_websockets = true -# or -responses_websockets_v2 = true -``` - -These flags are experimental and do not replace `wire_api = "responses"`. +Note: Codex itself does not currently expose a stable documented +`wire_api = "websocket"` or WebSocket-only provider mode. +`supports_websockets = true` enables WebSocket attempts but does not disable +HTTP fallback. Removed `responses_websockets` feature flags are not a +fail-closed transport control. Upstream websocket handshakes automatically honor standard proxy environment variables when they are present. `wss://` handshakes check `wss_proxy`, `socks_proxy`, `https_proxy`, and `all_proxy`; @@ -145,19 +247,19 @@ jq 'del(.openai)' ~/.local/share/opencode/auth.json > auth.json.tmp && mv auth.j "name": "GPT-5.6-Sol", "reasoning": true, "options": { "reasoningEffort": "xhigh", "reasoningSummary": "detailed" }, - "limit": { "context": 372000, "output": 65536 } + "limit": { "context": 272000, "output": 65536 } }, "gpt-5.6-terra": { "name": "GPT-5.6-Terra", "reasoning": true, "options": { "reasoningEffort": "high", "reasoningSummary": "detailed" }, - "limit": { "context": 372000, "output": 65536 } + "limit": { "context": 272000, "output": 65536 } }, "gpt-5.6-luna": { "name": "GPT-5.6-Luna", "reasoning": true, "options": { "reasoningEffort": "medium", "reasoningSummary": "detailed" }, - "limit": { "context": 372000, "output": 65536 } + "limit": { "context": 272000, "output": 65536 } }, "gpt-5.5": { "name": "GPT-5.5", @@ -206,8 +308,8 @@ opencode { "id": "gpt-5.6-sol", "name": "gpt-5.6-sol (codex-lb)", - "contextWindow": 372000, - "contextTokens": 372000, + "contextWindow": 272000, + "contextTokens": 272000, "maxTokens": 4096, "input": ["text"], "reasoning": false @@ -215,8 +317,8 @@ opencode { "id": "gpt-5.6-terra", "name": "gpt-5.6-terra (codex-lb)", - "contextWindow": 372000, - "contextTokens": 372000, + "contextWindow": 272000, + "contextTokens": 272000, "maxTokens": 4096, "input": ["text"], "reasoning": false @@ -224,8 +326,8 @@ opencode { "id": "gpt-5.6-luna", "name": "gpt-5.6-luna (codex-lb)", - "contextWindow": 372000, - "contextTokens": 372000, + "contextWindow": 272000, + "contextTokens": 272000, "maxTokens": 4096, "input": ["text"], "reasoning": false @@ -286,4 +388,4 @@ print(response.choices[0].message.content) --- -*Specs: [responses-api-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/responses-api-compat) · [chat-completions-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/chat-completions-compat) · [model-catalog-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/model-catalog-compat) · [runtime-portability](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/runtime-portability)* +*Specs: [responses-api-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/responses-api-compat) · [images-api-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/images-api-compat) · [chat-completions-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/chat-completions-compat) · [realtime-api-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/realtime-api-compat) · [proxy-admission-control](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/proxy-admission-control) · [proxy-warmup](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/proxy-warmup) · [files-upload-protocol](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/files-upload-protocol) · [audio-transcriptions-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/audio-transcriptions-compat) · [model-catalog-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/model-catalog-compat) · [runtime-portability](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/runtime-portability)* diff --git a/docs/deployment/kubernetes.md b/docs/deployment/kubernetes.md index 1cb2545532..8b9a8d8bd0 100644 --- a/docs/deployment/kubernetes.md +++ b/docs/deployment/kubernetes.md @@ -51,6 +51,20 @@ The Helm chart auto-configures HTTP `/responses` owner handoff for multi-replica In multi-replica setups, replicas must share the same encryption key (the Helm chart default) for bootstrap-token restart recovery and encrypted-data access to work. +### Account concurrency caps are cluster-wide + +Under the default `CODEX_LB_PROXY_ACCOUNT_CAPS_SCOPE=partitioned`, `CODEX_LB_PROXY_ACCOUNT_STREAM_LIMIT` (default 8) and `CODEX_LB_PROXY_ACCOUNT_RESPONSE_CREATE_LIMIT` are cluster-wide targets. Each replica enforces its own deterministic share of a **positive** cap — `floor(cap / replicas)`, with the remainder distributed one slot at a time and every share floored at 1 — so with the default stream cap and three replicas, one account gets 3/3/2 slots per replica, not 8 each. A cap of `0` stays unlimited on every replica; it is never floored to one slot. Setting the scope to `replica` opts out of partitioning entirely: every replica then enforces the full configured cap. + +Practical consequences: + +- Size a positive cap for the total per-account concurrency you want across the cluster; adding replicas re-partitions it rather than raising it — except when the cap is smaller than the replica count, where the floor of 1 makes the aggregate equal the replica count and grow with each added replica. Disconnect-heavy or agent workloads typically want `~8 × replicas`. +- On an initialized deployment the caps live in **dashboard settings** (Settings → routing), which override the environment values — raising the env var and restarting pods changes nothing once the deployment is initialized. Change the cap from the dashboard; the environment values only seed the initial dashboard row. +- `CODEX_LB_PROXY_ACCOUNT_STREAM_RECOVERY_RESERVE` (default 1) is subtracted from each replica's share at selection time, so small shares feel it disproportionately: a share of 2 leaves 1 slot for new selection. +- Persistent `account_stream_cap` errors with idle replicas are the undersizing signature; raise the cap first. +- Run one process per pod (`workers_per_instance` stays 1): shares are partitioned across ring members, and worker processes inside one pod would silently multiply the share. + +Semantics and sizing rationale: [proxy-admission-control](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/proxy-admission-control). + ## Graceful shutdown The chart's preStop hook commits one process drain deadline before Uvicorn @@ -193,4 +207,4 @@ For external database, production config, ingress, observability, and more see t --- -*Specs: [deployment-installation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation) · [deployment-networking](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-networking) · [replica-operations](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/replica-operations)* +*Specs: [deployment-installation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation) · [deployment-networking](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-networking) · [replica-operations](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/replica-operations) · [proxy-admission-control](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/proxy-admission-control)* diff --git a/docs/examples/codex/config.toml b/docs/examples/codex/config.toml new file mode 100644 index 0000000000..67dddd5500 --- /dev/null +++ b/docs/examples/codex/config.toml @@ -0,0 +1,22 @@ +# Machine-local Codex configuration: ~/.codex/config.toml +# The default provider remains ordinary codex-lb traffic. +model = "gpt-5.6-sol" +model_reasoning_effort = "xhigh" +model_provider = "codex-lb" + +[model_providers.codex-lb] +name = "openai" +base_url = "http://127.0.0.1:2455/backend-api/codex" +wire_api = "responses" +supports_websockets = true +requires_openai_auth = true + +# Opt-in only. ~/.codex/daybreak-blue.config.toml selects this provider. +[model_providers.codex-lb-daybreak-blue] +name = "openai" +base_url = "http://127.0.0.1:2455/backend-api/codex" +wire_api = "responses" +env_key = "CODEX_LB_API_KEY" +supports_websockets = true +requires_openai_auth = true +http_headers = { "X-Codex-LB-Required-Capability" = "trusted_cyber" } diff --git a/docs/examples/codex/daybreak-blue.config.toml b/docs/examples/codex/daybreak-blue.config.toml new file mode 100644 index 0000000000..b3dc79100f --- /dev/null +++ b/docs/examples/codex/daybreak-blue.config.toml @@ -0,0 +1,3 @@ +# Machine-local Codex profile: ~/.codex/daybreak-blue.config.toml +model = "gpt-5.6-sol" +model_provider = "codex-lb-daybreak-blue" diff --git a/docs/index.md b/docs/index.md index 668ef7f154..dbcd4f58c8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,6 +19,7 @@ Load balancer for ChatGPT accounts. Pool multiple accounts, track usage, manage - [Getting Started](getting-started.md) — Docker / uvx quick start, remote bootstrap token - [Client Setup](client-setup.md) — Codex CLI, OpenCode, OpenClaw, Python SDK - [Configuration](configuration.md) — the few settings that matter +- [Anonymous Telemetry](telemetry.md) — collected fields, consent, disabling, and retention - [Authentication](authentication.md) — dashboard auth modes - [Conversations](conversations.md) — dashboard view and conversation APIs - [API Keys](api-keys.md) — protecting proxy routes diff --git a/docs/reference/settings.md b/docs/reference/settings.md index 2551bac015..8cf3e6b72c 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -7,7 +7,7 @@ Regenerate with `uv run python scripts/generate_settings_reference.py`; `tests/unit/test_settings_reference.py` fails when this page drifts from `app/core/config/settings.py`. -codex-lb currently exposes 119 settings. Every setting is an environment +codex-lb currently exposes 132 settings. Every setting is an environment variable with the `CODEX_LB_` prefix (process environment or `.env` / `.env.local` next to the process). All defaults work with zero configuration — start from [Configuration](../configuration.md) for the handful that matter, @@ -32,11 +32,11 @@ the host side of the compose `ports` mapping instead. | Environment variable | Type | Default | | --- | --- | --- | | `CODEX_LB_DATABASE_ALEMBIC_AUTO_REMAP_ENABLED` | `bool` | `True` | -| `CODEX_LB_DATABASE_MAX_OVERFLOW` | `int` | `10` | +| `CODEX_LB_DATABASE_MAX_OVERFLOW` | `int` | `15` | | `CODEX_LB_DATABASE_MIGRATE_ON_STARTUP` | `bool` | `True` | | `CODEX_LB_DATABASE_MIGRATION_LOCK_TIMEOUT_SECONDS` | `float` | `300.0` | | `CODEX_LB_DATABASE_MIGRATIONS_FAIL_FAST` | `bool` | `True` | -| `CODEX_LB_DATABASE_POOL_SIZE` | `int` | `15` | +| `CODEX_LB_DATABASE_POOL_SIZE` | `int` | `25` | | `CODEX_LB_DATABASE_POSTGRES_SCHEMA` | `str \| None` | `None` | | `CODEX_LB_DATABASE_SQLITE_PRE_MIGRATE_BACKUP_ENABLED` | `bool` | `True` | | `CODEX_LB_DATABASE_SQLITE_PRE_MIGRATE_BACKUP_MAX_FILES` | `int` | `5` | @@ -83,6 +83,7 @@ the host side of the compose `ports` mapping instead. | Environment variable | Type | Default | | --- | --- | --- | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_ADVERTISE_BASE_URL` | `str \| None` | `None` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_AMBIGUOUS_CONTINUATION_RECOVERY_MODE` | `'fail_closed' \| 'client_full_history_once' \| 'server_anchored_replay_once' \| 'server_indefinite_recovery'` | `'fail_closed'` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_ANCHOR_POISON_FAILURE_THRESHOLD` | `int` | `7` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_CLEAN_CLOSE_RETRY_JITTER_MAX_SECONDS` | `float` | `2.0` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_CODEX_IDLE_TTL_SECONDS` | `float` | `900.0` | @@ -93,6 +94,13 @@ the host side of the compose `ports` mapping instead. | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_INSTANCE_ID` | `str` | process hostname | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_INSTANCE_RING` | `list[str]` | `[]` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_MAX_SESSIONS` | `int` | `256` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_EVENT_SPOOL_BATCH_SIZE` | `int` | `32` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_EVENT_SPOOL_FLUSH_INTERVAL_SECONDS` | `float` | `0.1` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_EVENT_SPOOL_MAX_BYTES` | `int` | `2097152` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_EVENT_SPOOL_MAX_PENDING_BYTES` | `int` | `33554432` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_EVENT_SPOOL_MAX_PENDING_EVENTS` | `int` | `2048` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_LEDGER_ENABLED` | `bool` | `True` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_SPOOL_RETENTION_SECONDS` | `float` | `604800` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_QUEUE_LIMIT` | `int` | `8` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_REQUEST_BUDGET_SECONDS` | `float` | `7200.0` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_STUCK_GATE_RETIRE_AFTER_SECONDS` | `float` | `300.0` | @@ -141,6 +149,7 @@ the host side of the compose `ports` mapping instead. | Environment variable | Type | Default | | --- | --- | --- | | `CODEX_LB_LIVE_USAGE_INGESTION_ENABLED` | `bool` | `True` | +| `CODEX_LB_RATE_LIMIT_RESET_CREDITS_REFRESH_ENABLED` | `bool` | `True` | | `CODEX_LB_RATE_LIMIT_RESET_CREDITS_REFRESH_INTERVAL_SECONDS` | `int` | `60` | | `CODEX_LB_REQUEST_LOG_RETENTION_DAYS` | `int` | `0` | | `CODEX_LB_USAGE_FETCH_MAX_RETRIES` | `int` | `2` | @@ -243,6 +252,10 @@ the host side of the compose `ports` mapping instead. | Environment variable | Type | Default | | --- | --- | --- | +| `CODEX_LB_EVENT_LOOP_LAG_WARN_THRESHOLD_SECONDS` | `float` | `0.5` | +| `CODEX_LB_TELEMETRY_ENABLED` | `bool \| None` | `None` | +| `CODEX_LB_TELEMETRY_ENDPOINT` | `str` | `'https://telemetry.tokmaxxing.com'` | +| `CODEX_LB_TIMEOUT_INVARIANT_VALIDATION_STRICT` | `bool` | `False` | | `CODEX_LB_WARMUP_MODEL` | `str` | `'gpt-5.4-mini'` | ## Removed / deprecated @@ -311,4 +324,4 @@ issue [#1340](https://github.com/Soju06/codex-lb/issues/1340)): --- -*Specs: [user-documentation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/user-documentation) · [deployment-installation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation)* +*Specs: [user-documentation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/user-documentation) · [responses-api-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/responses-api-compat) · [rate-limit-reset-credits](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/rate-limit-reset-credits) · [deployment-installation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation) · [proxy-runtime-observability](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/proxy-runtime-observability)* diff --git a/docs/routing.md b/docs/routing.md index ece8042279..ad68f1a405 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -17,6 +17,36 @@ For low-volume, policy-compliant personal use, start with **Capacity weighted** Change the strategy live in the dashboard under **Settings → Routing** — no restart required. +## Routing, quotas, and eligibility explainer + +### Account eligibility vs displayed status + +An account's badge (`Active`, `Paused`, `Limited`, …) is its **displayed status**, derived from the durable account state plus current usage. Eligibility is decided **per request**: the selector can skip an `Active` account because of a cooldown, error backoff, a quota threshold or exhaustion, model/plan incompatibility, or because a thread's continuation state is owned by a different account. `Active` therefore does not mean "will serve the next request". + +### Soft sticky routing vs hard Codex continuation affinity + +These are two different mechanisms: + +- **Soft sticky routing** (the `Sticky threads` toggle and session/thread locality) is a *preference*: keep requests for the same session on the same account when possible, mostly to preserve warm upstream prompt caches. When the preferred account is unavailable or over the sticky thresholds, traffic can move. +- **Hard Codex continuation affinity** binds a request to the account that owns its continuation state — an explicit Codex turn state, a stored `previous_response_id`/conversation, or uploaded file ids. This binding is **not controlled by `Sticky threads`**: turning the toggle off does not make owner-bound requests portable. codex-lb releases the binding only when it can prove the request is a safe, account-neutral replay (or the continuation is migrated). + +If a thread's owner account becomes unavailable, requests that still require that owner can fail with `No available accounts` even though the rest of the pool is healthy. Starting a fresh thread (no continuation state) routes normally. + +### Primary vs secondary quota, used vs remaining + +- **Primary quota** is the short **5-hour** usage window. +- **Secondary quota** is the longer window: **weekly** on most plans, or **monthly** on plans that report only a monthly window (the monthly window is normalized into the secondary slot for routing). + +Account pages display each window as **percent remaining**; the sticky reallocation thresholds in Settings are **percent used**. A `Sticky secondary threshold` of `70` means "move sticky sessions off an account once more than 70% of its secondary (weekly or monthly) window has been used" — in quota terms, once less than 30% remains. Note that routing evaluates thresholds against reported usage **plus temporary in-flight pressure** (concurrent requests and leased tokens), so reallocation can begin slightly before the raw account-page numbers reach the threshold. + +### Prefer earlier reset + +When enabled and several accounts are otherwise eligible, selection is restricted to the accounts whose selected quota window (5h or weekly) resets soonest. Weekly resets are compared in whole-day buckets; when the selected window has no known reset time, the other window is used as a fallback. The preference applies to the `Capacity weighted`, `Usage weighted`, and `Fill first` strategies; the fixed-order and draw-based strategies (`Round robin`, `Relative availability`, `Sequential drain`, `Reset drain`, `Single account`) ignore it. + +### Limit warm-up + +Limit warm-up sends **one small real request** (using the configured warm-up model and prompt) to an opted-in account when one of its quota windows is confirmed to have newly reset, verifying that the account responds. It consumes a small amount of quota. The optional staggered idle mode additionally pre-starts the 5h window of idle opted-in accounts before traffic arrives; the configured cooldown applies to these staggered idle probes, while ordinary reset-confirmed probes fire once per confirmed reset. Accounts opt in individually (`Enable warm-up` in account actions); the last attempt's result, model, and time are shown on the account list entry. + --- -*Spec: [account-routing](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/account-routing)* +*Specs: [account-routing](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/account-routing) · [frontend-architecture](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/frontend-architecture) · [usage-refresh-policy](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/usage-refresh-policy)* diff --git a/docs/screenshots/api-key-ultrafast-after.jpg b/docs/screenshots/api-key-ultrafast-after.jpg new file mode 100644 index 0000000000..e9c89139dc Binary files /dev/null and b/docs/screenshots/api-key-ultrafast-after.jpg differ diff --git a/docs/screenshots/api-key-ultrafast-before.jpg b/docs/screenshots/api-key-ultrafast-before.jpg new file mode 100644 index 0000000000..54ecb306f2 Binary files /dev/null and b/docs/screenshots/api-key-ultrafast-before.jpg differ diff --git a/docs/screenshots/apis-enforced-reasoning-before.jpg b/docs/screenshots/apis-enforced-reasoning-before.jpg new file mode 100644 index 0000000000..2e3767fa65 Binary files /dev/null and b/docs/screenshots/apis-enforced-reasoning-before.jpg differ diff --git a/docs/screenshots/apis-reasoning-efforts.jpg b/docs/screenshots/apis-reasoning-efforts.jpg new file mode 100644 index 0000000000..413053e0be Binary files /dev/null and b/docs/screenshots/apis-reasoning-efforts.jpg differ diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 0000000000..e540d2793c --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,92 @@ +# Anonymous telemetry + +codex-lb sends an anonymous usage snapshot to the project-operated collector at +`https://telemetry.tokmaxxing.com` when the service starts and every 24 hours. In a +multi-replica deployment, only the elected leader builds and sends the snapshot. + +## What is sent + +Before the first consent decision, the dashboard shows the current JSON envelope. You can also +view it later from Settings. The signed snapshot body has three fields: + +```json +{ + "instance_id": "", + "metrics": { "": "..." }, + "timestamp": "" +} +``` + +The versioned `metrics` schema contains only these fields: + +- `schema_version`, active `consent` (`undecided` or `enabled`), random `instance_id`, codex-lb + `version`, Python version, OS, architecture, and process uptime +- `deploy`: deployment method, database backend and size bucket, replica count, and whether + trusted reverse-proxy headers are enabled +- `accounts`: bucketed pool and plan counts, whether workspace accounts exist, routing policy, + limit warmup, and whether an egress proxy is used +- `usage_7d`: request/success/token aggregates, bucketed cost, request-kind and transport/service + tier shares, allowlisted client families and model names, bucketed output-token averages, + latency percentiles, rate-limit ratio, and allowlisted upstream error codes +- `features`: booleans for optional features plus bucketed API-key count and model-source count + +Registration sends `app_name`, `app_version`, `deployment_mode`, an intentionally empty +`environment`, the random `instance_id`, coarse `os_arch`, and the Ed25519 `public_key` used to +verify signed updates. Activation sends only `{"action": "activate"}`. + +The schema never includes account emails, workspace identifiers, client IP addresses, API keys, +request or response content, raw user-agent strings, per-account records, custom model names, or +free-text errors. Exact schemas and privacy constraints live in the +[telemetry OpenSpec capability](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/telemetry). + +## Consent and disabling + +Telemetry uses informed opt-out consent. With no override or saved decision, it is active and +the dashboard presents a one-time dialog with the current payload. Enabling or disabling saves +the decision, and the Settings toggle can change it later. + +For a headless or deployment-level kill switch, set: + +```bash +CODEX_LB_TELEMETRY_ENABLED=false +``` + +An environment value overrides the saved dashboard setting. When telemetry resolves to +disabled, codex-lb opens no connection to the telemetry endpoint. The environment kill switch +is always completely silent. + +When a dashboard decision changes telemetry from active to inactive, codex-lb makes one final +signed request to `POST /v1/optout` so aggregate opt-out counts remain accurate. If the instance +has not contacted the collector in this process, it first performs the normal registration and +activation. Re-enabling and later disabling from the dashboard sends one new notice for that new +transition. Repeating an already-disabled decision sends nothing. + +The opt-out request uses the same `X-Instance-ID` and Ed25519 `X-Signature` headers as a snapshot. +Its canonical JSON body is: + +```json +{ + "app_version": "", + "event": "optout", + "instance_id": "", + "occurred_at": "" +} +``` + +This single decision-time notice is the only exception to disabled telemetry silence. It is +sent only for a dashboard-driven active-to-inactive transition; setting +`CODEX_LB_TELEMETRY_ENABLED=false`, or changing a saved decision while either environment +override value controls telemetry, never sends it. + +## Retention and failures + +Each snapshot summarizes the previous seven days of data already present in `request_logs`. +codex-lb does not keep a separate local telemetry history and does not queue a failed send. The +collector's server-side retention duration is not currently specified; assume transmitted +snapshots remain stored until a published retention policy or explicit deletion. + +Snapshot and opt-out endpoint failures use a five-second total timeout, retry no more than once, +are logged only at debug level, and never interrupt proxy traffic or change the dashboard +settings response. + +*Source of truth: [telemetry OpenSpec capability](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/telemetry)* diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 80121d75ae..bec37fc76d 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -11,9 +11,9 @@ codex-lb refreshes usage on its own schedule and treats upstream samples conserv **Codex CLI falls back to POST instead of WebSockets.** Run the [WebSocket verification steps](client-setup.md#verify-websocket-transport). If codex-lb sits behind a reverse proxy, make sure it forwards WebSocket upgrades — see [Remote Access](deployment/remote.md). -## Fast Mode and service tiers +## Fast Mode, Ultrafast, and service tiers -Fast Mode and service-tier behavior is documented in the +Fast Mode, Ultrafast, and service-tier behavior is documented in the [Responses API compatibility context](https://github.com/Soju06/codex-lb/blob/main/openspec/specs/responses-api-compat/context.md#fast-mode-and-service-tiers). ## Old Codex sessions missing after migrating diff --git a/docs/usage-reporting.md b/docs/usage-reporting.md new file mode 100644 index 0000000000..5b2199612e --- /dev/null +++ b/docs/usage-reporting.md @@ -0,0 +1,26 @@ +# Usage Reporting + +codex-lb records the token counts reported in the terminal Responses API event. It does not retokenize prompts or responses, and it does not estimate hidden reasoning usage. + +For direct Codex traffic over HTTP or WebSocket, the reported buckets have these relationships: + +- `input_tokens` includes the full input count; cached input is reported as a subset. +- `output_tokens` includes all generated output, including reasoning tokens. +- `reasoning_tokens` is the reported reasoning subset of `output_tokens`. +- Total tokens are `input_tokens + output_tokens`. Do not add cached input or reasoning tokens again. + +## Dashboard + +The **Request Logs** token cell shows total tokens, with reported cached-input and reasoning counts underneath. Open **Details** to see the exact reported reasoning count and its relationship to output tokens. + +The **Reports** page shows the reported reasoning total for the selected date range and filters. Its coverage count states how many requests supplied a reasoning value. The daily breakdown and CSV export include the same reasoning field. + +## Missing Usage + +A reported zero remains `0`. A missing value remains unknown and appears as `—` in the daily report or is omitted from request details. codex-lb does not turn missing usage into zero. + +Reasoning usage may be missing when the upstream terminal event does not include it, when a stream ends before that event arrives, or for older request-log rows. The dashboard does not backfill those rows. Custom OpenAI-compatible model sources do not currently feed reasoning details into this reporting path. + +--- + +*Spec: [frontend-architecture](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/frontend-architecture)* diff --git a/frontend/browser-smoke/dashboard.spec.ts b/frontend/browser-smoke/dashboard.spec.ts index 57a55606d1..74db517532 100644 --- a/frontend/browser-smoke/dashboard.spec.ts +++ b/frontend/browser-smoke/dashboard.spec.ts @@ -9,6 +9,7 @@ const REQUIRED_API_PATHS = [ "/api/dashboard/projections", "/api/request-logs/options", "/api/request-logs", + "/api/settings/telemetry", ] as const; test("the built dashboard accepts real backend responses", async ({ page }) => { @@ -63,6 +64,22 @@ test("the built dashboard accepts real backend responses", async ({ page }) => { } DashboardProjectionsSchema.parse(await projectionsResponse.json()); + // First run against an empty database resolves telemetry consent as + // undecided/default, so the informed-consent dialog must appear before + // anything else. Exercise it as a first-class scenario: verify the exact + // transmitted envelope is rendered, then keep telemetry enabled to unblock + // the dashboard underneath. + const consentDialog = page.getByRole("dialog", { name: "Anonymous telemetry" }); + await expect(consentDialog).toBeVisible(); + await expect(consentDialog.getByText('"instance_id"').first()).toBeVisible(); + const consentDecision = page.waitForResponse( + (response) => + new URL(response.url()).pathname === "/api/settings/telemetry" && response.request().method() === "PUT", + ); + await consentDialog.getByRole("button", { name: "Keep enabled" }).click(); + expect((await consentDecision).ok()).toBe(true); + await expect(consentDialog).toBeHidden(); + await expect(page.getByRole("heading", { name: "Dashboard", exact: true })).toBeVisible(); await expect(page.getByText("No accounts connected yet", { exact: true })).toBeVisible(); await expect(page.getByText("No requests yet", { exact: true })).toBeVisible(); diff --git a/frontend/bun.lock b/frontend/bun.lock index f695b9ea44..dd14d41e7c 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -5,7 +5,7 @@ "": { "name": "frontend", "dependencies": { - "@hookform/resolvers": "^5.7.1", + "@hookform/resolvers": "^5.8.0", "@tailwindcss/vite": "^4.3.3", "@tanstack/react-query": "^5.101.4", "class-variance-authority": "^0.7.1", @@ -14,27 +14,27 @@ "i18next": "^26.3.6", "i18next-browser-languagedetector": "^8.2.1", "input-otp": "^1.4.2", - "lucide-react": "^1.30.0", + "lucide-react": "^1.31.0", "radix-ui": "^1.6.7", "react": "^19.2.8", "react-day-picker": "^10.0.1", "react-dom": "^19.2.8", - "react-hook-form": "^7.84.0", + "react-hook-form": "^7.85.0", "react-i18next": "^17.0.11", "react-router-dom": "^7.18.2", "recharts": "^3.10.1", - "sonner": "^2.0.7", + "sonner": "^2.0.8", "tailwind-merge": "^3.6.0", "tailwindcss": "^4.3.3", "zod": "^4.4.3", - "zustand": "^5.0.14", + "zustand": "^5.0.15", }, "devDependencies": { "@eslint/js": "^10.0.1", "@playwright/test": "^1.62.1", - "@testing-library/jest-dom": "^7.0.0", + "@testing-library/jest-dom": "^7.0.1", "@testing-library/react": "^16.3.2", - "@testing-library/user-event": "^14.6.3", + "@testing-library/user-event": "^14.6.4", "@types/node": "^26.2.0", "@types/react": "^19.2.18", "@types/react-dom": "^19.2.4", @@ -43,15 +43,15 @@ "@vitest/coverage-v8": "^4.1.10", "eslint": "^10.8.1", "eslint-plugin-react-hooks": "^7.1.1", - "eslint-plugin-react-refresh": "^0.5.3", - "globals": "^17.9.0", + "eslint-plugin-react-refresh": "^0.5.4", + "globals": "^17.11.0", "jsdom": "^30.0.1", "msw": "^2.15.0", - "react-doctor": "^0.9.6", - "shadcn": "^4.16.2", + "react-doctor": "^0.9.12", + "shadcn": "^4.18.0", "tw-animate-css": "^1.4.0", "typescript": "npm:typescript@~6.0.3", - "typescript-eslint": "^8.66.0", + "typescript-eslint": "^8.67.0", "vite": "^8.2.1", "vitest": "^4.1.10", }, @@ -180,7 +180,7 @@ "@hono/node-server": ["@hono/node-server@1.19.9", "", { "peerDependencies": { "hono": "^4" } }, "sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw=="], - "@hookform/resolvers": ["@hookform/resolvers@5.7.1", "", { "dependencies": { "@standard-schema/utils": "^0.3.0" }, "peerDependencies": { "@sinclair/typebox": ">=0.25.24", "@standard-schema/spec": "^1.0.0", "@typeschema/main": ">=0.13.7", "@vinejs/vine": "^2.0.0 || ^3.0.0 || ^4.0.0", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", "ajv-formats": "^2.1.1", "arktype": "^2.0.0", "ata-validator": "^1.2.0", "class-transformer": ">=0.4.0", "class-validator": ">=0.12.0", "computed-types": "^1.0.0", "effect": "^3.10.3", "fluentvalidation-ts": "^3.0.0", "fp-ts": "^2.7.0", "io-ts": "^2.0.0", "joi": "^17.0.0", "nope-validator": ">=0.12.0", "react-hook-form": "^7.55.0", "superstruct": ">=0.12.0", "typanion": "^3.3.2", "valibot": ">=0.31.0 || ^1.0.0-beta.4 || ^1.0.0-rc", "vest": ">=3.0.0", "yup": "^1.0.0", "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["@sinclair/typebox", "@standard-schema/spec", "@typeschema/main", "@vinejs/vine", "ajv", "ajv-errors", "ajv-formats", "arktype", "ata-validator", "class-transformer", "class-validator", "computed-types", "effect", "fluentvalidation-ts", "fp-ts", "io-ts", "joi", "nope-validator", "superstruct", "typanion", "valibot", "vest", "yup", "zod"] }, "sha512-8wS/P4UDr5sQDe4nFaV51TVyfDPrWgNIXweqG0Bs9Z5LSuzKLb+RQNPvkN2oHM5SRrJyWrVH/F+LOUcFjUyvwQ=="], + "@hookform/resolvers": ["@hookform/resolvers@5.8.0", "", { "dependencies": { "@standard-schema/utils": "^0.3.0" }, "peerDependencies": { "@sinclair/typebox": ">=0.25.24", "@standard-schema/spec": "^1.0.0", "@typeschema/main": ">=0.13.7", "@vinejs/vine": "^2.0.0 || ^3.0.0 || ^4.0.0", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", "ajv-formats": "^2.1.1", "arktype": "^2.0.0", "ata-validator": "^1.2.0", "class-transformer": ">=0.4.0", "class-validator": ">=0.12.0", "computed-types": "^1.0.0", "effect": "^3.10.3", "fluentvalidation-ts": "^3.0.0", "fp-ts": "^2.7.0", "io-ts": "^2.0.0", "joi": "^17.0.0", "nope-validator": ">=0.12.0", "react-hook-form": "^7.55.0", "superstruct": ">=0.12.0", "typanion": "^3.3.2", "valibot": ">=0.31.0 || ^1.0.0-beta.4 || ^1.0.0-rc", "vest": ">=6.0.0", "yup": "^1.0.0", "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["@sinclair/typebox", "@standard-schema/spec", "@typeschema/main", "@vinejs/vine", "ajv", "ajv-errors", "ajv-formats", "arktype", "ata-validator", "class-transformer", "class-validator", "computed-types", "effect", "fluentvalidation-ts", "fp-ts", "io-ts", "joi", "nope-validator", "superstruct", "typanion", "valibot", "vest", "yup", "zod"] }, "sha512-2m6GvRLmYYK1Fwt093lGMf7db9l/+8pNuAtwoNkpBntJT4xcA5lNthYGWKViOc3z2SuaPD0HjE81pyXmqc1JyA=="], "@humanfs/core": ["@humanfs/core@0.19.1", "", {}, "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA=="], @@ -254,47 +254,45 @@ "@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.41.1", "", {}, "sha512-/UhIkaZgPutTFmQ7RnIJGgDXZmtEJ7Dvi86xNTFWcnRxVRNk/aotsqDJYeEvDP+FSMB2SdW+pQzNMcWP0rwuNA=="], - "@oxc-parser/binding-android-arm-eabi": ["@oxc-parser/binding-android-arm-eabi@0.142.0", "", { "os": "android", "cpu": "arm" }, "sha512-ZiRGDutGsv1G6bL/ozy/koC0Sv39T1DqyoC4KD1DOy9ZoACm1O5UWhEK2c02Qdk+4lfLVkvFa/mQ0fm/4h1BtQ=="], + "@oxc-parser/binding-android-arm-eabi": ["@oxc-parser/binding-android-arm-eabi@0.143.0", "", { "os": "android", "cpu": "arm" }, "sha512-n9uozULWflPqBtdmI8lAabLqGKNgLVNN0ZH8HfgCwpKGNtzRzauB76jTiW/3YLkcA7N1zskpi9GdVnZuu1SAvg=="], - "@oxc-parser/binding-android-arm64": ["@oxc-parser/binding-android-arm64@0.142.0", "", { "os": "android", "cpu": "arm64" }, "sha512-WZkvGRLNQTz8lR9zP5nLjUdlroRCopBu3g9zF1p/laE6DzT1UbQo8Rdz5MWhaJUPYg/6gp+jo7HUgsyKaN1FtQ=="], + "@oxc-parser/binding-android-arm64": ["@oxc-parser/binding-android-arm64@0.143.0", "", { "os": "android", "cpu": "arm64" }, "sha512-9BbdjHETk6O3zH/DDid9IgBtF0GlpLabNKN231uraXpRDSfY+iiZxTP5bk1Z63GBownVdhdINFIeddmMz4MzpQ=="], - "@oxc-parser/binding-darwin-arm64": ["@oxc-parser/binding-darwin-arm64@0.142.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-l4khS8LQOOVYsGRVARo1gSaCT/aBSceUVXgtovWc2+drnxVuDr082WA3OCHVdVzIz5JIrP/y9CWsSKxBDNmYGg=="], + "@oxc-parser/binding-darwin-arm64": ["@oxc-parser/binding-darwin-arm64@0.143.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gh+6ecoHUy4/sUcolBl/1qPXKBbYNxFY0Pk0ujgQvINTMSftJY7o4yb8gOkDJPeZeB8+a+u7xTe6umoP8N5HFA=="], - "@oxc-parser/binding-darwin-x64": ["@oxc-parser/binding-darwin-x64@0.142.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-QBsNF3nqlXmcH2B1YOPqQYmCJoy4HuIjUxGbBO/k5JAJUl68ghU2psRY2zPk+RyBaWqKP/qfL4oaFgEMCdwskA=="], + "@oxc-parser/binding-darwin-x64": ["@oxc-parser/binding-darwin-x64@0.143.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-qd1hl2d+lXgHv/VQ/M9qm8TrMC5T4RqDBwtOnl+1D0QMjwcz+8AaB4JSg8STgeag0GP6a6L74XEGAsrTSJWNzQ=="], - "@oxc-parser/binding-freebsd-x64": ["@oxc-parser/binding-freebsd-x64@0.142.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-b7Q7m4Cqc6XqNhri3R+QhU+GVy646Pn+bkdhrDdWym/Fdi0ZUa+d73H9dm5H91JtbtAQ/z1d8XKMW3oOV8a4tQ=="], + "@oxc-parser/binding-freebsd-x64": ["@oxc-parser/binding-freebsd-x64@0.143.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-M5XXcNa7aOqLPKTR41msfghKu2yQ4xWvCm11/gwU0JzOzHNk5sgW//rVEjJ+LO48+VDAMzXTSzurUVxIDKwozw=="], - "@oxc-parser/binding-linux-arm-gnueabihf": ["@oxc-parser/binding-linux-arm-gnueabihf@0.142.0", "", { "os": "linux", "cpu": "arm" }, "sha512-3riVS5IhdH3uCZj1Y9ftDQlR0dvLsIlw/edrRqk8JhgNd5K0XSs+UBtgh50N13CAlW9/TXj6sVGXaKNBocd0Yg=="], + "@oxc-parser/binding-linux-arm-gnueabihf": ["@oxc-parser/binding-linux-arm-gnueabihf@0.143.0", "", { "os": "linux", "cpu": "arm" }, "sha512-T/GXusuOkPNQhCQCSBbcU/N8j0rAypuDBl1IyFK+lyYT594XsVz80clPC/OtbSSpBGyJxj8uYEfctxVuxVYoww=="], - "@oxc-parser/binding-linux-arm-musleabihf": ["@oxc-parser/binding-linux-arm-musleabihf@0.142.0", "", { "os": "linux", "cpu": "arm" }, "sha512-NmXUOpgpTSkhl795TiXmWppTwmSJ92RC1qvD6e4XOF+slgmo3e6Ah+kEu+6AN8s7NAOEwqGmir58MgSQSWmBSA=="], + "@oxc-parser/binding-linux-arm-musleabihf": ["@oxc-parser/binding-linux-arm-musleabihf@0.143.0", "", { "os": "linux", "cpu": "arm" }, "sha512-oKu4RcBlXSqo3OC62dp6YTnQaZIurNDpCX3BnAM3+bJxt7s8J2TJKMnC0UYer1qhlRaDCg6wkTaTw+2IlsZ12w=="], - "@oxc-parser/binding-linux-arm64-gnu": ["@oxc-parser/binding-linux-arm64-gnu@0.142.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-gc0EXsKtXgerujmU2Bql3u1L1HsSQ2774R83idq/FoNMPVV/RY/1ErFsvnit7KoiP/sLvzQixeUo4Ut0ic0wmw=="], + "@oxc-parser/binding-linux-arm64-gnu": ["@oxc-parser/binding-linux-arm64-gnu@0.143.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-WJBbD186AZmMGaSIhlktC+rPl8L3peCTXAh88Ih9uEvK0en2mPojGyCGYiL6mHtV1RPV3JyfJW5t6n5hh0lXhA=="], - "@oxc-parser/binding-linux-arm64-musl": ["@oxc-parser/binding-linux-arm64-musl@0.142.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-F2XvmWSE0uWpie+jHKKIFgdVOe9ypGhkEZxKx5DuW215K6cbAC274yYaPkcM7EqY4Df3Weyhpcz3lsURyH2LVg=="], + "@oxc-parser/binding-linux-arm64-musl": ["@oxc-parser/binding-linux-arm64-musl@0.143.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-t1AcYOwEzgceadT4v5e+vaCCb0AncCA3v5AyzfBAz/tMq11qzVccXKzNHtkWdjBsgvTKwRkaUF3QvT4kot8vcQ=="], - "@oxc-parser/binding-linux-ppc64-gnu": ["@oxc-parser/binding-linux-ppc64-gnu@0.142.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-wLMbT21U/QxknQsk+VvNF0b9D2/aGWhcaQQQ+VYlE8FwD5+GoWZIPPXNzyHmkYyhm0KB3itL+TBavjMatqNnYA=="], + "@oxc-parser/binding-linux-ppc64-gnu": ["@oxc-parser/binding-linux-ppc64-gnu@0.143.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-RsnO/NoD8376LMJq8JS8TwI0ieNaFRTuNe2GVJntQg6gwZNMENZsEbknHdVwjpOmxdGLGodcwaGSbAeRr5Bgjw=="], - "@oxc-parser/binding-linux-riscv64-gnu": ["@oxc-parser/binding-linux-riscv64-gnu@0.142.0", "", { "os": "linux", "cpu": "none" }, "sha512-+G8F/4ckwT7FCJV4H2bt09xEzJbjNCfuL4Sp1AYNaFtFMVtgIGMuJlteT82U+K0UIZ/DzAR/LDlMFnEuajG7Kw=="], + "@oxc-parser/binding-linux-riscv64-gnu": ["@oxc-parser/binding-linux-riscv64-gnu@0.143.0", "", { "os": "linux", "cpu": "none" }, "sha512-48fSVfR9TZi5CASZFyv0VC6z6BCoeihFsX031mAD/oSH7d9PYsPgIqza7d9mjP7Z2KTEpTFyH6SIu0Ui6R1vdg=="], - "@oxc-parser/binding-linux-riscv64-musl": ["@oxc-parser/binding-linux-riscv64-musl@0.142.0", "", { "os": "linux", "cpu": "none" }, "sha512-hTsHtTLxMAfCo+rpF5K3qZJKW2NpPN/CHd4mYB3y7XlSdspHkd2gehDIofP64AacA9nWQw2tY3O7wR6UY8IVOA=="], + "@oxc-parser/binding-linux-riscv64-musl": ["@oxc-parser/binding-linux-riscv64-musl@0.143.0", "", { "os": "linux", "cpu": "none" }, "sha512-T8CpdD+SfE01DnIOD4HpVxu0ZJOfMJ/VhCvikKfaXAxkZ+9veyLM/D2hpi7Y2hFUyPmVQO3FNZHmYzV/WlVR4g=="], - "@oxc-parser/binding-linux-s390x-gnu": ["@oxc-parser/binding-linux-s390x-gnu@0.142.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-6y7qYY3TCUDYjqswImdTGl92y+KA/80twALegQPN27kfY+bG7Ib1+L3jbmrCZQx6wrVnai9IPsEZp07I0hx7JQ=="], + "@oxc-parser/binding-linux-s390x-gnu": ["@oxc-parser/binding-linux-s390x-gnu@0.143.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-QLdeMsCcacenPEFsfxnBUDF1y6opyz5+fmOz9bfD5Y7fiGCMupUCuB3KTPQhNwshIG1P9fPqar9MHxuBDd4bwQ=="], - "@oxc-parser/binding-linux-x64-gnu": ["@oxc-parser/binding-linux-x64-gnu@0.142.0", "", { "os": "linux", "cpu": "x64" }, "sha512-i69kAWU+2LgoH5bR+zWiiu+UzAw7Oxkwv7COeJTeY19pn4e70nKQcr9Pm6cL2Z0Z54d+gl9qADlK/0yyuCPiBA=="], + "@oxc-parser/binding-linux-x64-gnu": ["@oxc-parser/binding-linux-x64-gnu@0.143.0", "", { "os": "linux", "cpu": "x64" }, "sha512-659ujfqLy6k7cuH3sbzhd8b+ztSq+i6E2E9pG78Q0BmHjAExfGIdgc8cGgMdwAozDXeZFHkJ+LXYJdWsaGdgyw=="], - "@oxc-parser/binding-linux-x64-musl": ["@oxc-parser/binding-linux-x64-musl@0.142.0", "", { "os": "linux", "cpu": "x64" }, "sha512-4SQs678MmjYVrmhAgCWD4o0vpaFszXw9xLX5p2Z9MMFcltxiLkA88wQjh80YHjPrXtpyZ2CWI5m+1yNKM0m2Pw=="], + "@oxc-parser/binding-linux-x64-musl": ["@oxc-parser/binding-linux-x64-musl@0.143.0", "", { "os": "linux", "cpu": "x64" }, "sha512-/Mw/9j4TfZcnKphPrzOE6t4MMknXadcAAuVUlDRTF/ETWB5xOgQvOJV2Mh9We/bWxZdoxaGAdc+hy4GuYwQ2yQ=="], - "@oxc-parser/binding-openharmony-arm64": ["@oxc-parser/binding-openharmony-arm64@0.142.0", "", { "os": "none", "cpu": "arm64" }, "sha512-YHpx9N7Ln3a++Tc8rv+H7mrK1zyJQOAwCFg8LZ3lTs1T5afGWeZrLPhPT9HLnIwSjCyJqPWVMIrMxbjcmBr2oQ=="], + "@oxc-parser/binding-openharmony-arm64": ["@oxc-parser/binding-openharmony-arm64@0.143.0", "", { "os": "none", "cpu": "arm64" }, "sha512-8rIKWR2BFuifbIK/1XB9wTaSdtuJ25dlE7ZQYDnEwj/2xH2vHsxnvIjHT3ZjSVuLLwGGlSslIG/fbOJ8TV8rTw=="], - "@oxc-parser/binding-wasm32-wasi": ["@oxc-parser/binding-wasm32-wasi@0.142.0", "", { "dependencies": { "@emnapi/core": "1.11.2", "@emnapi/runtime": "1.11.2", "@napi-rs/wasm-runtime": "^1.1.6" }, "cpu": "none" }, "sha512-3pLDyY3+oogW73RM5uehNgAiR/Xfb7fvO2Q1Z1gIqZ2+50XDVQmBVlRkHXZTU4gKnQHpwETNsYQVsJ3joVB2iA=="], + "@oxc-parser/binding-win32-arm64-msvc": ["@oxc-parser/binding-win32-arm64-msvc@0.143.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-5U9kQYMfRRI6Zq7KDxgbIP0RMnKrfn3gLepRMgJuRkPSUALTiRCk9d/uyhb4lGDjUdzwK7mBkKqhLgzBPCmLpQ=="], - "@oxc-parser/binding-win32-arm64-msvc": ["@oxc-parser/binding-win32-arm64-msvc@0.142.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-Had/VeVY28Oyb0K+Q4FV8KCzoBycIh93oDK6pCbya9lkzdq+ikMHMgBubsdqqlybjJmQRawCQRrnBRHyQwYvcQ=="], + "@oxc-parser/binding-win32-ia32-msvc": ["@oxc-parser/binding-win32-ia32-msvc@0.143.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-25P7AaHk4R88Yv2XH4gToDVmh0cOu+bEURQU10CRrmvgabfRArSGAP5osmwUKeSUHj0VS50upbpbRWWW/m7mHA=="], - "@oxc-parser/binding-win32-ia32-msvc": ["@oxc-parser/binding-win32-ia32-msvc@0.142.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-GGi3+YphVHavvgs6gum2UXoNCqzHAmPt/nXkn8ZQZstV2Q1qZD1Mn8fz/nWrDkefHQtrG/+1/XrbMxsBTo6Svw=="], + "@oxc-parser/binding-win32-x64-msvc": ["@oxc-parser/binding-win32-x64-msvc@0.143.0", "", { "os": "win32", "cpu": "x64" }, "sha512-ORMh3JE1s6V7ySicdRK7vgaDQnn5o+UHg9ct989PlWHbel8O9ARrmWXM6kZjrBMtNucxNayQ8g69G0VfWzhANw=="], - "@oxc-parser/binding-win32-x64-msvc": ["@oxc-parser/binding-win32-x64-msvc@0.142.0", "", { "os": "win32", "cpu": "x64" }, "sha512-Ny/Wv4Us1LGC/ljwNTp+Hx3r/pH15EFfeDF0p+n898gt+TtRd6C9SccHcuUhDiNTb8s5tt7jdeAMDRQZ4Vq6hg=="], - - "@oxc-project/types": ["@oxc-project/types@0.142.0", "", {}, "sha512-7W+2q5AKQVU36fkaryontrHn3YDt1RyUYXatw9i5H8ocYe2sPKSFB6eS8WNPeRKiN1qAWWZUPm7gwFzJGrccqQ=="], + "@oxc-project/types": ["@oxc-project/types@0.143.0", "", {}, "sha512-u6JZdLBTLotrNC9Vd6vPssINdzcCzleKAH6EJKImQb7GtYvX5keN2dxkoK44stCc4tffE6QQRtZTXVSzsLUlWA=="], "@oxc-resolver/binding-android-arm-eabi": ["@oxc-resolver/binding-android-arm-eabi@11.24.2", "", { "os": "android", "cpu": "arm" }, "sha512-y09e0L0SRI2OA2tUIrjBgoV3eH5hvUKXNkJqXmNo5V2WxIjyC7I7aJfRLMEVpA8yi95f90gFDvO0VMgrDw+vwA=="], @@ -334,43 +332,43 @@ "@oxc-resolver/binding-win32-x64-msvc": ["@oxc-resolver/binding-win32-x64-msvc@11.24.2", "", { "os": "win32", "cpu": "x64" }, "sha512-UqGPmo56KDfLlfXFAFIrNflHT8tFxWGEivWg3Zeyp4Uy2NlKN1FGPr6/BxcLGG3+kZ6Wp14g5Uj+n71boqZfiw=="], - "@oxlint/binding-android-arm-eabi": ["@oxlint/binding-android-arm-eabi@1.76.0", "", { "os": "android", "cpu": "arm" }, "sha512-ZHIE5Zt9AsPDcY4nOlofXt0YfneEeo+QrKMPcPzLf2Z6Q8VtV2W73d7SFJ920WUwyik783u/doKCs3KXdwG+7w=="], + "@oxlint/binding-android-arm-eabi": ["@oxlint/binding-android-arm-eabi@1.77.0", "", { "os": "android", "cpu": "arm" }, "sha512-E06sKWS6PiI6HRxS1wyQg22HvApt01hI7fV+T3wUk3OSbaaP4a3hYGY/MIQDmASqCiRjBdpRQYkgMkqH82cWmQ=="], - "@oxlint/binding-android-arm64": ["@oxlint/binding-android-arm64@1.76.0", "", { "os": "android", "cpu": "arm64" }, "sha512-shm/ngQilHK6bs+ElJWa4oHfNj5vL1Gl/iVEJldTQjpr0/67oSgr0KUpbmcnLig5Fo0v/l6j2567A7TOL89ONA=="], + "@oxlint/binding-android-arm64": ["@oxlint/binding-android-arm64@1.77.0", "", { "os": "android", "cpu": "arm64" }, "sha512-NvsKz0KZxTp9cYWPLf+FXaSZwB3oO3peAjtukpOMBgse2vhQSoIIVqeO1yR0lEo/UcdZIDL18uq+kL0LzQ0ytA=="], - "@oxlint/binding-darwin-arm64": ["@oxlint/binding-darwin-arm64@1.76.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-rvJmrAPKSQ9aWJ6wIS6CK2tJjwzfW0ApQH9qokq6sfDvmHwoyIHxHFMq7z7i7GiV6fdE6s8qvBqWKPTu8RmT6Q=="], + "@oxlint/binding-darwin-arm64": ["@oxlint/binding-darwin-arm64@1.77.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-bgjTn6nW4bQCFBvSvuHCpDD+sONvmpo4lGI4PxzMt1quBA+xYxhczk6RiCn3GZ9gY8uhaBbwhj9MdKGfu6T9DA=="], - "@oxlint/binding-darwin-x64": ["@oxlint/binding-darwin-x64@1.76.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-U/zYdb7VYKGY6pA9Vd2rYl9O/HlCylcOlb5PGPvVLtg+oLGsk6H3XGKEMHKyqD3nmmtmlmwb/8SwU2vfSAtvMw=="], + "@oxlint/binding-darwin-x64": ["@oxlint/binding-darwin-x64@1.77.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-aotaIttH1R6j1Rwhx0M0htgeZyGtVQqYNTVEYMN/UcgHPquGA6kmk9OyuDc3a2GKUQBC+3C3GVQCcrRPMYqAFA=="], - "@oxlint/binding-freebsd-x64": ["@oxlint/binding-freebsd-x64@1.76.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-WvKG9CAriuo0XNiFzpXjDngUZcRGFNpaK2kLyMUsnJlShxkT96u+BpJQ3KqdQwGOrvI14L6V8bAwXwAYNNY6Jg=="], + "@oxlint/binding-freebsd-x64": ["@oxlint/binding-freebsd-x64@1.77.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-nNx/wta7ksRAdYvq+l4AWjXkLxEXHALhENxjj2cYbQAIR4ybaA5L+hCbE63HOmft5czQ6ks+hb8vmEAnn7YGPg=="], - "@oxlint/binding-linux-arm-gnueabihf": ["@oxlint/binding-linux-arm-gnueabihf@1.76.0", "", { "os": "linux", "cpu": "arm" }, "sha512-qJ5+RH99TqFRq3UCDxkW0zJJu9c+OAHFY72vGlxZLEpuO+MpKo3POgqb8sYipL9KYm8XY6ofb0HsOuvY6hQNqQ=="], + "@oxlint/binding-linux-arm-gnueabihf": ["@oxlint/binding-linux-arm-gnueabihf@1.77.0", "", { "os": "linux", "cpu": "arm" }, "sha512-tMLLjM7xXtzXisVCzkOTXNCy9bZVId2wteNwjohlFDR/jY6WagpEDA1c1wu4xRc20Hojaxj+V6DSR7gbKxijWA=="], - "@oxlint/binding-linux-arm-musleabihf": ["@oxlint/binding-linux-arm-musleabihf@1.76.0", "", { "os": "linux", "cpu": "arm" }, "sha512-PvPCVptkgVARsucgIqFQQcSmJ6xc6GtnVB5bRBekRahTc9eObMtjHfMjy5M+C2tHt5UCMttWM9RuSk/H9NqYeg=="], + "@oxlint/binding-linux-arm-musleabihf": ["@oxlint/binding-linux-arm-musleabihf@1.77.0", "", { "os": "linux", "cpu": "arm" }, "sha512-MiAFDFaqR0tmHTAyo0YDcZ5hyLREdYw/RQhc2R3cbT+8O3tB+zqPM2th9TTQ+Uo3jn/embS+DO+HyX9ztCPkOQ=="], - "@oxlint/binding-linux-arm64-gnu": ["@oxlint/binding-linux-arm64-gnu@1.76.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-3KeFDx8Bu4HPAXbuHZOr/oHvN+QT+JQhMw/NYPz7Z071xLSsG27Jfh9PIQVEY7hk1I+jr43ExqRIeJ6VKk2yLw=="], + "@oxlint/binding-linux-arm64-gnu": ["@oxlint/binding-linux-arm64-gnu@1.77.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-/xqQ3B16i1T4cyt/9Mn+4CpzhUXoBXp7kVpIwzOXNFLj5JmK1bIjsbSnX296Gg8A/o7oDtKWikFgBx0SLwztkw=="], - "@oxlint/binding-linux-arm64-musl": ["@oxlint/binding-linux-arm64-musl@1.76.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-oPFkkKTgl0K/EIg9fQ8oA3IGcI05/Mq1en04iFa41mmNPT+6KEiByVazTOZZJiHMBBrbsns1YJ2e1Scqwzesjw=="], + "@oxlint/binding-linux-arm64-musl": ["@oxlint/binding-linux-arm64-musl@1.77.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-LSbwuRKiNCenPDcbARqAZ5RfBy7gmj7vOvfJRLeCDU3gFtSxWbhv/+VTlaUqzUhNj1gFLHB8h7ALnxa/Az6z6g=="], - "@oxlint/binding-linux-ppc64-gnu": ["@oxlint/binding-linux-ppc64-gnu@1.76.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-gN7yZ0eqflA5Fhf1wvHxGUltIV3FsvmB1zhNMDEK9vSHhc7E6qg9CuPeBgPZab66Tjzq6w6kHAtNEvnTHf4cyw=="], + "@oxlint/binding-linux-ppc64-gnu": ["@oxlint/binding-linux-ppc64-gnu@1.77.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-QWdcH31mXEUe5Nq1s0CfCpceaKjIo9uZtwDjAuL681g1axf+5x8xrg/eXWaw//4NCxYZ4V4e5Hu5tvdR+pTBlg=="], - "@oxlint/binding-linux-riscv64-gnu": ["@oxlint/binding-linux-riscv64-gnu@1.76.0", "", { "os": "linux", "cpu": "none" }, "sha512-S/HqMbn22mQrjtErUxEoS/a55u8kIeXvreIxiJu5G7Le3UecEd6SQZxrDIpuhtgaFnsY/nVra3ytP+pRljDilA=="], + "@oxlint/binding-linux-riscv64-gnu": ["@oxlint/binding-linux-riscv64-gnu@1.77.0", "", { "os": "linux", "cpu": "none" }, "sha512-GnOfYgJxbcElOiPZaDFDl406ONddwvOWk2jvAAAEjwAl4GofNoHF+/HHUIBYa6bFCArlcGPi0XjC4cU1pkgF/Q=="], - "@oxlint/binding-linux-riscv64-musl": ["@oxlint/binding-linux-riscv64-musl@1.76.0", "", { "os": "linux", "cpu": "none" }, "sha512-ZIga3097VJZolGZk6SrIAUokIGfRkxRlhiHDUznZptGBfwrhD7pNfD1rzEzsCwvk/1DX0A1bLz+liuNh5QKIVQ=="], + "@oxlint/binding-linux-riscv64-musl": ["@oxlint/binding-linux-riscv64-musl@1.77.0", "", { "os": "linux", "cpu": "none" }, "sha512-AyEMTUCf0xY+hHF+IxqXFQIX0yQOIR8ykpY0lJNOw9xYqOzUX8dyZfRvlG0RfXwuQn2eonf/8NrMmDSZJjdqsA=="], - "@oxlint/binding-linux-s390x-gnu": ["@oxlint/binding-linux-s390x-gnu@1.76.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-ZGiiA7pFzMJSyMWYZTVlPgbTsx+Vl8ihLGMIujPwaslUF7kIPPWAbVmAlTc+9lWDV+DCiB8Ikixu+lSHeOIIWQ=="], + "@oxlint/binding-linux-s390x-gnu": ["@oxlint/binding-linux-s390x-gnu@1.77.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-sPLzEcNvxd/oyVQ5oZo92CiHkFkpBeRop13E/P3TPY+hZfXHKCOWKI70TE2RYwMKFJDc20EMjH16L7NZICtKTw=="], - "@oxlint/binding-linux-x64-gnu": ["@oxlint/binding-linux-x64-gnu@1.76.0", "", { "os": "linux", "cpu": "x64" }, "sha512-JLiy5WuvEBFTT6ErIFV35SLzi0R7Iri6MKU6dZbTxfIx8pndbbPs3Mj780nMipBFcPkti+okAPOJ9POKkHFEgg=="], + "@oxlint/binding-linux-x64-gnu": ["@oxlint/binding-linux-x64-gnu@1.77.0", "", { "os": "linux", "cpu": "x64" }, "sha512-1Oh2ssH2L7lwyvkdSqaMUfsGfwU2Wfvew+obBUYjRVqhpBcUpwnsPSEr1IzVi9XqkuY10geiLsNKecqaZC34Dw=="], - "@oxlint/binding-linux-x64-musl": ["@oxlint/binding-linux-x64-musl@1.76.0", "", { "os": "linux", "cpu": "x64" }, "sha512-z7lgKQtbo/I1NIe8G5NHLesxJDv0tRSUWTpXKb9Pm3E9nKFKfO4IOSDtFroKgXtOYb0jQbcdH+0wzTyMXVes+A=="], + "@oxlint/binding-linux-x64-musl": ["@oxlint/binding-linux-x64-musl@1.77.0", "", { "os": "linux", "cpu": "x64" }, "sha512-0j/2wRgNGO+Qj/M1uu/p57h/hFTTWWcfie0ufkbabeus2s5+/QqkCflnMOwLLN5m2GsNeWp4xdl4cPa4n7QCOQ=="], - "@oxlint/binding-openharmony-arm64": ["@oxlint/binding-openharmony-arm64@1.76.0", "", { "os": "none", "cpu": "arm64" }, "sha512-JOjKymIpb9QcYfEhZsN6h4V9Ivd474W38cNIBRv6bg2TbIvogbMTH0Mg6YWW9TiRDqfcX+/Hyfsbo5vcSE5guQ=="], + "@oxlint/binding-openharmony-arm64": ["@oxlint/binding-openharmony-arm64@1.77.0", "", { "os": "none", "cpu": "arm64" }, "sha512-BJ/j54qS0usEnyDkLYURMj2iiD9h5Cyy+ppzeMSXBGRXaGRNWnj1Mw14NqWMR5E/PzdgB30OOCCzLzbRoduafw=="], - "@oxlint/binding-win32-arm64-msvc": ["@oxlint/binding-win32-arm64-msvc@1.76.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-pqDWZiwcmByWUEm1NFUBNiT6aentCcaoMWJv0HbXEmuYermJ4sg8ppVrshubYP2MZ6SHccJJcpr6x469PuDFIw=="], + "@oxlint/binding-win32-arm64-msvc": ["@oxlint/binding-win32-arm64-msvc@1.77.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-Yh8w+g2Lpx7StrvtYkoz9JJvXjB9wxgFChFNb85nrXm/wj/XTwGWS1hve9+900HL7llrntYB3YP+y32E3tRqzA=="], - "@oxlint/binding-win32-ia32-msvc": ["@oxlint/binding-win32-ia32-msvc@1.76.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-Ba0O659kgMv6pwO3z9PdO+K3aMxQRaw9HnG+e6AtOfgwcKFvYilciQYBoUBmxfQvOCKZe1SwjMkuB542NkuDMQ=="], + "@oxlint/binding-win32-ia32-msvc": ["@oxlint/binding-win32-ia32-msvc@1.77.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-zja5b7+6a7UsRFgAQSrnax5vrzliEyNPLCjfXONu/vTWswaIVZGFajJZptaeRvPE4LghtFdAzVFlexTm7MVTGA=="], - "@oxlint/binding-win32-x64-msvc": ["@oxlint/binding-win32-x64-msvc@1.76.0", "", { "os": "win32", "cpu": "x64" }, "sha512-5qcirPHO8nKfkoowEVWtpAoVTcYDy6g0UT0NGic450Qv8J2NrOqg4uQ8QppRP4MDTC7Xx47lbZnmadTH03CGGA=="], + "@oxlint/binding-win32-x64-msvc": ["@oxlint/binding-win32-x64-msvc@1.77.0", "", { "os": "win32", "cpu": "x64" }, "sha512-+teyvPDZ2RjUvo+SuCqS/UhaJl1QtdW5fWT5NJTV61V5MIuIS90Db9LixmtEGvXixyttiK62P96MSu3UlpviBw=="], "@playwright/test": ["@playwright/test@1.62.1", "", { "dependencies": { "playwright": "1.62.1" }, "bin": { "playwright": "cli.js" } }, "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ=="], @@ -614,11 +612,11 @@ "@testing-library/dom": ["@testing-library/dom@10.4.1", "", { "dependencies": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", "@types/aria-query": "^5.0.1", "aria-query": "5.3.0", "dom-accessibility-api": "^0.5.9", "lz-string": "^1.5.0", "picocolors": "1.1.1", "pretty-format": "^27.0.2" } }, "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg=="], - "@testing-library/jest-dom": ["@testing-library/jest-dom@7.0.0", "", { "dependencies": { "@adobe/css-tools": "^4.4.0", "aria-query": "^5.0.0", "css.escape": "^1.5.1", "dom-accessibility-api": "^0.6.3", "picocolors": "^1.1.1", "redent": "^3.0.0" }, "peerDependencies": { "@testing-library/dom": ">=10 <11" } }, "sha512-HKAH9C6mBo5yBG6yRO5i43L2iisencAo5z+o5P/saHUoY+miC5ivXRxHBJcFyB5ypPNxHJdK3BoF/3O4DIptMg=="], + "@testing-library/jest-dom": ["@testing-library/jest-dom@7.0.1", "", { "dependencies": { "@adobe/css-tools": "^4.4.0", "aria-query": "^5.0.0", "css.escape": "^1.5.1", "dom-accessibility-api": "^0.6.3", "picocolors": "^1.1.1", "redent": "^3.0.0" }, "peerDependencies": { "@testing-library/dom": ">=10 <11", "vitest": ">= 0.32" }, "optionalPeers": ["vitest"] }, "sha512-oMDTC3oA+6CXSO2JZnvOI7CA6oVub6kij5ggk9ohwye5slmkwxYDXcPOVxgMw/RQlticjtO0C1RZkR97HgrWMw=="], "@testing-library/react": ["@testing-library/react@16.3.2", "", { "dependencies": { "@babel/runtime": "^7.12.5" }, "peerDependencies": { "@testing-library/dom": "^10.0.0", "@types/react": "^18.0.0 || ^19.0.0", "@types/react-dom": "^18.0.0 || ^19.0.0", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-XU5/SytQM+ykqMnAnvB2umaJNIOsLF3PVv//1Ew4CTcpz0/BRyy/af40qqrt7SjKpDdT1saBMc42CUok5gaw+g=="], - "@testing-library/user-event": ["@testing-library/user-event@14.6.3", "", { "peerDependencies": { "@testing-library/dom": ">=7.21.4" } }, "sha512-6dBq67jT8lE+JTE8Exm02Kt6ze43hz1jdiSpSJwtTZiT1xQQ6b7nZYTTQ9njdArdU8XklOwaDp/AbT/eYSKF4g=="], + "@testing-library/user-event": ["@testing-library/user-event@14.6.4", "", { "peerDependencies": { "@testing-library/dom": ">=7.21.4" } }, "sha512-QCGwP6QrjypBLwyj5cuyfVamkaIEy/XGY+1VDehbtbQqOggYmTFpFOdWR5mPz14vX8vXLMVjDHlRNBcClyO9ew=="], "@ts-morph/common": ["@ts-morph/common@0.27.0", "", { "dependencies": { "fast-glob": "^3.3.3", "minimatch": "^10.0.1", "path-browserify": "^1.0.1" } }, "sha512-Wf29UqxWDpc+i61k3oIOzcUfQt79PIT9y/MWfAGlrkjg6lBC1hwDECLXPVJAhWjiGbfBCxZd65F/LIZF3+jeJQ=="], @@ -668,25 +666,25 @@ "@types/validate-npm-package-name": ["@types/validate-npm-package-name@4.0.2", "", {}, "sha512-lrpDziQipxCEeK5kWxvljWYhUvOiB2A9izZd9B2AFarYAkqZshb4lPbRs7zKEic6eGtH8V/2qJW+dPp9OtF6bw=="], - "@typescript-eslint/eslint-plugin": ["@typescript-eslint/eslint-plugin@8.66.0", "", { "dependencies": { "@eslint-community/regexpp": "^4.12.2", "@typescript-eslint/scope-manager": "8.66.0", "@typescript-eslint/type-utils": "8.66.0", "@typescript-eslint/utils": "8.66.0", "@typescript-eslint/visitor-keys": "8.66.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "@typescript-eslint/parser": "^8.66.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-p088eaGrzYz1s+7cov0aMOCkNGTJlVxF4jgubf28c8L0Cv9Rloj8YBHnv4hXLq6IIEE1AsjNWavO+k+8kP2Y0A=="], + "@typescript-eslint/eslint-plugin": ["@typescript-eslint/eslint-plugin@8.67.0", "", { "dependencies": { "@eslint-community/regexpp": "^4.12.2", "@typescript-eslint/scope-manager": "8.67.0", "@typescript-eslint/type-utils": "8.67.0", "@typescript-eslint/utils": "8.67.0", "@typescript-eslint/visitor-keys": "8.67.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "@typescript-eslint/parser": "^8.67.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-Un7Heoyj65NREbKAyIrFxeM143NZpExWmy1Nep4DLeQOeLlTeumPjoNKnBrU5D5moWXbPJgRa5Uwcdu0faVNGQ=="], - "@typescript-eslint/parser": ["@typescript-eslint/parser@8.66.0", "", { "dependencies": { "@typescript-eslint/scope-manager": "8.66.0", "@typescript-eslint/types": "8.66.0", "@typescript-eslint/typescript-estree": "8.66.0", "@typescript-eslint/visitor-keys": "8.66.0", "debug": "^4.4.3" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-X6ypGChaWYk6PBtUg2BwuTZEFFcHJAtGTVJ9/lCTOufhZ4i9fNolQNnktq+kkMCwMj7V8Svsq7+TxSDslmhE0g=="], + "@typescript-eslint/parser": ["@typescript-eslint/parser@8.67.0", "", { "dependencies": { "@typescript-eslint/scope-manager": "8.67.0", "@typescript-eslint/types": "8.67.0", "@typescript-eslint/typescript-estree": "8.67.0", "@typescript-eslint/visitor-keys": "8.67.0", "debug": "^4.4.3" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-fUBfTuuEulWqX6V8+O3PtScV01tzYYRUDTAirHFKoRAt7nOzoGiPt0M/bB47wWNy0coOOcgEwAMUtBpykMxl6w=="], - "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.66.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.66.0", "@typescript-eslint/types": "^8.66.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-7MthGPTt4BP69lSryqpqq8HQqxuzynssckL/jyDyk3+TNMQ3y2jFWkptCrktWvBrP+EH787Nl5N5Qpw7WZg+5g=="], + "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.67.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.67.0", "@typescript-eslint/types": "^8.67.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-cvE8c7ulYeXN9fYuszhCeCsbzyVEXuhrRCybnBre7TUmqb5nRmBfQAwCj0O3WJFDeyAZt4VYv51vMCC9LHSdYw=="], - "@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.66.0", "", { "dependencies": { "@typescript-eslint/types": "8.66.0", "@typescript-eslint/visitor-keys": "8.66.0" } }, "sha512-8TGcH25j9zqJ/IULB/ppyhRvxA8QYfFEZ7nfbg6/BN9spDgb8fPWQXlE5l8TWBL50EtUx007uZ1o9VOwrq2/9g=="], + "@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.67.0", "", { "dependencies": { "@typescript-eslint/types": "8.67.0", "@typescript-eslint/visitor-keys": "8.67.0" } }, "sha512-EgvsleTwS4E+WzzSvem8fAUubLwatMNF1B5hHSLQxcvs7q2dtRhGyujHwLJSYlG41niJ7GP24Aha2+0mb1b2kg=="], - "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.66.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-9D5gLYZG4rOjcoag8MQ/fWI8WqA9wcPDyOGyWtWFhvM1lHRbliqUSPIY5J3zqCU1tvSwzXxnnjhQhz5Ne7mJ4g=="], + "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.67.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-vV+LUSv5njUWsknE71fqKTlXUva+R76SaeORd6Zojcunk/6DvKFXONU3BrAs2H49mbygUXt6gbYunzwqNwlhdg=="], - "@typescript-eslint/type-utils": ["@typescript-eslint/type-utils@8.66.0", "", { "dependencies": { "@typescript-eslint/types": "8.66.0", "@typescript-eslint/typescript-estree": "8.66.0", "@typescript-eslint/utils": "8.66.0", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-LG2dWfjZQQp0ADtAu/EWJVayefGL2UEZ3CDeI44D9v3rXB/WYUqE/jpO28KrEKul5AySrmI+Zh1v6v+xW2U9+g=="], + "@typescript-eslint/type-utils": ["@typescript-eslint/type-utils@8.67.0", "", { "dependencies": { "@typescript-eslint/types": "8.67.0", "@typescript-eslint/typescript-estree": "8.67.0", "@typescript-eslint/utils": "8.67.0", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-aVWDXbRmdXO9siTfX4ditQI1T9+zVcNazT48EJCD0v40/9RIFoUgZ05CmGEq9H2gixRpjUn/iplwvlcvutJW/Q=="], - "@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], + "@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], - "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.66.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.66.0", "@typescript-eslint/tsconfig-utils": "8.66.0", "@typescript-eslint/types": "8.66.0", "@typescript-eslint/visitor-keys": "8.66.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-8/x4INiiQb10jGgXYD7116/zQ+OL84ZIFn0za68wwFHCanT/VLbBEroWht8RV8fn0/ZCAoazHLQgwUC0UQcDfg=="], + "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.67.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.67.0", "@typescript-eslint/tsconfig-utils": "8.67.0", "@typescript-eslint/types": "8.67.0", "@typescript-eslint/visitor-keys": "8.67.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-EKQBCE9yNlRJYm7jdTW5AhDacDUmSwQb0FAJAmK2EKYrNXIsa2vxcSZx6PvJ/dEdI6lS+Y9W+EXckLj0iPFGcw=="], - "@typescript-eslint/utils": ["@typescript-eslint/utils@8.66.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.66.0", "@typescript-eslint/types": "8.66.0", "@typescript-eslint/typescript-estree": "8.66.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-jasearZPolBw5NJNYGMwxzHMF83niVWmMU1VdHzG1CyfI2VS7f7nZltnKtHcg20hW+7Uo5GfK4MeDPoU3qI8EA=="], + "@typescript-eslint/utils": ["@typescript-eslint/utils@8.67.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.67.0", "@typescript-eslint/types": "8.67.0", "@typescript-eslint/typescript-estree": "8.67.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-U9D1FdwEWBwok3hxxSdhclMb0twvt9QnjIQ0VfQ1AiX2epnpSgv2ubVDsayOFyY8K6FX+AQ7E0FKWVG3iKsj1A=="], - "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.66.0", "", { "dependencies": { "@typescript-eslint/types": "8.66.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-dkKR8q+lKciskj1Y3vthHktl+3cMLWGyVUP23bRiPZ5O9BRT++4EqDDV+TVeIKBL1VXVEqrJlz8MYbcnvJcAlg=="], + "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.67.0", "", { "dependencies": { "@typescript-eslint/types": "8.67.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-fkv8dHRDqfGtTHuJeebdrQ7cX6Ad4WAS00rgHh9UGvMycF1mjBfsxry1XsLIFhWZ6Judlh6UdzK+TYlbpCXgnA=="], "@typescript/native": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], @@ -910,7 +908,7 @@ "dequal": ["dequal@2.0.3", "", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="], - "deslop-js": ["deslop-js@0.9.6", "", { "dependencies": { "@oxc-project/types": "^0.142.0", "fast-glob": "^3.3.3", "minimatch": "^10.2.5", "oxc-parser": "^0.142.0", "oxc-resolver": "^11.24.2", "typescript": ">=5.0.4 <6" } }, "sha512-ADY8/3JsK0epUcdXez54MhY4CGWeppT032Ko1fJkwFfmHog5hBJtMeihUeHs5PF7eG6Y17YbKcNQXSp3B67GRA=="], + "deslop-js": ["deslop-js@0.9.12", "", { "dependencies": { "@oxc-project/types": "^0.143.0", "fast-glob": "^3.3.3", "minimatch": "^10.2.5", "oxc-parser": "^0.143.0", "oxc-resolver": "^11.24.2", "typescript": ">=5.0.4 <6" } }, "sha512-Ku6Zngzmu4EISb58WUkRKZytfgWjJ18Cic7lb4wl+/BF0tHWRvpBOLlA0FP35r82mV45Y72AK3RPC1Nw0GLbIw=="], "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], @@ -964,7 +962,7 @@ "eslint-plugin-react-hooks": ["eslint-plugin-react-hooks@7.1.1", "", { "dependencies": { "@babel/core": "^7.24.4", "@babel/parser": "^7.24.4", "hermes-parser": "^0.25.1", "zod": "^3.25.0 || ^4.0.0", "zod-validation-error": "^3.5.0 || ^4.0.0" }, "peerDependencies": { "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0 || ^10.0.0" } }, "sha512-f2I7Gw6JbvCexzIInuSbZpfdQ44D7iqdWX01FKLvrPgqxoE7oMj8clOfto8U6vYiz4yd5oKu39rRSVOe1zRu0g=="], - "eslint-plugin-react-refresh": ["eslint-plugin-react-refresh@0.5.3", "", { "peerDependencies": { "eslint": "^9 || ^10" } }, "sha512-5EMmLCV98Pi4o/f/3DP/v/tNqLHMIc9I8LKClNDWhZ9JTho89/kQcitCXQBMG7sAfVRK0Ie3T2EDOzp1YXYiVA=="], + "eslint-plugin-react-refresh": ["eslint-plugin-react-refresh@0.5.4", "", { "peerDependencies": { "eslint": "^9 || ^10" } }, "sha512-7bqTKz7T0r+HKWFarNXByDE9/5+73wI2ru+M3zuqGbR7s/b/5/pQJXZoufWlrngqGqoZto73ZkGumCdLxk+4rw=="], "eslint-scope": ["eslint-scope@9.1.2", "", { "dependencies": { "@types/esrecurse": "^4.3.1", "@types/estree": "^1.0.8", "esrecurse": "^4.3.0", "estraverse": "^5.2.0" } }, "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ=="], @@ -1064,7 +1062,7 @@ "glob-parent": ["glob-parent@6.0.2", "", { "dependencies": { "is-glob": "^4.0.3" } }, "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A=="], - "globals": ["globals@17.9.0", "", {}, "sha512-m/MvAW61QVU5VDNF1Vj8axt016h8w7L5TU1e9zlab7XIttAT2YAlCwl75K1fOqvMM9apmD7lbCIRhpfkhmxhCg=="], + "globals": ["globals@17.11.0", "", {}, "sha512-Z2I8hM+PbJDXQDq3Icgpzv+mPdwr68iZUU9d5WW4FuXfDUQfkZaZuvjMv42/5crNyw154+9+VWXbYrUgDXbxNw=="], "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], @@ -1120,7 +1118,7 @@ "internmap": ["internmap@2.0.3", "", {}, "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg=="], - "ip-address": ["ip-address@10.0.1", "", {}, "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA=="], + "ip-address": ["ip-address@10.5.0", "", {}, "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g=="], "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], @@ -1234,7 +1232,7 @@ "lru-cache": ["lru-cache@11.5.2", "", {}, "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g=="], - "lucide-react": ["lucide-react@1.30.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-tUIr2jXLbWpCkdtH8XP7P7YppM9ueWgTky99lpWDY6z5REs6B+O6ZQ3U5tHkUUY59ANyOv/PBcs8E4Fe3KO3eA=="], + "lucide-react": ["lucide-react@1.31.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-G8u2eEtoHUnUa9f8lbvqDhCiORMnYLdUEo06EEG9MQvHQrInKcX3Pa2TH39MM5qyzRcWETxB0+aOwAPI1g1kEg=="], "lz-string": ["lz-string@1.5.0", "", { "bin": "bin/bin.js" }, "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ=="], @@ -1312,13 +1310,13 @@ "outvariant": ["outvariant@1.4.3", "", {}, "sha512-+Sl2UErvtsoajRDKCE5/dBz4DIvHXQQnAxtQTF04OJxY0+DyZXSo5P5Bb7XYWOh81syohlYL24hbDwxedPUJCA=="], - "oxc-parser": ["oxc-parser@0.142.0", "", { "dependencies": { "@oxc-project/types": "^0.142.0" }, "optionalDependencies": { "@oxc-parser/binding-android-arm-eabi": "0.142.0", "@oxc-parser/binding-android-arm64": "0.142.0", "@oxc-parser/binding-darwin-arm64": "0.142.0", "@oxc-parser/binding-darwin-x64": "0.142.0", "@oxc-parser/binding-freebsd-x64": "0.142.0", "@oxc-parser/binding-linux-arm-gnueabihf": "0.142.0", "@oxc-parser/binding-linux-arm-musleabihf": "0.142.0", "@oxc-parser/binding-linux-arm64-gnu": "0.142.0", "@oxc-parser/binding-linux-arm64-musl": "0.142.0", "@oxc-parser/binding-linux-ppc64-gnu": "0.142.0", "@oxc-parser/binding-linux-riscv64-gnu": "0.142.0", "@oxc-parser/binding-linux-riscv64-musl": "0.142.0", "@oxc-parser/binding-linux-s390x-gnu": "0.142.0", "@oxc-parser/binding-linux-x64-gnu": "0.142.0", "@oxc-parser/binding-linux-x64-musl": "0.142.0", "@oxc-parser/binding-openharmony-arm64": "0.142.0", "@oxc-parser/binding-wasm32-wasi": "0.142.0", "@oxc-parser/binding-win32-arm64-msvc": "0.142.0", "@oxc-parser/binding-win32-ia32-msvc": "0.142.0", "@oxc-parser/binding-win32-x64-msvc": "0.142.0" } }, "sha512-kKR+jPiRJYJDexVoziIg/FVGvr1fT1FZSSJOk6tVoMKKSlsf1Cso+cgGCJkOEDWOP174vRntCPFKg+AS7InWvw=="], + "oxc-parser": ["oxc-parser@0.143.0", "", { "dependencies": { "@oxc-project/types": "^0.143.0" }, "optionalDependencies": { "@oxc-parser/binding-android-arm-eabi": "0.143.0", "@oxc-parser/binding-android-arm64": "0.143.0", "@oxc-parser/binding-darwin-arm64": "0.143.0", "@oxc-parser/binding-darwin-x64": "0.143.0", "@oxc-parser/binding-freebsd-x64": "0.143.0", "@oxc-parser/binding-linux-arm-gnueabihf": "0.143.0", "@oxc-parser/binding-linux-arm-musleabihf": "0.143.0", "@oxc-parser/binding-linux-arm64-gnu": "0.143.0", "@oxc-parser/binding-linux-arm64-musl": "0.143.0", "@oxc-parser/binding-linux-ppc64-gnu": "0.143.0", "@oxc-parser/binding-linux-riscv64-gnu": "0.143.0", "@oxc-parser/binding-linux-riscv64-musl": "0.143.0", "@oxc-parser/binding-linux-s390x-gnu": "0.143.0", "@oxc-parser/binding-linux-x64-gnu": "0.143.0", "@oxc-parser/binding-linux-x64-musl": "0.143.0", "@oxc-parser/binding-openharmony-arm64": "0.143.0", "@oxc-parser/binding-win32-arm64-msvc": "0.143.0", "@oxc-parser/binding-win32-ia32-msvc": "0.143.0", "@oxc-parser/binding-win32-x64-msvc": "0.143.0" } }, "sha512-ov0NzaDCOInknS7mP1cwKdJERt3utPW8ldjtdUXQ8Ty0GEFD08wk422vCUN0d7pST6kqtV7dxoI9w1Zi0l/9TA=="], "oxc-resolver": ["oxc-resolver@11.24.2", "", { "optionalDependencies": { "@oxc-resolver/binding-android-arm-eabi": "11.24.2", "@oxc-resolver/binding-android-arm64": "11.24.2", "@oxc-resolver/binding-darwin-arm64": "11.24.2", "@oxc-resolver/binding-darwin-x64": "11.24.2", "@oxc-resolver/binding-freebsd-x64": "11.24.2", "@oxc-resolver/binding-linux-arm-gnueabihf": "11.24.2", "@oxc-resolver/binding-linux-arm-musleabihf": "11.24.2", "@oxc-resolver/binding-linux-arm64-gnu": "11.24.2", "@oxc-resolver/binding-linux-arm64-musl": "11.24.2", "@oxc-resolver/binding-linux-ppc64-gnu": "11.24.2", "@oxc-resolver/binding-linux-riscv64-gnu": "11.24.2", "@oxc-resolver/binding-linux-riscv64-musl": "11.24.2", "@oxc-resolver/binding-linux-s390x-gnu": "11.24.2", "@oxc-resolver/binding-linux-x64-gnu": "11.24.2", "@oxc-resolver/binding-linux-x64-musl": "11.24.2", "@oxc-resolver/binding-openharmony-arm64": "11.24.2", "@oxc-resolver/binding-wasm32-wasi": "11.24.2", "@oxc-resolver/binding-win32-arm64-msvc": "11.24.2", "@oxc-resolver/binding-win32-x64-msvc": "11.24.2" } }, "sha512-FY91FiDBj7ls5MsFS9jN3tjz2o0/zsdSsymlakySaBwVJZorHhkWyICLZMKxlu1R9vYo+sd3z1jwb4J8x7bNDw=="], - "oxlint": ["oxlint@1.76.0", "", { "optionalDependencies": { "@oxlint/binding-android-arm-eabi": "1.76.0", "@oxlint/binding-android-arm64": "1.76.0", "@oxlint/binding-darwin-arm64": "1.76.0", "@oxlint/binding-darwin-x64": "1.76.0", "@oxlint/binding-freebsd-x64": "1.76.0", "@oxlint/binding-linux-arm-gnueabihf": "1.76.0", "@oxlint/binding-linux-arm-musleabihf": "1.76.0", "@oxlint/binding-linux-arm64-gnu": "1.76.0", "@oxlint/binding-linux-arm64-musl": "1.76.0", "@oxlint/binding-linux-ppc64-gnu": "1.76.0", "@oxlint/binding-linux-riscv64-gnu": "1.76.0", "@oxlint/binding-linux-riscv64-musl": "1.76.0", "@oxlint/binding-linux-s390x-gnu": "1.76.0", "@oxlint/binding-linux-x64-gnu": "1.76.0", "@oxlint/binding-linux-x64-musl": "1.76.0", "@oxlint/binding-openharmony-arm64": "1.76.0", "@oxlint/binding-win32-arm64-msvc": "1.76.0", "@oxlint/binding-win32-ia32-msvc": "1.76.0", "@oxlint/binding-win32-x64-msvc": "1.76.0" }, "peerDependencies": { "oxlint-tsgolint": ">=7.0.2001", "vite-plus": "*" }, "optionalPeers": ["oxlint-tsgolint", "vite-plus"], "bin": { "oxlint": "bin/oxlint" } }, "sha512-6QoFioEU4fNdiUx/2Eo6TRd6NG7H7njnRCz8rhB66cZmMHDTqcm1Rjvl8Wry+ZTQMBAmyb4Mlf62Mk5X+eHSOw=="], + "oxlint": ["oxlint@1.77.0", "", { "optionalDependencies": { "@oxlint/binding-android-arm-eabi": "1.77.0", "@oxlint/binding-android-arm64": "1.77.0", "@oxlint/binding-darwin-arm64": "1.77.0", "@oxlint/binding-darwin-x64": "1.77.0", "@oxlint/binding-freebsd-x64": "1.77.0", "@oxlint/binding-linux-arm-gnueabihf": "1.77.0", "@oxlint/binding-linux-arm-musleabihf": "1.77.0", "@oxlint/binding-linux-arm64-gnu": "1.77.0", "@oxlint/binding-linux-arm64-musl": "1.77.0", "@oxlint/binding-linux-ppc64-gnu": "1.77.0", "@oxlint/binding-linux-riscv64-gnu": "1.77.0", "@oxlint/binding-linux-riscv64-musl": "1.77.0", "@oxlint/binding-linux-s390x-gnu": "1.77.0", "@oxlint/binding-linux-x64-gnu": "1.77.0", "@oxlint/binding-linux-x64-musl": "1.77.0", "@oxlint/binding-openharmony-arm64": "1.77.0", "@oxlint/binding-win32-arm64-msvc": "1.77.0", "@oxlint/binding-win32-ia32-msvc": "1.77.0", "@oxlint/binding-win32-x64-msvc": "1.77.0" }, "peerDependencies": { "oxlint-tsgolint": ">=7.0.2001", "vite-plus": "*" }, "optionalPeers": ["oxlint-tsgolint", "vite-plus"], "bin": { "oxlint": "bin/oxlint" } }, "sha512-qnGh8XJHaQ0dprrDXNQZgS0FgjI6v+V3+X8DwmaV++5Aamy6jGKfDdQ1TUvhUxtmKFAbEf4/WeO5QZX+5WSngg=="], - "oxlint-plugin-react-doctor": ["oxlint-plugin-react-doctor@0.9.6", "", { "dependencies": { "@shaderfrog/glsl-parser": "^7.0.1", "@typescript-eslint/types": "^8.59.3", "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", "lightningcss": "^1.33.0", "oxc-parser": "^0.142.0" } }, "sha512-kk39ffbDFaL0UfAx7ndltBsxal1hRkr+qYYsSv57PsD+9DnjcV0Y0jZ6NYMhBmHuvPC7so2CN6dNml+fOGrpxw=="], + "oxlint-plugin-react-doctor": ["oxlint-plugin-react-doctor@0.9.12", "", { "dependencies": { "@shaderfrog/glsl-parser": "^7.0.1", "@typescript-eslint/types": "^8.59.3", "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", "lightningcss": "^1.33.0", "oxc-parser": "^0.143.0" } }, "sha512-BplcCUU/tGByFGgY1YIax6evUmjk0K8zOGGrdPs3A3dqamrzjUvVuJdYpM1Ftyt/B5fFx6+VwRrWi3YZbIz4VQ=="], "p-limit": ["p-limit@3.1.0", "", { "dependencies": { "yocto-queue": "^0.1.0" } }, "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ=="], @@ -1386,11 +1384,11 @@ "react-day-picker": ["react-day-picker@10.0.1", "", { "dependencies": { "@date-fns/tz": "^1.4.1", "date-fns": "^4.1.0" }, "peerDependencies": { "@types/react": ">=16.8.0", "react": ">=16.8.0" }, "optionalPeers": ["@types/react"] }, "sha512-eNh6BlwcYInWaJtRv18mXQ06Ys/H6rdTZAnTaSdOYJuTpwP1JMCHNd1FDRadA+gbeinq+psdULN5Xnowy9mV8w=="], - "react-doctor": ["react-doctor@0.9.6", "", { "dependencies": { "@astrojs/compiler": "^4.0.0", "@babel/code-frame": "^7.29.0", "@sentry/node": "^10.54.0", "agent-install": "0.0.5", "conf": "^15.1.0", "confbox": "^0.2.4", "deslop-js": "0.9.6", "eslint-plugin-react-hooks": "^7.1.1", "figures": "^6.1.0", "jiti": "^2.7.0", "magicast": "^0.5.3", "oxc-resolver": "^11.24.2", "oxlint": ">=1.76.0 <1.77.0", "oxlint-plugin-react-doctor": "0.9.6", "prompts": "^2.4.2", "typescript": ">=5.0.4 <6", "vscode-languageserver": "^9.0.1", "vscode-languageserver-textdocument": "^1.0.12", "vscode-uri": "^3.1.0", "yaml": "^2.9.0", "yoga-layout": "~3.2.1" }, "bin": { "react-doctor": "bin/react-doctor.js" } }, "sha512-X3ZLL6UQfzqIyY1HDKEgUOnoxKreEfy9KphiV2aYhiotgCsR81LBl92XgyZUbZeRgpQuwbIDOiyxQnRRkcX42A=="], + "react-doctor": ["react-doctor@0.9.12", "", { "dependencies": { "@astrojs/compiler": "^4.0.0", "@sentry/node": "^10.54.0", "agent-install": "0.0.5", "conf": "^15.1.0", "confbox": "^0.2.4", "deslop-js": "0.9.12", "eslint-plugin-react-hooks": "^7.1.1", "jiti": "^2.7.0", "magicast": "^0.5.3", "oxc-resolver": "^11.24.2", "oxlint": ">=1.77.0 <1.78.0", "oxlint-plugin-react-doctor": "0.9.12", "prompts": "^2.4.2", "typescript": ">=5.0.4 <6", "vscode-languageserver": "^9.0.1", "vscode-languageserver-textdocument": "^1.0.12", "vscode-uri": "^3.1.0", "yaml": "^2.9.0", "yoga-layout": "~3.2.1" }, "bin": { "react-doctor": "bin/react-doctor.js" } }, "sha512-H7RNg13RYKwpQvi3+O3IkSj8pAD1pGTxSetxu7aOwXX3wmF+BydCLDiWxkPT9Pq7bIMHjuJ0taSZLsxEjlNjcA=="], "react-dom": ["react-dom@19.2.8", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.8" } }, "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ=="], - "react-hook-form": ["react-hook-form@7.84.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17 || ^18 || ^19" } }, "sha512-+hWvQP6GLco56mDwrbU4XnHix8t1z90ltZsDIrREl+jnQFQxYLX8oAzqe/Xn8nHpmoXTY5M6oEXrAhbP1qevNQ=="], + "react-hook-form": ["react-hook-form@7.85.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17 || ^18 || ^19" } }, "sha512-U2MTriFXnclmV4rOE20p2DcRFv5WEg3FIcBFOKcOLFHDVvGIMPvLTkTWefUsonmlaVy23khVDxDWym6uJVGOzw=="], "react-i18next": ["react-i18next@17.0.11", "", { "dependencies": { "@babel/runtime": "^7.29.2", "html-parse-stringify": "^4.0.1", "use-sync-external-store": "^1.6.0" }, "peerDependencies": { "i18next": ">= 26.2.0", "react": ">= 16.8.0", "typescript": "^5 || ^6 || ^7" }, "optionalPeers": ["typescript"] }, "sha512-cDtkXgxjuFTWUH6V+aQn1Ve5vDiUztCNPWW5GtSHDccsgRXO1nE6QFWCEmc1KAutrb3OUv87wFShJL5RhUwPXg=="], @@ -1458,7 +1456,7 @@ "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="], - "shadcn": ["shadcn@4.16.2", "", { "dependencies": { "@babel/core": "^7.28.0", "@babel/parser": "^7.28.0", "@babel/plugin-transform-typescript": "^7.28.0", "@babel/preset-typescript": "^7.27.1", "@dotenvx/dotenvx": "^1.48.4", "@modelcontextprotocol/sdk": "^1.26.0", "@types/validate-npm-package-name": "^4.0.2", "browserslist": "^4.26.2", "commander": "^14.0.0", "cosmiconfig": "^9.0.0", "dedent": "^1.6.0", "deepmerge": "^4.3.1", "diff": "^8.0.2", "execa": "^9.6.0", "fast-glob": "^3.3.3", "fs-extra": "^11.3.1", "fuzzysort": "^3.1.0", "kleur": "^4.1.5", "open": "^11.0.0", "ora": "^8.2.0", "postcss": "^8.5.6", "postcss-selector-parser": "^7.1.0", "prompts": "^2.4.2", "recast": "^0.23.11", "stringify-object": "^5.0.0", "tailwind-merge": "^3.0.1", "ts-morph": "^26.0.0", "tsconfig-paths": "^4.2.0", "undici": "^7.27.2", "validate-npm-package-name": "^7.0.1", "zod": "^3.24.1", "zod-to-json-schema": "^3.24.6" }, "bin": { "shadcn": "dist/index.js" } }, "sha512-M1AvZKFWcCzWRDoyApIqJMSLIpY8Ev4uBGuiPLSFmiTbixXhPmzotSTvLzFmBrfoIxG9aIg2dZOETblEaXGUnQ=="], + "shadcn": ["shadcn@4.18.0", "", { "dependencies": { "@babel/core": "^7.28.0", "@babel/parser": "^7.28.0", "@babel/plugin-transform-typescript": "^7.28.0", "@babel/preset-typescript": "^7.27.1", "@dotenvx/dotenvx": "^1.48.4", "@modelcontextprotocol/sdk": "^1.26.0", "@types/validate-npm-package-name": "^4.0.2", "browserslist": "^4.26.2", "commander": "^14.0.0", "cosmiconfig": "^9.0.0", "dedent": "^1.6.0", "deepmerge": "^4.3.1", "diff": "^8.0.2", "execa": "^9.6.0", "fast-glob": "^3.3.3", "fs-extra": "^11.3.1", "fuzzysort": "^3.1.0", "kleur": "^4.1.5", "open": "^11.0.0", "ora": "^8.2.0", "postcss": "^8.5.6", "postcss-selector-parser": "^7.1.0", "prompts": "^2.4.2", "recast": "^0.23.11", "socks": "^2.8.8", "stringify-object": "^5.0.0", "tailwind-merge": "^3.0.1", "ts-morph": "^26.0.0", "tsconfig-paths": "^4.2.0", "undici": "^7.27.2", "validate-npm-package-name": "^7.0.1", "zod": "^3.24.1", "zod-to-json-schema": "^3.24.6" }, "bin": { "shadcn": "dist/index.js" } }, "sha512-tUFZgkYmfVNQVm3xX7lhSzOvDsp+O14ac5dwgXIr5mIsr79ISueb/Mu+ZtWMz0DH6v77u4eYyvbQ9TTMpSn3aw=="], "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], @@ -1478,7 +1476,11 @@ "sisteransi": ["sisteransi@1.0.5", "", {}, "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="], - "sonner": ["sonner@2.0.7", "", { "peerDependencies": { "react": "^18.0.0 || ^19.0.0 || ^19.0.0-rc", "react-dom": "^18.0.0 || ^19.0.0 || ^19.0.0-rc" } }, "sha512-W6ZN4p58k8aDKA4XPcx2hpIQXBRAgyiWVkYhT7CvK6D3iAu7xjvVyhQHg2/iaKJZ1XVJ4r7XuwGL+WGEK37i9w=="], + "smart-buffer": ["smart-buffer@4.2.0", "", {}, "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg=="], + + "socks": ["socks@2.8.9", "", { "dependencies": { "ip-address": "^10.1.1", "smart-buffer": "^4.2.0" } }, "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw=="], + + "sonner": ["sonner@2.0.8", "", { "peerDependencies": { "@types/react": "^18.0.0 || ^19.0.0", "react": "^18.0.0 || ^19.0.0 || ^19.0.0-rc", "react-dom": "^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-UM/ByIoFra8yzV75n1o0Puu0bw5U/9UNnDacrJNspekBewIfsQ3D6ez1nvlWpt7aTsO6rujQtifBpycwIivqlg=="], "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], @@ -1562,7 +1564,7 @@ "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], - "typescript-eslint": ["typescript-eslint@8.66.0", "", { "dependencies": { "@typescript-eslint/eslint-plugin": "8.66.0", "@typescript-eslint/parser": "8.66.0", "@typescript-eslint/typescript-estree": "8.66.0", "@typescript-eslint/utils": "8.66.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-QlEbBPz/RuJ1XUHj29nm3t0F/O/cSlEnntozqPOYHnnTGAXFamnMBu5i9Vn6vhUPHGAjR+Vl+5J8vPN/BMUrJw=="], + "typescript-eslint": ["typescript-eslint@8.67.0", "", { "dependencies": { "@typescript-eslint/eslint-plugin": "8.67.0", "@typescript-eslint/parser": "8.67.0", "@typescript-eslint/typescript-estree": "8.67.0", "@typescript-eslint/utils": "8.67.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-S2udFs8tCKEKffuJ4TB1idGUZiXdCPGi3IPBGWXarbLQ5UPXORV8QEVzJ4gCRduURMb5EkpNCdjbk0eDIuI8Yg=="], "uint8array-extras": ["uint8array-extras@1.5.0", "", {}, "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A=="], @@ -1660,7 +1662,7 @@ "zod-validation-error": ["zod-validation-error@4.0.2", "", { "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-Q6/nZLe6jxuU80qb/4uJ4t5v2VEZ44lzQjPDhYJNztRQ4wyWc6VF3D3Kb/fAuPetZQnhS3hnajCf9CsWesghLQ=="], - "zustand": ["zustand@5.0.14", "", { "peerDependencies": { "@types/react": ">=18.0.0", "immer": ">=9.0.6", "react": ">=18.0.0", "use-sync-external-store": ">=1.2.0" }, "optionalPeers": ["@types/react", "immer", "react", "use-sync-external-store"] }, "sha512-/8tAspM5LMPr28b3fwLYrtdj77ECpfZviaP75CMTnwO8ISyaE4GDIG/9rDDYq/cH9D2Xw2A2RXglLInmVBQB/g=="], + "zustand": ["zustand@5.0.15", "", { "peerDependencies": { "@types/react": ">=18.0.0", "immer": ">=9.0.6", "react": ">=18.0.0", "use-sync-external-store": ">=1.2.0" }, "optionalPeers": ["@types/react", "immer", "react", "use-sync-external-store"] }, "sha512-MpSEjRiBkA9crSYeOUH32rJC7SVqAbm0Fqcqge/bUi2PPoLcBWKOsG+C8mevmpr8TwXHBVkChbbJiyvkE+i/3A=="], "@babel/code-frame/js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], @@ -1714,21 +1716,21 @@ "@typescript-eslint/eslint-plugin/ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="], - "@typescript-eslint/parser/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/parser/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], - "@typescript-eslint/project-service/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/project-service/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], - "@typescript-eslint/scope-manager/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/scope-manager/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], - "@typescript-eslint/type-utils/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/type-utils/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], - "@typescript-eslint/typescript-estree/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/typescript-estree/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], "@typescript-eslint/typescript-estree/semver": ["semver@7.7.4", "", { "bin": "bin/semver.js" }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], - "@typescript-eslint/utils/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/utils/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], - "@typescript-eslint/visitor-keys/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/visitor-keys/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], "ajv-formats/ajv": ["ajv@8.17.1", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g=="], @@ -1746,6 +1748,8 @@ "express/cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="], + "express-rate-limit/ip-address": ["ip-address@10.0.1", "", {}, "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA=="], + "fast-glob/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], "html-encoding-sniffer/@exodus/bytes": ["@exodus/bytes@1.14.0", "", { "peerDependencies": { "@noble/hashes": "^1.8.0 || ^2.0.0" } }, "sha512-YiY1OmY6Qhkvmly8vZiD8wZRpW/npGZNg+0Sk8mstxirRHCg6lolHt5tSODCfuNPE/fBsAqRwDJE417x7jDDHA=="], @@ -1778,6 +1782,8 @@ "restore-cursor/onetime": ["onetime@7.0.0", "", { "dependencies": { "mimic-function": "^5.0.0" } }, "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ=="], + "rolldown/@oxc-project/types": ["@oxc-project/types@0.142.0", "", {}, "sha512-7W+2q5AKQVU36fkaryontrHn3YDt1RyUYXatw9i5H8ocYe2sPKSFB6eS8WNPeRKiN1qAWWZUPm7gwFzJGrccqQ=="], + "router/path-to-regexp": ["path-to-regexp@8.3.0", "", {}, "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA=="], "shadcn/@babel/parser": ["@babel/parser@7.29.7", "", { "dependencies": { "@babel/types": "^7.29.7" }, "bin": "./bin/babel-parser.js" }, "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg=="], diff --git a/frontend/package.json b/frontend/package.json index ae9bc40d0f..bb4081cfe4 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -19,7 +19,7 @@ "test:browser-smoke": "playwright test --config browser-smoke/playwright.config.ts" }, "dependencies": { - "@hookform/resolvers": "^5.7.1", + "@hookform/resolvers": "^5.8.0", "@tailwindcss/vite": "^4.3.3", "@tanstack/react-query": "^5.101.4", "class-variance-authority": "^0.7.1", @@ -28,26 +28,26 @@ "i18next": "^26.3.6", "i18next-browser-languagedetector": "^8.2.1", "input-otp": "^1.4.2", - "lucide-react": "^1.30.0", + "lucide-react": "^1.31.0", "radix-ui": "^1.6.7", "react": "^19.2.8", "react-day-picker": "^10.0.1", "react-dom": "^19.2.8", - "react-hook-form": "^7.84.0", + "react-hook-form": "^7.85.0", "react-i18next": "^17.0.11", "react-router-dom": "^7.18.2", "recharts": "^3.10.1", - "sonner": "^2.0.7", + "sonner": "^2.0.8", "tailwind-merge": "^3.6.0", "tailwindcss": "^4.3.3", "zod": "^4.4.3", - "zustand": "^5.0.14" + "zustand": "^5.0.15" }, "devDependencies": { "@eslint/js": "^10.0.1", - "@testing-library/jest-dom": "^7.0.0", + "@testing-library/jest-dom": "^7.0.1", "@testing-library/react": "^16.3.2", - "@testing-library/user-event": "^14.6.3", + "@testing-library/user-event": "^14.6.4", "@types/node": "^26.2.0", "@types/react": "^19.2.18", "@types/react-dom": "^19.2.4", @@ -55,16 +55,16 @@ "@vitest/coverage-v8": "^4.1.10", "eslint": "^10.8.1", "eslint-plugin-react-hooks": "^7.1.1", - "eslint-plugin-react-refresh": "^0.5.3", - "globals": "^17.9.0", + "eslint-plugin-react-refresh": "^0.5.4", + "globals": "^17.11.0", "jsdom": "^30.0.1", "msw": "^2.15.0", - "react-doctor": "^0.9.6", - "shadcn": "^4.16.2", + "react-doctor": "^0.9.12", + "shadcn": "^4.18.0", "tw-animate-css": "^1.4.0", "@typescript/native": "npm:typescript@~7.0.2", "typescript": "npm:typescript@~6.0.3", - "typescript-eslint": "^8.66.0", + "typescript-eslint": "^8.67.0", "@playwright/test": "^1.62.1", "vite": "^8.2.1", "vitest": "^4.1.10" diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 53ae46bb49..fbc03db24b 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -10,6 +10,7 @@ import { Toaster } from "@/components/ui/sonner"; import { TooltipProvider } from "@/components/ui/tooltip"; import { AuthGate } from "@/features/auth/components/auth-gate"; import { useAuthStore } from "@/features/auth/hooks/use-auth"; +import { TelemetryConsentDialog } from "@/features/settings/components/telemetry-consent-dialog"; import { useTimeFormatStore } from "@/hooks/use-time-format"; // Route-level code splitting: only the visited page's chunk loads, instead @@ -61,6 +62,7 @@ function AppLayout() { + ); } @@ -79,7 +81,7 @@ export default function App() { } /> } /> } /> - } /> + } /> diff --git a/frontend/src/__integration__/apis-page-flow.test.tsx b/frontend/src/__integration__/apis-page-flow.test.tsx index 40ea6f0335..ede134fcd7 100644 --- a/frontend/src/__integration__/apis-page-flow.test.tsx +++ b/frontend/src/__integration__/apis-page-flow.test.tsx @@ -4,7 +4,7 @@ import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it } from "vitest"; import App from "@/App"; -import { createApiKey, createApiKeyUsage7Day } from "@/test/mocks/factories"; +import { createApiKey, createApiKeyCreateResponse, createApiKeyUsage7Day } from "@/test/mocks/factories"; import { server } from "@/test/mocks/server"; import { renderWithProviders } from "@/test/utils"; @@ -54,6 +54,35 @@ describe("apis page integration", () => { expect(await screen.findByRole("button", { name: /Created from APIs page/i })).toBeInTheDocument(); }); + it("creates a key with a selectable reasoning-effort policy", async () => { + const user = userEvent.setup(); + let requestBody: unknown; + server.use( + http.post("/api/api-keys/", async ({ request }) => { + requestBody = await request.json(); + return HttpResponse.json(createApiKeyCreateResponse({ name: "Selectable effort key" })); + }), + ); + renderWithProviders(); + + await user.click(await screen.findByRole("button", { name: "Create API Key" })); + const createDialog = await screen.findByRole("dialog", { name: "Create API key" }); + await user.type(within(createDialog).getByLabelText("Name"), "Selectable effort key"); + await user.click(within(createDialog).getByRole("button", { name: "Allowed efforts: All efforts" })); + await user.click(screen.getByRole("menuitemcheckbox", { name: /^Low$/ })); + await user.keyboard("{Escape}"); + + expect(within(createDialog).getByLabelText("Enforced Effort")).toBeDisabled(); + + await user.click(within(createDialog).getByRole("button", { name: "Create" })); + await waitFor(() => { + expect(requestBody).toMatchObject({ + allowedReasoningEfforts: ["low"], + enforcedReasoningEffort: null, + }); + }); + }); + it("edits, toggles, regenerates, and deletes the selected key", async () => { const user = userEvent.setup(); renderWithProviders(); @@ -119,7 +148,7 @@ describe("apis page integration", () => { renderWithProviders(); expect(await screen.findByRole("heading", { name: "APIs" })).toBeInTheDocument(); - expect(await screen.findByText("No matching API keys")).toBeInTheDocument(); + expect(await screen.findByText("No API keys yet")).toBeInTheDocument(); expect(screen.getByText("Select an API key")).toBeInTheDocument(); }); diff --git a/frontend/src/__integration__/firewall-flow.test.tsx b/frontend/src/__integration__/firewall-flow.test.tsx index 999226688e..6aa5dd73ad 100644 --- a/frontend/src/__integration__/firewall-flow.test.tsx +++ b/frontend/src/__integration__/firewall-flow.test.tsx @@ -55,6 +55,7 @@ describe("firewall flow integration", () => { // Scope queries to the firewall section const firewallSection = firewallHeading.closest("section")!; + expect(firewallSection).toHaveClass("scroll-mt-16"); const fw = within(firewallSection); await user.type(fw.getByPlaceholderText("127.0.0.1 or 2001:db8::1"), "127.0.0.1"); @@ -78,6 +79,9 @@ describe("firewall flow integration", () => { expect(await screen.findByRole("heading", { name: "Settings" })).toBeInTheDocument(); expect(window.location.pathname).toBe("/settings"); - expect(await screen.findByRole("button", { name: "Show advanced settings" })).toBeInTheDocument(); + expect(window.location.search).toBe("?advanced=1"); + expect(window.location.hash).toBe("#firewall"); + expect(await screen.findByRole("heading", { name: "Firewall" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Hide advanced settings" })).toBeInTheDocument(); }); }); diff --git a/frontend/src/__integration__/reports-date-range-flow.test.tsx b/frontend/src/__integration__/reports-date-range-flow.test.tsx index 0b5695d24f..bc30e5b8f5 100644 --- a/frontend/src/__integration__/reports-date-range-flow.test.tsx +++ b/frontend/src/__integration__/reports-date-range-flow.test.tsx @@ -1,4 +1,4 @@ -import { fireEvent, screen, waitFor } from "@testing-library/react"; +import { fireEvent, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { HttpResponse, http } from "msw"; import { describe, expect, it } from "vitest"; @@ -14,8 +14,11 @@ const EMPTY_REPORT: ReportsResponse = { totalCostUsd: 0, totalInputTokens: 0, totalOutputTokens: 0, + totalReasoningTokens: 0, + reasoningUsageKnownRequests: 0, totalCachedTokens: 0, totalRequests: 0, + totalCancelled: 0, totalErrors: 0, totalConversations: 0, activeAccounts: 0, @@ -218,7 +221,13 @@ describe("reports date-range flow integration", () => { expect( await screen.findByText("Start date must be on or before end date."), ).toBeInTheDocument(); - const retryButton = await screen.findByRole("button", { name: "Retry" }); + const accountErrorText = await screen.findByText( + /Failed to load account options:/i, + ); + const accountErrorContainer = accountErrorText.parentElement!.parentElement!; + const retryButton = within(accountErrorContainer).getByRole("button", { + name: "Retry", + }); const accountRequestsBeforeRetry = accountRequests; const reportsRequestsBeforeRetry = reportsRequests.length; diff --git a/frontend/src/__integration__/telemetry-consent-flow.test.tsx b/frontend/src/__integration__/telemetry-consent-flow.test.tsx new file mode 100644 index 0000000000..8ee9677c3a --- /dev/null +++ b/frontend/src/__integration__/telemetry-consent-flow.test.tsx @@ -0,0 +1,50 @@ +import { screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { HttpResponse, http } from "msw"; +import { describe, expect, it } from "vitest"; + +import App from "@/App"; +import { createTelemetryConsent } from "@/test/mocks/factories"; +import { server } from "@/test/mocks/server"; +import { renderWithProviders } from "@/test/utils"; + +describe("telemetry consent flow integration", () => { + it("shows the one-time consent dialog on dashboard entry and persists the decision", async () => { + const user = userEvent.setup({ delay: null }); + let putBody: unknown = null; + let consent = createTelemetryConsent({ state: "undecided", source: "default", active: true }); + server.use( + http.get("/api/settings/telemetry", () => HttpResponse.json(consent)), + http.put("/api/settings/telemetry", async ({ request }) => { + putBody = await request.json(); + consent = createTelemetryConsent({ state: "disabled", source: "persisted", active: false }); + return HttpResponse.json(consent); + }), + ); + + window.history.pushState({}, "", "/dashboard"); + renderWithProviders(); + + const dialog = await screen.findByRole("dialog", { name: "Anonymous telemetry" }); + // The dialog renders the full transmitted envelope, not just the metrics. + expect(dialog).toHaveTextContent('"instance_id": "00000000-0000-4000-8000-000000000000"'); + expect(dialog).toHaveTextContent('"timestamp": "2026-08-06T00:00:00Z"'); + expect(dialog).toHaveTextContent('"schema_version": 1'); + + await user.click(screen.getByRole("button", { name: "Disable telemetry" })); + + await waitFor(() => expect(putBody).toEqual({ enabled: false })); + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + }); + + it("does not show the consent dialog when consent is already decided", async () => { + window.history.pushState({}, "", "/dashboard"); + const { queryClient } = renderWithProviders(); + + // Default mock state is enabled/persisted. + await waitFor(() => + expect(queryClient.getQueryState(["settings", "telemetry"])?.status).toBe("success"), + ); + expect(screen.queryByRole("dialog", { name: "Anonymous telemetry" })).not.toBeInTheDocument(); + }); +}); diff --git a/frontend/src/components/copy-button.test.tsx b/frontend/src/components/copy-button.test.tsx index 6332d26a19..9043615e9b 100644 --- a/frontend/src/components/copy-button.test.tsx +++ b/frontend/src/components/copy-button.test.tsx @@ -53,6 +53,59 @@ describe("CopyButton", () => { expect(screen.getByRole("button", { name: "Copy" })).toBeInTheDocument(); }); + it("clears the feedback timer when unmounted", async () => { + const writeText = vi.fn().mockResolvedValue(undefined); + Object.defineProperty(window, "isSecureContext", { + configurable: true, + value: true, + }); + Object.defineProperty(navigator, "clipboard", { + configurable: true, + value: { writeText }, + }); + + const { unmount } = render(); + await act(async () => { + fireEvent.click(screen.getByRole("button", { name: "Copy" })); + await Promise.resolve(); + }); + + expect(vi.getTimerCount()).toBe(1); + unmount(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("ignores clipboard completion after unmount", async () => { + let resolveWrite!: () => void; + const writeText = vi.fn( + () => + new Promise((resolve) => { + resolveWrite = resolve; + }), + ); + Object.defineProperty(window, "isSecureContext", { + configurable: true, + value: true, + }); + Object.defineProperty(navigator, "clipboard", { + configurable: true, + value: { writeText }, + }); + + const { unmount } = render(); + fireEvent.click(screen.getByRole("button", { name: "Copy" })); + expect(writeText).toHaveBeenCalledWith("secret-value"); + + unmount(); + await act(async () => { + resolveWrite(); + await Promise.resolve(); + }); + + expect(toastSuccess).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }); + it("shows error toast when clipboard write fails", async () => { const writeText = vi.fn().mockRejectedValue(new Error("clipboard blocked")); Object.defineProperty(window, "isSecureContext", { diff --git a/frontend/src/components/copy-button.tsx b/frontend/src/components/copy-button.tsx index 3d0469406a..3b458da589 100644 --- a/frontend/src/components/copy-button.tsx +++ b/frontend/src/components/copy-button.tsx @@ -1,5 +1,5 @@ import { Check, Copy } from "lucide-react"; -import { useState, type MouseEvent } from "react"; +import { useEffect, useRef, useState, type MouseEvent } from "react"; import { useTranslation } from "react-i18next"; import { toast } from "sonner"; @@ -15,8 +15,20 @@ export type CopyButtonProps = { export function CopyButton({ value, label, iconOnly = false }: CopyButtonProps) { const { t } = useTranslation(); const [copied, setCopied] = useState(false); + const mountedRef = useRef(false); + const resetTimerRef = useRef | null>(null); const labelText = label ?? t("components.copyButton.copy"); + useEffect(() => { + mountedRef.current = true; + return () => { + mountedRef.current = false; + if (resetTimerRef.current !== null) { + clearTimeout(resetTimerRef.current); + } + }; + }, []); + const handleCopy = async (event: MouseEvent) => { const trigger = event.currentTarget; const dialogContainer = trigger.closest("[role='dialog']"); @@ -25,16 +37,27 @@ export function CopyButton({ value, label, iconOnly = false }: CopyButtonProps) const copiedToClipboard = await copyToClipboard(value, { container: dialogContainer instanceof HTMLElement ? dialogContainer : undefined, }); + if (!mountedRef.current) { + return; + } if (copiedToClipboard) { setCopied(true); toast.success(t("components.copyButton.toasts.copied")); - setTimeout(() => setCopied(false), 1200); + if (resetTimerRef.current !== null) { + clearTimeout(resetTimerRef.current); + } + resetTimerRef.current = setTimeout(() => { + resetTimerRef.current = null; + setCopied(false); + }, 1200); return; } toast.error(t("components.copyButton.toasts.failed")); } catch { - toast.error(t("components.copyButton.toasts.failed")); + if (mountedRef.current) { + toast.error(t("components.copyButton.toasts.failed")); + } } }; const copiedLabel = t("components.copyButton.copied"); diff --git a/frontend/src/components/empty-state.tsx b/frontend/src/components/empty-state.tsx index 332c61ebfd..2fbf97f5ca 100644 --- a/frontend/src/components/empty-state.tsx +++ b/frontend/src/components/empty-state.tsx @@ -1,12 +1,14 @@ import type { LucideIcon } from "lucide-react"; +import type { ReactNode } from "react"; export type EmptyStateProps = { icon: LucideIcon; title: string; description?: string; + action?: ReactNode; }; -export function EmptyState({ icon: Icon, title, description }: EmptyStateProps) { +export function EmptyState({ icon: Icon, title, description, action }: EmptyStateProps) { return (
@@ -16,6 +18,7 @@ export function EmptyState({ icon: Icon, title, description }: EmptyStateProps)

{title}

{description ?

{description}

: null}
+ {action ?
{action}
: null}
); } diff --git a/frontend/src/components/status-badge.tsx b/frontend/src/components/status-badge.tsx index 7936c5c1fa..2d2b3e1372 100644 --- a/frontend/src/components/status-badge.tsx +++ b/frontend/src/components/status-badge.tsx @@ -16,15 +16,16 @@ const statusClassMap: Record = { export type StatusBadgeProps = { status: StatusValue; + title?: string; }; -export function StatusBadge({ status }: StatusBadgeProps) { +export function StatusBadge({ status, title }: StatusBadgeProps) { const { t } = useTranslation(); const className = statusClassMap[status] ?? statusClassMap.deactivated; const label = t(`common.status.${status}`, { defaultValue: status }); return ( - + {label} diff --git a/frontend/src/features/accounts/components/account-list-item.test.tsx b/frontend/src/features/accounts/components/account-list-item.test.tsx index 91f9cc0a68..ded9bbfd11 100644 --- a/frontend/src/features/accounts/components/account-list-item.test.tsx +++ b/frontend/src/features/accounts/components/account-list-item.test.tsx @@ -248,6 +248,30 @@ describe("AccountListItem", () => { expect(screen.queryByText("99+")).not.toBeInTheDocument(); }); + it("explains that Active is the displayed status, not per-request eligibility", () => { + const account = createAccountSummary({ status: "active" }); + + render(); + + // The hint lives on the focusable row (accessible description for + // keyboard/screen-reader users) and on the badge for pointer hover. + const hints = screen.getAllByTitle(/Active is the account's displayed status/i); + expect(hints.length).toBeGreaterThan(0); + expect(screen.getByRole("button")).toHaveAttribute( + "title", + expect.stringMatching(/Active is the account's displayed status/i), + ); + }); + + it("omits the eligibility hint for non-active statuses", () => { + const account = createAccountSummary({ status: "paused" }); + + render(); + + expect(screen.queryByTitle(/Active is the account's displayed status/i)).not.toBeInTheDocument(); + expect(screen.getByRole("button")).not.toHaveAttribute("title"); + }); + it("hides the reset-credit badge when badge display is disabled", () => { const account = createAccountSummary({ availableResetCredits: 3 }); diff --git a/frontend/src/features/accounts/components/account-list-item.tsx b/frontend/src/features/accounts/components/account-list-item.tsx index 7a794d7b98..0aff88162e 100644 --- a/frontend/src/features/accounts/components/account-list-item.tsx +++ b/frontend/src/features/accounts/components/account-list-item.tsx @@ -80,10 +80,15 @@ export function AccountListItem({ : t("accounts.listItem.noAttempts"); const availableResetCredits = account.availableResetCredits ?? 0; const resetBadgeLabel = availableResetCredits > 99 ? "99+" : String(availableResetCredits); + const statusEligibilityHint = status === "active" ? t("accounts.listItem.statusActiveHint") : undefined; return ( + + + {REASONING_EFFORTS.map((effort) => ( + toggle(effort)} + onSelect={(event) => event.preventDefault()} + > + {t(`common.reasoning.${effort}`)} + + ))} + + + ); +} diff --git a/frontend/src/features/api-keys/schemas.test.ts b/frontend/src/features/api-keys/schemas.test.ts index ec77e0cdd7..10b77f3870 100644 --- a/frontend/src/features/api-keys/schemas.test.ts +++ b/frontend/src/features/api-keys/schemas.test.ts @@ -189,6 +189,33 @@ describe("ApiKeyCreateRequestSchema", () => { expect(parsed.enforcedReasoningEffort).toBe("ultra"); }); + it("accepts Ultrafast service tier in create payload", () => { + const parsed = ApiKeyCreateRequestSchema.parse({ + name: "Ultrafast key", + enforcedServiceTier: "ultrafast", + }); + + expect(parsed.enforcedServiceTier).toBe("ultrafast"); + }); + + it("accepts a non-empty allowed reasoning effort list", () => { + const parsed = ApiKeyCreateRequestSchema.parse({ + name: "Selectable reasoning key", + allowedReasoningEfforts: ["low", "high", "xhigh"], + }); + + expect(parsed.allowedReasoningEfforts).toEqual(["low", "high", "xhigh"]); + }); + + it("rejects an empty allowed reasoning effort list", () => { + const result = ApiKeyCreateRequestSchema.safeParse({ + name: "Empty reasoning key", + allowedReasoningEfforts: [], + }); + + expect(result.success).toBe(false); + }); + it("rejects invalid traffic class in create payload", () => { const result = ApiKeyCreateRequestSchema.safeParse({ name: "Bad Key", @@ -259,6 +286,14 @@ describe("ApiKeyUpdateRequestSchema", () => { expect(parsed.trafficClass).toBe("opportunistic"); }); + + it("accepts Ultrafast service tier in update payload", () => { + const parsed = ApiKeyUpdateRequestSchema.parse({ + enforcedServiceTier: "ultrafast", + }); + + expect(parsed.enforcedServiceTier).toBe("ultrafast"); + }); }); describe("LimitRuleCreateSchema", () => { diff --git a/frontend/src/features/api-keys/schemas.ts b/frontend/src/features/api-keys/schemas.ts index 5f69b5e7d7..2bbd8b91b0 100644 --- a/frontend/src/features/api-keys/schemas.ts +++ b/frontend/src/features/api-keys/schemas.ts @@ -30,7 +30,7 @@ const ApiKeyUsageSummarySchema = z.object({ totalCostUsd: z.number().nonnegative().default(0), }); -const SERVICE_TIERS = ["auto", "default", "priority", "flex"] as const; +const SERVICE_TIERS = ["auto", "default", "priority", "flex", "ultrafast"] as const; export type ServiceTierType = (typeof SERVICE_TIERS)[number]; export const TRAFFIC_CLASSES = ["foreground", "opportunistic"] as const; @@ -48,6 +48,7 @@ export const ApiKeySchema = z.object({ allowedModels: z.array(z.string()).nullable(), applyToCodexModel: z.boolean().default(false), enforcedModel: z.string().nullable().default(null), + allowedReasoningEfforts: z.array(z.enum(REASONING_EFFORTS)).nullable().default(null), trafficClass: z .enum(TRAFFIC_CLASSES) .default("foreground"), @@ -88,6 +89,7 @@ export const ApiKeyCreateRequestSchema = z.object({ trafficClass: z.enum(TRAFFIC_CLASSES).optional(), transportPolicyOverride: z.enum(TRANSPORT_POLICY_OVERRIDES).nullable().optional(), enforcedModel: z.string().min(1).nullable().optional(), + allowedReasoningEfforts: z.array(z.enum(REASONING_EFFORTS)).min(1).nullable().optional(), enforcedReasoningEffort: z.enum(ENFORCED_REASONING_EFFORTS).nullable().optional(), enforcedServiceTier: z .enum(SERVICE_TIERS) @@ -112,6 +114,7 @@ export const ApiKeyUpdateRequestSchema = z.object({ trafficClass: z.enum(TRAFFIC_CLASSES).optional(), transportPolicyOverride: z.enum(TRANSPORT_POLICY_OVERRIDES).nullable().optional(), enforcedModel: z.string().min(1).nullable().optional(), + allowedReasoningEfforts: z.array(z.enum(REASONING_EFFORTS)).min(1).nullable().optional(), enforcedReasoningEffort: z.enum(ENFORCED_REASONING_EFFORTS).nullable().optional(), enforcedServiceTier: z .enum(SERVICE_TIERS) diff --git a/frontend/src/features/apis/components/api-key-info.test.tsx b/frontend/src/features/apis/components/api-key-info.test.tsx index a968639f7b..a66e177deb 100644 --- a/frontend/src/features/apis/components/api-key-info.test.tsx +++ b/frontend/src/features/apis/components/api-key-info.test.tsx @@ -13,7 +13,8 @@ describe("ApiKeyInfo", () => { keyPrefix: "sk-special", allowedModels: ["gpt-5.1", "gpt-4o-mini"], enforcedModel: "gpt-5.1", - enforcedReasoningEffort: "high", + enforcedReasoningEffort: null, + allowedReasoningEfforts: ["low", "high", "xhigh"], })} />, ); @@ -23,7 +24,8 @@ describe("ApiKeyInfo", () => { expect(screen.getByText("gpt-5.1, gpt-4o-mini")).toBeInTheDocument(); expect(screen.getByText("Foreground")).toBeInTheDocument(); expect(screen.getByText("Enforced Model")).toBeInTheDocument(); - expect(screen.getByText("Enforced Effort")).toBeInTheDocument(); + expect(screen.getByText("Allowed efforts")).toBeInTheDocument(); + expect(screen.getByText("Low, High, Extra high")).toBeInTheDocument(); }); it("renders opportunistic traffic class when set", () => { diff --git a/frontend/src/features/apis/components/api-key-info.tsx b/frontend/src/features/apis/components/api-key-info.tsx index cfcb848b50..e834cf9958 100644 --- a/frontend/src/features/apis/components/api-key-info.tsx +++ b/frontend/src/features/apis/components/api-key-info.tsx @@ -47,6 +47,9 @@ export function ApiKeyInfo({ const models = apiKey.allowedModels?.join(", ") || t("apiKeys.modelSelect.all"); const enforcedModel = apiKey.enforcedModel || null; const enforcedEffort = apiKey.enforcedReasoningEffort || null; + const allowedEfforts = apiKey.allowedReasoningEfforts + ?.map((effort) => t(`common.reasoning.${effort}`)) + .join(", ") || null; const trafficClass = apiKey.trafficClass === "opportunistic" ? t("common.traffic.opportunistic") : t("common.traffic.foreground"); const usage = allowUsageSummaryFallback ? (usageSummary ?? apiKey.usageSummary) @@ -83,6 +86,12 @@ export function ApiKeyInfo({
{enforcedEffort}
) : null} + {allowedEfforts ? ( +
+
{t("apiKeys.form.allowedReasoningEfforts")}
+
{allowedEfforts}
+
+ ) : null}
{t("apiKeys.table.expiry")}
{ + it("shows first-run empty copy when no API keys exist", () => { + render( + {}} onOpenCreate={() => {}} />, + ); + + expect(screen.getByText("No API keys yet")).toBeInTheDocument(); + expect(screen.getByText("Create an API key to authenticate clients.")).toBeInTheDocument(); + expect(screen.queryByText("Adjust filters")).not.toBeInTheDocument(); + }); + + it("shows filter-empty copy when keys exist but none match", async () => { + const user = userEvent.setup(); + + render( + {}} + onOpenCreate={() => {}} + />, + ); + + await user.type(screen.getByPlaceholderText("Search API keys..."), "not-found"); + + expect(screen.getByText("No matching API keys")).toBeInTheDocument(); + expect(screen.getByText("Adjust filters")).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/features/apis/components/api-list.tsx b/frontend/src/features/apis/components/api-list.tsx index 3731bc2437..3a293f399e 100644 --- a/frontend/src/features/apis/components/api-list.tsx +++ b/frontend/src/features/apis/components/api-list.tsx @@ -93,8 +93,12 @@ export function ApiList({ apiKeys, selectedKeyId, onSelect, onOpenCreate }: ApiL
{filtered.length === 0 ? (
-

{t("apis.list.noMatches")}

-

{t("accounts.list.adjustFilters")}

+

+ {apiKeys.length === 0 ? t("apis.list.emptyTitle") : t("apis.list.noMatches")} +

+

+ {apiKeys.length === 0 ? t("apis.list.emptyDescription") : t("accounts.list.adjustFilters")} +

) : ( filtered.map((apiKey) => ( diff --git a/frontend/src/features/dashboard/components/account-cards.test.tsx b/frontend/src/features/dashboard/components/account-cards.test.tsx index 50be15a461..d95e0d685b 100644 --- a/frontend/src/features/dashboard/components/account-cards.test.tsx +++ b/frontend/src/features/dashboard/components/account-cards.test.tsx @@ -1,4 +1,5 @@ import { render, screen } from "@testing-library/react"; +import { MemoryRouter } from "react-router-dom"; import { describe, expect, it, vi } from "vitest"; import { AccountCards } from "@/features/dashboard/components/account-cards"; @@ -87,4 +88,14 @@ describe("AccountCards", () => { expect(screen.queryByText((_content, el) => el?.tagName === "P" && !!el.textContent?.match(/dup@example\.com .* ID d48f0bfc\.\.\.12b5d5/))).not.toBeInTheDocument(); expect(screen.getByText((_content, el) => el?.tagName === "P" && !!el.textContent?.match(/dup@example\.com .* ID 7f9de2ad\.\.\.a95cee/))).toBeInTheDocument(); }); + + it("links the empty-account state to the Accounts page", () => { + render( + + + , + ); + + expect(screen.getByRole("link", { name: "Add accounts" })).toHaveAttribute("href", "/accounts"); + }); }); diff --git a/frontend/src/features/dashboard/components/account-cards.tsx b/frontend/src/features/dashboard/components/account-cards.tsx index 802f7fac32..a645a28e5a 100644 --- a/frontend/src/features/dashboard/components/account-cards.tsx +++ b/frontend/src/features/dashboard/components/account-cards.tsx @@ -1,7 +1,9 @@ import { Users } from "lucide-react"; import { useTranslation } from "react-i18next"; +import { Link } from "react-router-dom"; import { EmptyState } from "@/components/empty-state"; +import { Button } from "@/components/ui/button"; import { AccountCard, type AccountCardProps } from "@/features/dashboard/components/account-card"; import type { AccountSummary } from "@/features/dashboard/schemas"; @@ -25,6 +27,11 @@ export function AccountCards({ accounts, readOnly = false, onAction }: AccountCa icon={Users} title={t("dashboard.accounts.emptyTitle")} description={t("dashboard.accounts.emptyDescription")} + action={ + + } /> ); } diff --git a/frontend/src/features/dashboard/components/account-list.test.tsx b/frontend/src/features/dashboard/components/account-list.test.tsx index c5dfa2f23f..616d3b377f 100644 --- a/frontend/src/features/dashboard/components/account-list.test.tsx +++ b/frontend/src/features/dashboard/components/account-list.test.tsx @@ -1,5 +1,6 @@ import { act, render, screen, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; +import { MemoryRouter } from "react-router-dom"; import { afterEach, describe, expect, it, vi } from "vitest"; import { AccountList } from "@/features/dashboard/components/account-list"; @@ -349,4 +350,14 @@ describe("AccountList", () => { const resetButton = screen.getByRole("button", { name: "Redeem reset credit for Many Reset Account" }); expect(within(resetButton).getByText("99+")).toBeInTheDocument(); }); + + it("links the empty-account state to the Accounts page", () => { + render( + + + , + ); + + expect(screen.getByRole("link", { name: "Add accounts" })).toHaveAttribute("href", "/accounts"); + }); }); diff --git a/frontend/src/features/dashboard/components/account-list.tsx b/frontend/src/features/dashboard/components/account-list.tsx index dbffa14b1f..88a840d67c 100644 --- a/frontend/src/features/dashboard/components/account-list.tsx +++ b/frontend/src/features/dashboard/components/account-list.tsx @@ -1,6 +1,7 @@ import { ArrowDown, ArrowUp, ArrowUpDown, Clock, ExternalLink, List, Play, RotateCcw, Zap } from "lucide-react"; import { useMemo, useState } from "react"; import { useTranslation } from "react-i18next"; +import { Link } from "react-router-dom"; import { EmptyState } from "@/components/empty-state"; import { StatusBadge } from "@/components/status-badge"; @@ -304,6 +305,11 @@ export function AccountList({ icon={List} title={t("dashboard.accountList.emptyTitle")} description={t("dashboard.accountList.emptyDescription")} + action={ + + } /> ); } diff --git a/frontend/src/features/dashboard/components/dashboard-page.test.tsx b/frontend/src/features/dashboard/components/dashboard-page.test.tsx index 97e50f1ea3..a7a2203b4e 100644 --- a/frontend/src/features/dashboard/components/dashboard-page.test.tsx +++ b/frontend/src/features/dashboard/components/dashboard-page.test.tsx @@ -9,17 +9,26 @@ import { useAuthStore } from "@/features/auth/hooks/use-auth"; import { useDashboard, useDashboardProjections } from "@/features/dashboard/hooks/use-dashboard"; import { useRequestLogs } from "@/features/dashboard/hooks/use-request-logs"; import { useConversations } from "@/features/dashboard/hooks/use-conversations"; +import { REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY } from "@/features/dashboard/hooks/use-request-log-table-preferences"; import { buildDashboardView } from "@/features/dashboard/utils"; -import { useDashboardPreferencesStore } from "@/hooks/use-dashboard-preferences"; import type { AccountListSort } from "@/features/dashboard/components/account-list"; +import type { RecentRequestsTableProps } from "@/features/dashboard/components/recent-requests-table"; +import { useDashboardPreferencesStore } from "@/hooks/use-dashboard-preferences"; import { DashboardPage } from "./dashboard-page"; -const { accountCardsSpy, accountListSpy, accountSummaryLineSpy, conversationsViewSpy } = vi.hoisted(() => ({ +const { + accountCardsSpy, + accountListSpy, + accountSummaryLineSpy, + conversationsViewSpy, + recentRequestsTableSpy, +} = vi.hoisted(() => ({ accountCardsSpy: vi.fn(), accountListSpy: vi.fn(), accountSummaryLineSpy: vi.fn(), conversationsViewSpy: vi.fn(), + recentRequestsTableSpy: vi.fn(), })); vi.mock("@/features/accounts/hooks/use-accounts", () => ({ @@ -31,9 +40,13 @@ vi.mock("@/features/dashboard/hooks/use-dashboard", () => ({ useDashboardProjections: vi.fn(), })); -vi.mock("@/features/dashboard/hooks/use-request-logs", () => ({ - useRequestLogs: vi.fn(), -})); +vi.mock("@/features/dashboard/hooks/use-request-logs", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + useRequestLogs: vi.fn(), + }; +}); vi.mock("@/features/dashboard/hooks/use-conversations", () => ({ useConversations: vi.fn(), @@ -115,7 +128,10 @@ vi.mock("@/features/dashboard/components/filters/request-filters", async () => { }); vi.mock("@/features/dashboard/components/recent-requests-table", () => ({ - RecentRequestsTable: () =>
, + RecentRequestsTable: (props: RecentRequestsTableProps) => { + recentRequestsTableSpy(props); + return
; + }, })); vi.mock("@/features/dashboard/components/stats-grid", () => ({ @@ -162,12 +178,14 @@ describe("DashboardPage", () => { accountListSpy.mockReset(); accountSummaryLineSpy.mockReset(); conversationsViewSpy.mockReset(); + recentRequestsTableSpy.mockReset(); useAccountMutationsMock.mockReset(); useDashboardMock.mockReset(); useDashboardProjectionsMock.mockReset(); useRequestLogsMock.mockReset(); useConversationsMock.mockReset(); buildDashboardViewMock.mockReset(); + window.localStorage.removeItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY); useDashboardPreferencesStore.setState({ accountBurnrateEnabled: true, accountViewMode: "cards", @@ -481,6 +499,49 @@ describe("DashboardPage", () => { expect(useConversationsMock.mock.calls.every(([options]) => options !== undefined && options.enabled === false)).toBe(true); }); + it("customizes and restores the request-log table without a global width control", async () => { + const user = userEvent.setup(); + mockReadyDashboard(); + + renderWithProviders(); + + expect(screen.getByRole("button", { name: "Columns (12)" })).toBeInTheDocument(); + expect(screen.queryByRole("slider")).not.toBeInTheDocument(); + expect(screen.queryByText(/^Width$/)).not.toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Columns (12)" })); + expect(screen.getByText("Visible columns")).toBeInTheDocument(); + await user.click(screen.getByRole("menuitemcheckbox", { name: "Plan" })); + + expect(screen.getByRole("menu", { name: "Columns (11)" })).toBeInTheDocument(); + const selectedProps = recentRequestsTableSpy.mock.lastCall?.[0] as + | RecentRequestsTableProps + | undefined; + expect(selectedProps?.visibleColumns).not.toContain("plan"); + + act(() => { + selectedProps?.onColumnWidthChange?.("account", 240); + }); + const resizedProps = recentRequestsTableSpy.mock.lastCall?.[0] as + | RecentRequestsTableProps + | undefined; + expect(resizedProps?.columnWidths?.account).toBe(240); + + await user.keyboard("{Escape}"); + await user.click( + screen.getByRole("button", { name: "Restore default column layout" }), + ); + + const restoredProps = recentRequestsTableSpy.mock.lastCall?.[0] as + | RecentRequestsTableProps + | undefined; + expect(restoredProps?.visibleColumns).toHaveLength(12); + expect(restoredProps?.columnWidths).toEqual({}); + expect( + window.localStorage.getItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY), + ).toBeNull(); + }); + it("renders the account summary line in the Accounts header using overview accounts", () => { const overview = mockReadyDashboard(); diff --git a/frontend/src/features/dashboard/components/dashboard-page.tsx b/frontend/src/features/dashboard/components/dashboard-page.tsx index 54160071a0..fbdb5ecf48 100644 --- a/frontend/src/features/dashboard/components/dashboard-page.tsx +++ b/frontend/src/features/dashboard/components/dashboard-page.tsx @@ -2,10 +2,18 @@ import { useCallback, useEffect, useMemo } from "react"; import { Trans, useTranslation } from "react-i18next"; import { useNavigate, useSearchParams } from "react-router-dom"; import { useQueryClient } from "@tanstack/react-query"; -import { RefreshCw } from "lucide-react"; +import { Columns3, RefreshCw, RotateCcw } from "lucide-react"; import { AlertMessage } from "@/components/alert-message"; import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuCheckboxItem, + DropdownMenuContent, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; import { SpinnerBlock } from "@/components/ui/spinner"; import { useDialogState } from "@/hooks/use-dialog-state"; import { useAccountMutations } from "@/features/accounts/hooks/use-accounts"; @@ -27,7 +35,9 @@ import { WeeklyCreditsPaceCard } from "@/features/dashboard/components/weekly-cr import { useAuthStore } from "@/features/auth/hooks/use-auth"; import { useDashboard, useDashboardProjections } from "@/features/dashboard/hooks/use-dashboard"; import { useConversations } from "@/features/dashboard/hooks/use-conversations"; +import { useRequestLogTablePreferences } from "@/features/dashboard/hooks/use-request-log-table-preferences"; import { useRequestLogs } from "@/features/dashboard/hooks/use-request-logs"; +import { REQUEST_LOG_COLUMN_OPTIONS } from "@/features/dashboard/request-log-columns"; import { buildDashboardView } from "@/features/dashboard/utils"; import { DEFAULT_OVERVIEW_TIMEFRAME, @@ -48,6 +58,13 @@ const MODEL_OPTION_DELIMITER = ":::"; export function DashboardPage() { const { t, i18n } = useTranslation(); + const { + visibleColumns, + columnWidths, + toggleColumn, + setColumnWidth, + restoreDefaultLayout, + } = useRequestLogTablePreferences(); const resolvedLanguage = i18n.resolvedLanguage; const navigate = useNavigate(); const [searchParams, setSearchParams] = useSearchParams(); @@ -93,7 +110,7 @@ export function DashboardPage() { enabled: isAdmin && dashboardView === "conversations", }); const { conversationsQuery } = conversationsState; - const { filters, logsQuery, optionsQuery, updateFilters } = useRequestLogs({ + const { filters, emptyStateFiltersApplied, logsQuery, optionsQuery, updateFilters } = useRequestLogs({ enabled: dashboardView === "request-logs", }); const { resumeMutation, limitWarmupMutation } = useAccountMutations(); @@ -439,7 +456,51 @@ export function DashboardPage() { onChange={handleDashboardViewChange} showConversations={isAdmin} /> -
+
+ {dashboardView === "request-logs" ? ( + <> + + + + + + + {t("dashboard.requests.columnLayout.visibleColumns")} + + + {REQUEST_LOG_COLUMN_OPTIONS.map((column) => { + const isVisible = visibleColumns.includes(column.id); + return ( + toggleColumn(column.id)} + onSelect={(event) => event.preventDefault()} + > + {t(column.translationKey)} + + ); + })} + + + + + ) : null}
{isAdmin && dashboardView === "conversations" ? : logsQuery.isPending && !logPage ? (
@@ -502,9 +563,13 @@ export function DashboardPage() { requests={view.requestLogs} accounts={overview?.accounts ?? []} total={logPage?.total ?? 0} + visibleColumns={visibleColumns} + columnWidths={columnWidths} + onColumnWidthChange={setColumnWidth} limit={filters.limit} offset={filters.offset} hasMore={logPage?.hasMore ?? false} + filtersApplied={emptyStateFiltersApplied} onLimitChange={(limit) => updateFilters({ limit, offset: 0 })} onOffsetChange={(offset) => updateFilters({ offset })} onConversationClick={handleConversationClick} diff --git a/frontend/src/features/dashboard/components/filters/request-filters.test.tsx b/frontend/src/features/dashboard/components/filters/request-filters.test.tsx index a31d7ef10f..dda77b6502 100644 --- a/frontend/src/features/dashboard/components/filters/request-filters.test.tsx +++ b/frontend/src/features/dashboard/components/filters/request-filters.test.tsx @@ -1,4 +1,5 @@ import { fireEvent, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; import { RequestFilters, type RequestFiltersProps } from "@/features/dashboard/components/filters/request-filters"; @@ -17,14 +18,17 @@ const BASE_FILTERS: FilterState = { offset: 0, }; -function renderFilters(overrides: Partial = {}) { +function renderFilters( + overrides: Partial = {}, + statusOptions: RequestFiltersProps["statusOptions"] = EMPTY_OPTIONS, +) { const filters = { ...BASE_FILTERS, ...overrides }; const props: RequestFiltersProps = { filters, accountOptions: EMPTY_OPTIONS, apiKeyOptions: EMPTY_OPTIONS, modelOptions: EMPTY_OPTIONS, - statusOptions: EMPTY_OPTIONS, + statusOptions, onSearchChange: vi.fn(), onTimeframeChange: vi.fn(), onAccountChange: vi.fn(), @@ -39,6 +43,22 @@ function renderFilters(overrides: Partial = {}) { } describe("RequestFilters conversation badge", () => { + it("renders cancelled as a selectable status option", async () => { + const user = userEvent.setup(); + const props = renderFilters({}, [{ value: "cancelled", label: "Cancelled" }]); + + await user.click(screen.getByRole("button", { name: "Statuses" })); + const [option] = await screen.findAllByRole("menuitemcheckbox"); + expect(option).toBeDefined(); + if (!option) { + throw new Error("Expected the cancelled status option"); + } + + await user.click(option); + + expect(props.onStatusChange).toHaveBeenCalledWith(["cancelled"]); + }); + it("renders no badge when conversationId is null", () => { renderFilters(); expect(screen.queryByText(/conv/i)).not.toBeInTheDocument(); diff --git a/frontend/src/features/dashboard/components/recent-requests-table.test.tsx b/frontend/src/features/dashboard/components/recent-requests-table.test.tsx index f9f29d8d81..eb91bb6c19 100644 --- a/frontend/src/features/dashboard/components/recent-requests-table.test.tsx +++ b/frontend/src/features/dashboard/components/recent-requests-table.test.tsx @@ -3,6 +3,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useAuthStore } from "@/features/auth/hooks/use-auth"; import { RecentRequestsTable } from "@/features/dashboard/components/recent-requests-table"; +import { + ALL_REQUEST_LOG_COLUMNS, + MAX_REQUEST_LOG_COLUMN_WIDTH, + MIN_REQUEST_LOG_COLUMN_WIDTH, + REQUEST_LOG_COLUMN_WIDTH_STEP, +} from "@/features/dashboard/request-log-columns"; +import type { RequestLog } from "@/features/dashboard/schemas"; const ISO = "2026-01-01T12:00:00+00:00"; const NULL_FAILURE_METADATA = { @@ -48,6 +55,41 @@ const PAGINATION_PROPS = { onOffsetChange: vi.fn(), }; +const LAYOUT_REQUEST = { + requestedAt: ISO, + accountId: "acc-layout", + planType: "plus", + apiKeyName: "Layout Key", + apiKeyId: "key-layout", + requestId: "req-layout", + conversationId: null, + requestKind: "normal", + model: "gpt-5.1", + source: null, + serviceTier: null, + requestedServiceTier: null, + actualServiceTier: null, + transport: "http", + upstreamTransport: "http", + status: "ok", + errorCode: null, + errorMessage: null, + ...NULL_FAILURE_METADATA, + ...NULL_USERAGENT_METADATA, + tokens: 1200, + inputTokens: 1000, + outputTokens: 200, + outputTokensRaw: 200, + reasoningTokens: 0, + latencyFirstTokenMs: 200, + latencyQueueMs: null, + cachedInputTokens: 0, + reasoningEffort: null, + costUsd: 0.01, + costBreakdown: null, + latencyMs: 1000, +} satisfies RequestLog; + function openRequestDetails() { fireEvent.click(screen.getByRole("button", { name: "View Details" })); return screen.getByRole("dialog"); @@ -74,6 +116,127 @@ describe("RecentRequestsTable", () => { } }); + it("renders every existing column when layout props are omitted", () => { + render( + , + ); + + expect(screen.getAllByRole("columnheader")).toHaveLength(ALL_REQUEST_LOG_COLUMNS.length); + expect(screen.getByText("Layout Key")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "View Details" })).toBeInTheDocument(); + }); + + it("renders only selected headers and matching row cells", () => { + render( + , + ); + + expect(screen.getAllByRole("columnheader")).toHaveLength(2); + expect(screen.getByRole("columnheader", { name: "Time" })).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Model" })).toBeInTheDocument(); + expect(screen.queryByRole("columnheader", { name: "API Key" })).not.toBeInTheDocument(); + expect(screen.queryByText("Layout Key")).not.toBeInTheDocument(); + expect(screen.getByText("gpt-5.1")).toBeInTheDocument(); + }); + + it("resizes only the selected column by pointer and clamps it to bounds", () => { + const onColumnWidthChange = vi.fn(); + render( + , + ); + + const accountSeparator = screen.getByRole("separator", { + name: "Resize Account column", + }); + fireEvent.pointerDown(accountSeparator, { pointerId: 7, clientX: 100 }); + fireEvent.pointerMove(accountSeparator, { pointerId: 7, clientX: 164 }); + fireEvent.pointerUp(accountSeparator, { pointerId: 7, clientX: 164 }); + + expect(onColumnWidthChange).toHaveBeenCalledWith("account", 224); + expect(onColumnWidthChange).not.toHaveBeenCalledWith("time", expect.any(Number)); + + onColumnWidthChange.mockClear(); + fireEvent.pointerDown(accountSeparator, { pointerId: 8, clientX: 100 }); + fireEvent.pointerMove(accountSeparator, { pointerId: 8, clientX: 10_000 }); + expect(onColumnWidthChange).toHaveBeenLastCalledWith( + "account", + MAX_REQUEST_LOG_COLUMN_WIDTH, + ); + }); + + it("resizes with arrow keys within bounds and sums visible widths", () => { + const onColumnWidthChange = vi.fn(); + render( + , + ); + + expect(screen.getByRole("table")).toHaveStyle({ + width: `${MIN_REQUEST_LOG_COLUMN_WIDTH + 200}px`, + minWidth: `${MIN_REQUEST_LOG_COLUMN_WIDTH + 200}px`, + }); + + const timeSeparator = screen.getByRole("separator", { + name: "Resize Time column", + }); + fireEvent.keyDown(timeSeparator, { key: "ArrowLeft" }); + expect(onColumnWidthChange).toHaveBeenLastCalledWith( + "time", + MIN_REQUEST_LOG_COLUMN_WIDTH, + ); + + const accountSeparator = screen.getByRole("separator", { + name: "Resize Account column", + }); + fireEvent.keyDown(accountSeparator, { key: "ArrowRight" }); + expect(onColumnWidthChange).toHaveBeenLastCalledWith( + "account", + 200 + REQUEST_LOG_COLUMN_WIDTH_STEP, + ); + }); + + it("pins the table to the configured width sum so surplus space is not redistributed", () => { + render( + , + ); + + // An explicit width (not merely a minimum) keeps configured column widths + // independent when their sum is smaller than the container. + expect(screen.getByRole("table")).toHaveStyle({ + width: "272px", + minWidth: "272px", + }); + }); + it("renders rows with status badges and supports request details and copy actions", async () => { const longError = "Rate limit reached while processing this request ".repeat(3); const writeText = vi.fn().mockResolvedValue(undefined); @@ -124,6 +287,11 @@ describe("RecentRequestsTable", () => { ...NULL_FAILURE_METADATA, ...NULL_USERAGENT_METADATA, upstreamTransport: "auto", + upstreamProxyRouteMode: "account_bound", + upstreamProxyPoolId: "pool-1", + upstreamProxyEndpointId: "endpoint-1", + upstreamProxyFallbackUsed: true, + upstreamProxyFailClosedReason: "no_healthy_endpoint", tokens: 1200, inputTokens: 1000, outputTokens: 200, @@ -163,6 +331,16 @@ describe("RecentRequestsTable", () => { expect(within(dialog).getByText("rate_limit_exceeded")).toBeInTheDocument(); expect(dialog.textContent).toContain("Rate limit reached while processing this request"); expect(within(dialog).getByText("1.0 s")).toBeInTheDocument(); + expect(within(dialog).getByText("Route mode")).toBeInTheDocument(); + expect(within(dialog).getByText("account_bound")).toBeInTheDocument(); + expect(within(dialog).getByText("Proxy pool")).toBeInTheDocument(); + expect(within(dialog).getByText("pool-1")).toBeInTheDocument(); + expect(within(dialog).getByText("Proxy endpoint")).toBeInTheDocument(); + expect(within(dialog).getByText("endpoint-1")).toBeInTheDocument(); + expect(within(dialog).getByText("Same-pool fallback")).toBeInTheDocument(); + expect(within(dialog).getByText("Used")).toBeInTheDocument(); + expect(within(dialog).getByText("Fail-closed reason")).toBeInTheDocument(); + expect(within(dialog).getByText("no_healthy_endpoint")).toBeInTheDocument(); await act(async () => { fireEvent.click(screen.getByRole("button", { name: "Copy Request ID" })); @@ -181,6 +359,55 @@ describe("RecentRequestsTable", () => { expect(writeText).toHaveBeenCalledWith(longError); }); + it("renders cancelled requests with a distinct non-error badge", () => { + render( + , + ); + + const badge = screen.getByText("Cancelled"); + + expect(badge).toHaveClass("bg-sky-500/15"); + expect(badge).not.toHaveClass("bg-zinc-500/15"); + }); + it("shows TTFT and output-token TPS beside tokens", () => { render( { expect(within(row as HTMLElement).getByText("200.0")).toBeInTheDocument(); }); + it("shows reasoning as secondary token metadata and an included-output detail", () => { + render( + , + ); + + expect(screen.getByText("1.2K")).toBeInTheDocument(); + expect(screen.getByText("80 reasoning")).toBeInTheDocument(); + + const dialog = openRequestDetails(); + const reasoningLabel = within(dialog).getByText( + "Reasoning tokens (included in output)", + ); + expect(reasoningLabel.parentElement?.parentElement).toHaveTextContent("80"); + }); + + it("renders a known zero reasoning count", () => { + render( + , + ); + + expect(screen.getByText("0 reasoning")).toBeInTheDocument(); + const dialog = openRequestDetails(); + const reasoningLabel = within(dialog).getByText( + "Reasoning tokens (included in output)", + ); + expect(reasoningLabel.parentElement?.parentElement).toHaveTextContent("0"); + }); + + it("omits unknown reasoning usage instead of estimating it", () => { + render( + , + ); + + expect(screen.queryByText(/reasoning/i)).not.toBeInTheDocument(); + const dialog = openRequestDetails(); + expect( + within(dialog).queryByText("Reasoning tokens (included in output)"), + ).not.toBeInTheDocument(); + }); + it("does not calculate TPS from fallback output tokens", () => { render( { expect(within(row as HTMLElement).queryByText("250.0")).not.toBeInTheDocument(); }); - it("renders empty state", () => { + it("renders first-run empty copy when no filters are applied", () => { render(); + expect(screen.getByText("No requests yet")).toBeInTheDocument(); + expect( + screen.getByText("Requests will appear here after clients start using the proxy."), + ).toBeInTheDocument(); + expect(screen.queryByText("No request logs match the current filters.")).not.toBeInTheDocument(); + }); + + it("renders filter-empty copy when a later page has no rows but logs exist", () => { + render( + , + ); + expect(screen.getByText("No matching requests")).toBeInTheDocument(); + expect(screen.getByText("No request logs match the current filters.")).toBeInTheDocument(); + expect(screen.queryByText("No requests yet")).not.toBeInTheDocument(); + }); + + it("renders filter-empty copy when filters are applied", () => { + render( + , + ); + expect(screen.getByText("No matching requests")).toBeInTheDocument(); expect(screen.getByText("No request logs match the current filters.")).toBeInTheDocument(); }); diff --git a/frontend/src/features/dashboard/components/recent-requests-table.tsx b/frontend/src/features/dashboard/components/recent-requests-table.tsx index 53be393677..32489cf4c3 100644 --- a/frontend/src/features/dashboard/components/recent-requests-table.tsx +++ b/frontend/src/features/dashboard/components/recent-requests-table.tsx @@ -1,5 +1,11 @@ import { Inbox } from "lucide-react"; -import { useMemo, useState } from "react"; +import { + useMemo, + useRef, + useState, + type KeyboardEvent, + type PointerEvent, +} from "react"; import { useTranslation } from "react-i18next"; import { isEmailLabel } from "@/components/blur-email"; @@ -26,9 +32,20 @@ import { } from "@/components/ui/table"; import { PaginationControls } from "@/features/dashboard/components/filters/pagination-controls"; import { RequestArchivePanel } from "@/features/conversation-archive/components/request-archive-panel"; +import { + ALL_REQUEST_LOG_COLUMNS, + MAX_REQUEST_LOG_COLUMN_WIDTH, + MIN_REQUEST_LOG_COLUMN_WIDTH, + REQUEST_LOG_COLUMN_DEFAULT_WIDTHS, + REQUEST_LOG_COLUMN_WIDTH_STEP, + clampRequestLogColumnWidth, + type RequestLogColumnId, + type RequestLogColumnWidths, +} from "@/features/dashboard/request-log-columns"; import type { AccountSummary, RequestLog } from "@/features/dashboard/schemas"; import { useAuthStore } from "@/features/auth/hooks/use-auth"; import { useDateDisplayFormatStore } from "@/hooks/use-date-format"; +import { cn } from "@/lib/utils"; import { REQUEST_STATUS_LABELS } from "@/utils/constants"; import { formatDateTimeInline, @@ -42,6 +59,7 @@ import { const STATUS_CLASS_MAP: Record = { ok: "bg-emerald-500/15 text-emerald-700 border-emerald-500/20 hover:bg-emerald-500/20 dark:text-emerald-400", + cancelled: "bg-sky-500/15 text-sky-700 border-sky-500/20 hover:bg-sky-500/20 dark:text-sky-400", rate_limit: "bg-orange-500/15 text-orange-700 border-orange-500/20 hover:bg-orange-500/20 dark:text-orange-400", quota: "bg-red-500/15 text-red-700 border-red-500/20 hover:bg-red-500/20 dark:text-red-400", error: "bg-zinc-500/15 text-zinc-700 border-zinc-500/20 hover:bg-zinc-500/20 dark:text-zinc-400", @@ -82,11 +100,133 @@ export type RecentRequestsTableProps = { limit: number; offset: number; hasMore: boolean; + filtersApplied?: boolean; + visibleColumns?: readonly RequestLogColumnId[]; + columnWidths?: RequestLogColumnWidths; + onColumnWidthChange?: (column: RequestLogColumnId, width: number) => void; onLimitChange: (limit: number) => void; onOffsetChange: (offset: number) => void; onConversationClick?: (conversationId: string) => void; }; +type RequestLogTableHeadProps = { + column: RequestLogColumnId; + label: string; + resizeLabel: string; + className?: string; + width?: number; + onWidthChange?: (column: RequestLogColumnId, width: number) => void; +}; + +function RequestLogTableHead({ + column, + label, + resizeLabel, + className, + width, + onWidthChange, +}: RequestLogTableHeadProps) { + const resizeState = useRef<{ + pointerId: number; + startX: number; + startWidth: number; + } | null>(null); + const resolvedWidth = clampRequestLogColumnWidth( + width ?? REQUEST_LOG_COLUMN_DEFAULT_WIDTHS[column], + ); + + const handlePointerDown = (event: PointerEvent) => { + if (!onWidthChange) { + return; + } + + event.preventDefault(); + const measuredWidth = event.currentTarget.parentElement?.getBoundingClientRect().width; + resizeState.current = { + pointerId: event.pointerId, + startX: event.clientX, + startWidth: measuredWidth && measuredWidth > 0 ? measuredWidth : resolvedWidth, + }; + event.currentTarget.setPointerCapture(event.pointerId); + }; + + const handlePointerMove = (event: PointerEvent) => { + const state = resizeState.current; + if (!state || state.pointerId !== event.pointerId || !onWidthChange) { + return; + } + + onWidthChange( + column, + clampRequestLogColumnWidth(state.startWidth + event.clientX - state.startX), + ); + }; + + const handlePointerEnd = (event: PointerEvent) => { + if (resizeState.current?.pointerId !== event.pointerId) { + return; + } + + resizeState.current = null; + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId); + } + }; + + const handleKeyDown = (event: KeyboardEvent) => { + if (!onWidthChange || (event.key !== "ArrowLeft" && event.key !== "ArrowRight")) { + return; + } + + event.preventDefault(); + const direction = event.key === "ArrowLeft" ? -1 : 1; + onWidthChange( + column, + clampRequestLogColumnWidth( + resolvedWidth + direction * REQUEST_LOG_COLUMN_WIDTH_STEP, + ), + ); + }; + + return ( + + {label} + {onWidthChange ? ( +
{ + resizeState.current = null; + }} + onKeyDown={handleKeyDown} + > +
+ ) : null} + + ); +} + function formatRequestCostSummary(request: RequestLog | null, t: ReturnType["t"]): string | null { if (!request || request.status !== "ok") { return null; @@ -170,6 +310,10 @@ export function RecentRequestsTable({ limit, offset, hasMore, + filtersApplied = false, + visibleColumns: configuredVisibleColumns, + columnWidths, + onColumnWidthChange, onLimitChange, onOffsetChange, onConversationClick, @@ -180,6 +324,25 @@ export function RecentRequestsTable({ const isAdmin = useAuthStore((state) => state.role === "admin"); const dateDisplayFormat = useDateDisplayFormatStore((state) => state.dateDisplayFormat); const selectedRequestCostSummary = formatRequestCostSummary(selectedRequest, t); + const visibleColumns = configuredVisibleColumns ?? ALL_REQUEST_LOG_COLUMNS; + const visibleColumnSet = useMemo(() => new Set(visibleColumns), [visibleColumns]); + const hasConfiguredLayout = + configuredVisibleColumns !== undefined || + columnWidths !== undefined || + onColumnWidthChange !== undefined; + const tableWidth = hasConfiguredLayout + ? visibleColumns.reduce( + (totalWidth, column) => + totalWidth + + clampRequestLogColumnWidth( + columnWidths?.[column] ?? REQUEST_LOG_COLUMN_DEFAULT_WIDTHS[column], + ), + 0, + ) + : undefined; + const isColumnVisible = (column: RequestLogColumnId) => visibleColumnSet.has(column); + const resizeLabel = (label: string) => + t("dashboard.requests.resizeColumn", { column: label }); const accountLabelMap = useMemo(() => { const index = new Map(); @@ -202,11 +365,20 @@ export function RecentRequestsTable({ }, [accounts]); if (requests.length === 0) { + const emptyFromExistingLogs = filtersApplied || total > 0; return ( ); } @@ -215,21 +387,24 @@ export function RecentRequestsTable({
- +
- {t("dashboard.requests.columns.time")} - {t("dashboard.requests.columns.account")} - {t("dashboard.requests.columns.plan")} - {t("dashboard.requests.columns.apiKey")} - {t("dashboard.requests.columns.model")} - {t("dashboard.requests.columns.transport")} - {t("dashboard.requests.columns.status")} - TTFT - TPS - {t("dashboard.requests.columns.tokens")} - {t("dashboard.requests.columns.cost")} - {t("dashboard.requests.columns.details")} + {isColumnVisible("time") ? : null} + {isColumnVisible("account") ? : null} + {isColumnVisible("plan") ? : null} + {isColumnVisible("apiKey") ? : null} + {isColumnVisible("model") ? : null} + {isColumnVisible("transport") ? : null} + {isColumnVisible("status") ? : null} + {isColumnVisible("ttft") ? : null} + {isColumnVisible("tps") ? : null} + {isColumnVisible("tokens") ? : null} + {isColumnVisible("cost") ? : null} + {isColumnVisible("details") ? : null} @@ -249,20 +424,20 @@ export function RecentRequestsTable({ return ( - + {isColumnVisible("time") ?
{time.primary}
{time.secondary}
- - + : null} + {isColumnVisible("account") ? {isEmailLabel && blurred ? ( {accountLabel} ) : ( accountLabel )} - - + : null} + {isColumnVisible("plan") ? {planType ? ( -- )} - - + : null} + {isColumnVisible("apiKey") ? {request.apiKeyName || "--"} - - + : null} + {isColumnVisible("model") ?
{formatModelLabel(request.model, request.reasoningEffort, visibleServiceTier)} @@ -293,8 +468,8 @@ export function RecentRequestsTable({
) : null} - - + : null} + {isColumnVisible("transport") ? {request.transport ? (
-- )} - - + : null} + {isColumnVisible("status") ? {t(`dashboard.requestStatus.${request.status}`, { defaultValue: REQUEST_STATUS_LABELS[request.status] ?? request.status })} - - + : null} + {isColumnVisible("ttft") ? {formatCompactElapsed(request.latencyFirstTokenMs) ?? "--"} - - + : null} + {isColumnVisible("tps") ? {generationSpeed ?? "--"} - - + : null} + {isColumnVisible("tokens") ?
{formatCompactNumber(request.tokens)}
{request.cachedInputTokens != null && request.cachedInputTokens > 0 && ( @@ -336,12 +511,19 @@ export function RecentRequestsTable({ {t("common.units.cachedShort", { count: formatCompactNumber(request.cachedInputTokens) })}
)} + {request.reasoningTokens != null ? ( +
+ {t("dashboard.requests.reasoningTokensShort", { + count: formatCompactNumber(request.reasoningTokens), + })} +
+ ) : null}
- - + : null} + {isColumnVisible("cost") ? {formatCurrency(request.costUsd)} - - + : null} + {isColumnVisible("details") ? {hasError ? (
{request.errorCode ? ( @@ -375,7 +557,7 @@ export function RecentRequestsTable({ {t("dashboard.requests.viewDetails")} )} - + : null} ); })} @@ -396,12 +578,12 @@ export function RecentRequestsTable({
{ if (!open) setSelectedRequest(null); }}> - + {t("dashboard.requestDetails.title")} {t("dashboard.requestDetails.description")} -
+
+ {selectedRequest?.reasoningTokens != null ? ( + + ) : null}
+ {selectedRequest?.upstreamProxyRouteMode || + selectedRequest?.upstreamProxyPoolId || + selectedRequest?.upstreamProxyEndpointId || + selectedRequest?.upstreamProxyFallbackUsed != null || + selectedRequest?.upstreamProxyFailClosedReason ? ( +
+ {selectedRequest.upstreamProxyRouteMode ? ( + + ) : null} + {selectedRequest.upstreamProxyPoolId ? ( + + ) : null} + {selectedRequest.upstreamProxyEndpointId ? ( + + ) : null} + {selectedRequest.upstreamProxyFallbackUsed != null ? ( + + ) : null} + {selectedRequest.upstreamProxyFailClosedReason ? ( + + ) : null} +
+ ) : null} {isAdmin ? ( { + beforeEach(() => { + window.localStorage.removeItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY); + }); + + it("shows every request-log column by default", () => { + const { result } = renderHook(() => useRequestLogTablePreferences()); + + expect(result.current.visibleColumns).toEqual(ALL_REQUEST_LOG_COLUMNS); + expect(result.current.columnWidths).toEqual({}); + }); + + it("persists visible columns and individual widths across remounts", () => { + const { result, unmount } = renderHook(() => useRequestLogTablePreferences()); + + act(() => { + result.current.toggleColumn("plan"); + result.current.setColumnWidth("account", 284); + }); + + expect(result.current.visibleColumns).not.toContain("plan"); + expect(result.current.columnWidths.account).toBe(284); + + unmount(); + const restored = renderHook(() => useRequestLogTablePreferences()); + expect(restored.result.current.visibleColumns).not.toContain("plan"); + expect(restored.result.current.columnWidths.account).toBe(284); + }); + + it("clamps finite widths and ignores malformed width entries", () => { + window.localStorage.setItem( + REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY, + JSON.stringify({ + visibleColumns: ["time", "account"], + columnWidths: { + time: 1, + account: "wide", + details: 10_000, + unknown: 200, + }, + }), + ); + + const { result } = renderHook(() => useRequestLogTablePreferences()); + + expect(result.current.columnWidths).toEqual({ + time: MIN_REQUEST_LOG_COLUMN_WIDTH, + details: MAX_REQUEST_LOG_COLUMN_WIDTH, + }); + }); + + it("keeps the final visible column selected", () => { + window.localStorage.setItem( + REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY, + JSON.stringify({ visibleColumns: ["time"], columnWidths: {} }), + ); + const { result } = renderHook(() => useRequestLogTablePreferences()); + + act(() => { + result.current.toggleColumn("time"); + }); + + expect(result.current.visibleColumns).toEqual(["time"]); + }); + + it.each([ + "{not-json", + JSON.stringify({ visibleColumns: [], columnWidths: {} }), + JSON.stringify({ visibleColumns: ["time", "retired-column"], columnWidths: {} }), + ])("falls back safely for malformed or stale preferences", (stored) => { + window.localStorage.setItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY, stored); + + const { result } = renderHook(() => useRequestLogTablePreferences()); + + expect(result.current.visibleColumns).toEqual(ALL_REQUEST_LOG_COLUMNS); + }); + + it("restores default widths too when stored visibility is stale", () => { + window.localStorage.setItem( + REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY, + JSON.stringify({ + visibleColumns: ["time", "retired-column"], + columnWidths: { time: 240, account: 320 }, + }), + ); + + const { result } = renderHook(() => useRequestLogTablePreferences()); + + expect(result.current.visibleColumns).toEqual(ALL_REQUEST_LOG_COLUMNS); + expect(result.current.columnWidths).toEqual({}); + }); + + it("restores all columns, clears widths, and removes stored customization", () => { + const { result } = renderHook(() => useRequestLogTablePreferences()); + act(() => { + result.current.toggleColumn("plan"); + result.current.setColumnWidth("account", 240); + }); + act(() => { + result.current.restoreDefaultLayout(); + }); + + expect(result.current.visibleColumns).toEqual(ALL_REQUEST_LOG_COLUMNS); + expect(result.current.columnWidths).toEqual({}); + expect( + window.localStorage.getItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY), + ).toBeNull(); + }); +}); diff --git a/frontend/src/features/dashboard/hooks/use-request-log-table-preferences.ts b/frontend/src/features/dashboard/hooks/use-request-log-table-preferences.ts new file mode 100644 index 0000000000..b94fbbed82 --- /dev/null +++ b/frontend/src/features/dashboard/hooks/use-request-log-table-preferences.ts @@ -0,0 +1,164 @@ +import { useCallback, useState } from "react"; + +import { + ALL_REQUEST_LOG_COLUMNS, + DEFAULT_REQUEST_LOG_COLUMNS, + clampRequestLogColumnWidth, + type RequestLogColumnId, + type RequestLogColumnWidths, +} from "@/features/dashboard/request-log-columns"; + +export const REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY = + "codex-lb-dashboard-request-log-columns:v1"; + +type RequestLogTablePreferences = { + visibleColumns: RequestLogColumnId[]; + columnWidths: RequestLogColumnWidths; +}; + +const supportedColumns = new Set(ALL_REQUEST_LOG_COLUMNS); + +function isRequestLogColumnId(value: unknown): value is RequestLogColumnId { + return typeof value === "string" && supportedColumns.has(value as RequestLogColumnId); +} + +function normalizeColumnWidths(value: unknown): RequestLogColumnWidths { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return {}; + } + + const stored = value as Record; + const widths: RequestLogColumnWidths = {}; + for (const column of ALL_REQUEST_LOG_COLUMNS) { + const width = stored[column]; + if (typeof width === "number" && Number.isFinite(width)) { + widths[column] = clampRequestLogColumnWidth(width); + } + } + return widths; +} + +function defaultPreferences(): RequestLogTablePreferences { + return { + visibleColumns: [...DEFAULT_REQUEST_LOG_COLUMNS], + columnWidths: {}, + }; +} + +function normalizeVisibleColumns(value: unknown): RequestLogColumnId[] | null { + if ( + !Array.isArray(value) || + value.length === 0 || + value.some((column) => !isRequestLogColumnId(column)) + ) { + return null; + } + + const selected = new Set(value); + return ALL_REQUEST_LOG_COLUMNS.filter((column) => selected.has(column)); +} + +function loadPreferences(): RequestLogTablePreferences { + if (typeof window === "undefined") { + return defaultPreferences(); + } + + try { + const stored = window.localStorage.getItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY); + if (!stored) { + return defaultPreferences(); + } + + const parsed = JSON.parse(stored) as { + visibleColumns?: unknown; + columnWidths?: unknown; + }; + const visibleColumns = normalizeVisibleColumns(parsed.visibleColumns); + if (visibleColumns === null) { + // Stale or malformed visibility invalidates the whole stored layout so the + // dashboard restores the complete default layout, widths included. + return defaultPreferences(); + } + return { + visibleColumns, + columnWidths: normalizeColumnWidths(parsed.columnWidths), + }; + } catch { + return defaultPreferences(); + } +} + +function persistPreferences(preferences: RequestLogTablePreferences): void { + if (typeof window === "undefined") { + return; + } + + try { + window.localStorage.setItem( + REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY, + JSON.stringify(preferences), + ); + } catch { + // Browser storage may be unavailable in private or locked-down sessions. + } +} + +export function useRequestLogTablePreferences() { + const [preferences, setPreferences] = useState(loadPreferences); + + const toggleColumn = useCallback((column: RequestLogColumnId) => { + setPreferences((current) => { + const isVisible = current.visibleColumns.includes(column); + if (isVisible && current.visibleColumns.length === 1) { + return current; + } + + const selected = new Set(current.visibleColumns); + if (isVisible) { + selected.delete(column); + } else { + selected.add(column); + } + const next = { + ...current, + visibleColumns: ALL_REQUEST_LOG_COLUMNS.filter((candidate) => selected.has(candidate)), + }; + persistPreferences(next); + return next; + }); + }, []); + + const setColumnWidth = useCallback((column: RequestLogColumnId, width: number) => { + setPreferences((current) => { + const next = { + ...current, + columnWidths: { + ...current.columnWidths, + [column]: clampRequestLogColumnWidth(width), + }, + }; + persistPreferences(next); + return next; + }); + }, []); + + const restoreDefaultLayout = useCallback(() => { + const next = defaultPreferences(); + if (typeof window !== "undefined") { + try { + window.localStorage.removeItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY); + } catch { + // Browser storage may be unavailable in private or locked-down sessions. + } + } + setPreferences(next); + }, []); + + return { + visibleColumns: preferences.visibleColumns, + columnWidths: preferences.columnWidths, + toggleColumn, + setColumnWidth, + restoreDefaultLayout, + }; +} diff --git a/frontend/src/features/dashboard/hooks/use-request-logs.test.ts b/frontend/src/features/dashboard/hooks/use-request-logs.test.ts index 0eea93bcb0..fcae731258 100644 --- a/frontend/src/features/dashboard/hooks/use-request-logs.test.ts +++ b/frontend/src/features/dashboard/hooks/use-request-logs.test.ts @@ -2,10 +2,15 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { act, renderHook, waitFor } from "@testing-library/react"; import { HttpResponse, http } from "msw"; import { createElement, type PropsWithChildren, useEffect } from "react"; -import { MemoryRouter, useLocation } from "react-router-dom"; +import { + MemoryRouter, + useLocation, + useNavigate, + type NavigateFunction, +} from "react-router-dom"; import { describe, expect, it } from "vitest"; -import { useRequestLogs } from "@/features/dashboard/hooks/use-request-logs"; +import { requestLogFiltersApplied, useRequestLogs } from "@/features/dashboard/hooks/use-request-logs"; import { server } from "@/test/mocks/server"; function createTestQueryClient(): QueryClient { @@ -19,13 +24,24 @@ function createTestQueryClient(): QueryClient { }); } -function LocationSpy({ onChange }: { onChange?: (search: string) => void }) { +function LocationSpy({ + onChange, + onNavigateReady, +}: { + onChange?: (search: string) => void; + onNavigateReady?: (navigate: NavigateFunction) => void; +}) { const routeLocation = useLocation(); + const navigate = useNavigate(); useEffect(() => { onChange?.(routeLocation.search); }, [routeLocation.search, onChange]); + useEffect(() => { + onNavigateReady?.(navigate); + }, [navigate, onNavigateReady]); + return null; } @@ -33,6 +49,7 @@ function createWrapper( queryClient: QueryClient, initialEntry = "/dashboard", onLocationChange?: (search: string) => void, + onNavigateReady?: (navigate: NavigateFunction) => void, ) { return function Wrapper({ children }: PropsWithChildren) { return createElement( @@ -41,7 +58,7 @@ function createWrapper( createElement( MemoryRouter, { initialEntries: [initialEntry] }, - createElement(LocationSpy, { onChange: onLocationChange }), + createElement(LocationSpy, { onChange: onLocationChange, onNavigateReady }), children, ), ); @@ -105,6 +122,87 @@ describe("useRequestLogs", () => { expect(locationSearch).toContain("search=quota"); }); + it("keeps filtered-empty semantics while clearing a filter", async () => { + let releaseUnfiltered: (() => void) | undefined; + const unfilteredResponse = new Promise((resolve) => { + releaseUnfiltered = resolve; + }); + server.use( + http.get("/api/request-logs", async ({ request }) => { + const url = new URL(request.url); + if (url.searchParams.get("search")) { + return HttpResponse.json({ requests: [], total: 0, hasMore: false }); + } + await unfilteredResponse; + return HttpResponse.json({ requests: [], total: 1, hasMore: false }); + }), + ); + + const queryClient = createTestQueryClient(); + const wrapper = createWrapper(queryClient, "/dashboard?search=missing"); + const { result } = renderHook(() => useRequestLogs(), { wrapper }); + + await waitFor(() => expect(result.current.logsQuery.isSuccess).toBe(true)); + expect(result.current.emptyStateFiltersApplied).toBe(true); + + act(() => { + result.current.updateFilters({ search: "", offset: 0 }); + }); + + await waitFor(() => expect(result.current.filters.search).toBe("")); + expect(result.current.logsQuery.isPlaceholderData).toBe(true); + expect(result.current.logsQuery.data?.total).toBe(0); + expect(result.current.emptyStateFiltersApplied).toBe(true); + + releaseUnfiltered?.(); + await waitFor(() => expect(result.current.logsQuery.data?.total).toBe(1)); + expect(result.current.emptyStateFiltersApplied).toBe(false); + }); + + it("keeps filtered-empty semantics when route navigation clears filters", async () => { + let releaseUnfiltered: (() => void) | undefined; + let navigate: NavigateFunction | undefined; + const unfilteredResponse = new Promise((resolve) => { + releaseUnfiltered = resolve; + }); + server.use( + http.get("/api/request-logs", async ({ request }) => { + const url = new URL(request.url); + if (url.searchParams.get("search")) { + return HttpResponse.json({ requests: [], total: 0, hasMore: false }); + } + await unfilteredResponse; + return HttpResponse.json({ requests: [], total: 1, hasMore: false }); + }), + ); + + const queryClient = createTestQueryClient(); + const wrapper = createWrapper( + queryClient, + "/dashboard?search=missing", + undefined, + (routerNavigate) => { + navigate = routerNavigate; + }, + ); + const { result } = renderHook(() => useRequestLogs(), { wrapper }); + + await waitFor(() => expect(result.current.logsQuery.isSuccess).toBe(true)); + await waitFor(() => expect(navigate).toBeDefined()); + + act(() => { + navigate?.("/dashboard"); + }); + + await waitFor(() => expect(result.current.filters.search).toBe("")); + expect(result.current.logsQuery.isPlaceholderData).toBe(true); + expect(result.current.emptyStateFiltersApplied).toBe(true); + + releaseUnfiltered?.(); + await waitFor(() => expect(result.current.logsQuery.data?.total).toBe(1)); + expect(result.current.emptyStateFiltersApplied).toBe(false); + }); + it("supports pagination updates with total/hasMore response", async () => { const queryClient = createTestQueryClient(); const wrapper = createWrapper(queryClient, "/dashboard?limit=1&offset=0"); @@ -433,3 +531,27 @@ describe("useRequestLogs", () => { expect(apiParams.some((p) => p === rawId)).toBe(true); }); }); + +describe("requestLogFiltersApplied", () => { + const defaults = { + search: "", + timeframe: "all" as const, + accountIds: [], + apiKeyIds: [], + modelOptions: [], + statuses: [], + conversationId: null, + limit: 25, + offset: 0, + }; + + it("is false for default request-log filters", () => { + expect(requestLogFiltersApplied(defaults)).toBe(false); + }); + + it("is true when a narrowing filter is set", () => { + expect(requestLogFiltersApplied({ ...defaults, timeframe: "24h" })).toBe(true); + expect(requestLogFiltersApplied({ ...defaults, search: "rate" })).toBe(true); + expect(requestLogFiltersApplied({ ...defaults, conversationId: "conv-1" })).toBe(true); + }); +}); diff --git a/frontend/src/features/dashboard/hooks/use-request-logs.ts b/frontend/src/features/dashboard/hooks/use-request-logs.ts index 0b54552aec..c479fae6a8 100644 --- a/frontend/src/features/dashboard/hooks/use-request-logs.ts +++ b/frontend/src/features/dashboard/hooks/use-request-logs.ts @@ -22,6 +22,18 @@ const DEFAULT_FILTER_STATE: FilterState = { offset: 0, }; +export function requestLogFiltersApplied(filters: FilterState): boolean { + return ( + filters.search.trim() !== "" || + filters.timeframe !== DEFAULT_FILTER_STATE.timeframe || + filters.accountIds.length > 0 || + filters.apiKeyIds.length > 0 || + filters.modelOptions.length > 0 || + filters.statuses.length > 0 || + Boolean(filters.conversationId) + ); +} + const REQUEST_LOG_PARAM_KEYS = [ "search", "timeframe", @@ -114,6 +126,7 @@ export function useRequestLogs(options: UseRequestLogsOptions = {}) { const [searchParams, setSearchParams] = useSearchParams(); const filters = useMemo(() => parseFilterState(searchParams), [searchParams]); + const filtersApplied = requestLogFiltersApplied(filters); const since = useMemo(() => timeframeToSinceIso(filters.timeframe), [filters.timeframe]); const listFilters = useMemo( () => ({ @@ -140,28 +153,36 @@ export function useRequestLogs(options: UseRequestLogsOptions = {}) { ); const { - data: logsData, + data: logsResult, error: logsError, isFetching: logsIsFetching, isLoading: logsIsLoading, isPending: logsIsPending, + isPlaceholderData: logsIsPlaceholderData, isSuccess: logsIsSuccess, refetch: refetchLogs, } = useQuery({ queryKey: ["dashboard", "request-logs", listFilters], - queryFn: () => getRequestLogs(listFilters), + queryFn: async () => ({ + page: await getRequestLogs(listFilters), + filtersApplied, + }), enabled, refetchInterval: 30_000, refetchIntervalInBackground: false, refetchOnWindowFocus: true, placeholderData: keepPreviousData, }); + const logsData = logsResult?.page; + const emptyStateFiltersApplied = + filtersApplied || (logsIsPlaceholderData && Boolean(logsResult?.filtersApplied)); const logsQuery = { data: logsData, error: logsError, isFetching: logsIsFetching, isLoading: logsIsLoading, isPending: logsIsPending, + isPlaceholderData: logsIsPlaceholderData, isSuccess: logsIsSuccess, refetch: refetchLogs, }; @@ -204,6 +225,7 @@ export function useRequestLogs(options: UseRequestLogsOptions = {}) { filters, listFilters, facetFilters, + emptyStateFiltersApplied, logsQuery, optionsQuery, updateFilters, diff --git a/frontend/src/features/dashboard/request-log-columns.ts b/frontend/src/features/dashboard/request-log-columns.ts new file mode 100644 index 0000000000..db9dc094d2 --- /dev/null +++ b/frontend/src/features/dashboard/request-log-columns.ts @@ -0,0 +1,50 @@ +export const REQUEST_LOG_COLUMN_OPTIONS = [ + { id: "time", translationKey: "dashboard.requests.columns.time" }, + { id: "account", translationKey: "dashboard.requests.columns.account" }, + { id: "plan", translationKey: "dashboard.requests.columns.plan" }, + { id: "apiKey", translationKey: "dashboard.requests.columns.apiKey" }, + { id: "model", translationKey: "dashboard.requests.columns.model" }, + { id: "transport", translationKey: "dashboard.requests.columns.transport" }, + { id: "status", translationKey: "dashboard.requests.columns.status" }, + { id: "ttft", translationKey: "dashboard.requests.columns.ttft" }, + { id: "tps", translationKey: "dashboard.requests.columns.tps" }, + { id: "tokens", translationKey: "dashboard.requests.columns.tokens" }, + { id: "cost", translationKey: "dashboard.requests.columns.cost" }, + { id: "details", translationKey: "dashboard.requests.columns.details" }, +] as const; + +export type RequestLogColumnId = (typeof REQUEST_LOG_COLUMN_OPTIONS)[number]["id"]; + +export const ALL_REQUEST_LOG_COLUMNS: readonly RequestLogColumnId[] = + REQUEST_LOG_COLUMN_OPTIONS.map((column) => column.id); + +export const DEFAULT_REQUEST_LOG_COLUMNS: readonly RequestLogColumnId[] = [ + ...ALL_REQUEST_LOG_COLUMNS, +]; + +export const MIN_REQUEST_LOG_COLUMN_WIDTH = 64; +export const MAX_REQUEST_LOG_COLUMN_WIDTH = 720; +export const REQUEST_LOG_COLUMN_WIDTH_STEP = 8; + +export type RequestLogColumnWidths = Partial>; + +export const REQUEST_LOG_COLUMN_DEFAULT_WIDTHS: Record = { + time: 112, + account: 160, + plan: 96, + apiKey: 144, + model: 180, + transport: 128, + status: 96, + ttft: 80, + tps: 80, + tokens: 96, + cost: 64, + details: 288, +}; + +export function clampRequestLogColumnWidth(width: number): number { + return Math.round( + Math.max(MIN_REQUEST_LOG_COLUMN_WIDTH, Math.min(MAX_REQUEST_LOG_COLUMN_WIDTH, width)), + ); +} diff --git a/frontend/src/features/dashboard/schemas.test.ts b/frontend/src/features/dashboard/schemas.test.ts index 7d57df7a7d..d083e50e1f 100644 --- a/frontend/src/features/dashboard/schemas.test.ts +++ b/frontend/src/features/dashboard/schemas.test.ts @@ -57,6 +57,7 @@ describe("DashboardOverviewSchema", () => { cachedInputTokens: 300, errorRate: 0.02, errorCount: 10, + cancelledCount: 3, topError: null, }, comparison: { @@ -81,6 +82,7 @@ describe("DashboardOverviewSchema", () => { expect(parsed.accounts).toHaveLength(0); expect(parsed.summary.comparison?.previous.requests).toBe(250); + expect(parsed.summary.metrics?.cancelledCount).toBe(3); }); it("drops legacy request_logs field from parse result", () => { @@ -197,6 +199,11 @@ describe("RequestLogsResponseSchema", () => { connectionRequestKind: "prewarm", model: "gpt-5.1", transport: "websocket", + upstreamProxyRouteMode: "account_bound", + upstreamProxyPoolId: "pool-1", + upstreamProxyEndpointId: "endpoint-1", + upstreamProxyFallbackUsed: true, + upstreamProxyFailClosedReason: null, useragent: "Mozilla/5.0", useragentGroup: "Mozilla", clientIp: "203.0.113.7", @@ -237,6 +244,11 @@ describe("RequestLogsResponseSchema", () => { expect(parsed.requests[0]?.connectionRequestKind).toBe("prewarm"); expect(parsed.requests[0]?.planType).toBe("plus"); expect(parsed.requests[0]?.transport).toBe("websocket"); + expect(parsed.requests[0]?.upstreamProxyRouteMode).toBe("account_bound"); + expect(parsed.requests[0]?.upstreamProxyPoolId).toBe("pool-1"); + expect(parsed.requests[0]?.upstreamProxyEndpointId).toBe("endpoint-1"); + expect(parsed.requests[0]?.upstreamProxyFallbackUsed).toBe(true); + expect(parsed.requests[0]?.upstreamProxyFailClosedReason).toBeNull(); expect(parsed.requests[0]?.useragent).toBe("Mozilla/5.0"); expect(parsed.requests[0]?.useragentGroup).toBe("Mozilla"); expect(parsed.requests[0]?.clientIp).toBe("203.0.113.7"); diff --git a/frontend/src/features/dashboard/schemas.ts b/frontend/src/features/dashboard/schemas.ts index ec47088615..7530cd4fae 100644 --- a/frontend/src/features/dashboard/schemas.ts +++ b/frontend/src/features/dashboard/schemas.ts @@ -64,6 +64,7 @@ const DashboardMetricsSchema = z.object({ cachedInputTokens: z.number().nullable(), errorRate: z.number().nullable(), errorCount: z.number().nullable(), + cancelledCount: z.number().int().nonnegative().nullable().optional(), topError: z.string().nullable(), conversations: z.number().int().nullable().optional().default(null), conversationRequests: z.number().int().nonnegative().optional().default(0), @@ -183,6 +184,11 @@ export const RequestLogSchema = z.object({ modelSourceKind: z.string().nullable().optional(), transport: z.string().nullable().optional().default(null), upstreamTransport: z.string().nullable().optional(), + upstreamProxyRouteMode: z.string().nullable().optional(), + upstreamProxyPoolId: z.string().nullable().optional(), + upstreamProxyEndpointId: z.string().nullable().optional(), + upstreamProxyFallbackUsed: z.boolean().nullable().optional(), + upstreamProxyFailClosedReason: z.string().nullable().optional(), useragent: z.string().nullable().optional().default(null), useragentGroup: z.string().nullable().optional().default(null), clientIp: z.string().nullable().optional().default(null), diff --git a/frontend/src/features/firewall/components/firewall-section.tsx b/frontend/src/features/firewall/components/firewall-section.tsx index 0ac64b5cf2..d47ddb7fd0 100644 --- a/frontend/src/features/firewall/components/firewall-section.tsx +++ b/frontend/src/features/firewall/components/firewall-section.tsx @@ -60,7 +60,7 @@ export function FirewallSection({ disabled = false }: FirewallSectionProps) { }; return ( -
+
+ + {draft.supportsReasoning ? ( +
+
+
{t("modelSources.fields.reasoningEfforts")}
+

+ {t("modelSources.fields.reasoningEffortsDescription")} +

+
+ +
+ + { + const reasoningEffortsInput = event.target.value; + const reasoningEfforts = parseReasoningEffortsInput(reasoningEffortsInput); + updateDraft({ + reasoningEffortsInput, + reasoningEfforts, + defaultReasoningEffort: reasoningEfforts.includes(draft.defaultReasoningEffort) + ? draft.defaultReasoningEffort + : (reasoningEfforts[0] ?? ""), + }); + }} + placeholder={t("modelSources.fields.reasoningEffortsPlaceholder")} + autoComplete="off" + /> +
+ +
+ + +
+
+ ) : null} ); } diff --git a/frontend/src/features/model-sources/components/model-source-form.test.ts b/frontend/src/features/model-sources/components/model-source-form.test.ts new file mode 100644 index 0000000000..4485a631f2 --- /dev/null +++ b/frontend/src/features/model-sources/components/model-source-form.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { mergeReasoningMetadata, parseReasoningEffortsInput } from "./model-source-form"; + +describe("model-source-form reasoning effort normalization", () => { + it("trims whitespace and preserves casing for effort values", () => { + expect(parseReasoningEffortsInput(" Ultra, xhigh , low ")).toEqual([ + "Ultra", + "xhigh", + "low", + ]); + }); + + it("preserves casing for declared default reasoning effort", () => { + const metadata = mergeReasoningMetadata( + null, + true, + ["Ultra", "provider-specific", "xhigh"], + " provider-specific ", + ); + const parsed = JSON.parse(metadata ?? "{}"); + + expect(parsed.supported_reasoning_levels).toEqual(["Ultra", "provider-specific", "xhigh"]); + expect(parsed.default_reasoning_level).toBe("provider-specific"); + }); +}); diff --git a/frontend/src/features/model-sources/components/model-source-form.ts b/frontend/src/features/model-sources/components/model-source-form.ts index f2764a15f6..a8590b6740 100644 --- a/frontend/src/features/model-sources/components/model-source-form.ts +++ b/frontend/src/features/model-sources/components/model-source-form.ts @@ -31,10 +31,14 @@ export type ModelSourceDraft = { supportsChatCompletions: boolean; supportsResponses: boolean; supportsAudioTranscriptions: boolean; + supportsEmbeddings: boolean; supportsStreaming: boolean; supportsTools: boolean; supportsVision: boolean; supportsReasoning: boolean; + reasoningEffortsInput: string; + reasoningEfforts: string[]; + defaultReasoningEffort: string; contextWindow: string; maxOutputTokens: string; inputPer1M: string; @@ -47,10 +51,14 @@ export const initialModelSourceDraft: ModelSourceDraft = { supportsChatCompletions: true, supportsResponses: false, supportsAudioTranscriptions: false, + supportsEmbeddings: false, supportsStreaming: true, supportsTools: false, supportsVision: false, supportsReasoning: false, + reasoningEffortsInput: "", + reasoningEfforts: [], + defaultReasoningEffort: "", contextWindow: "", maxOutputTokens: "", inputPer1M: "", @@ -84,9 +92,44 @@ function parseNonNegativeFloat(value: string): number | undefined { // model's raw metadata JSON, which the proxy reads to pass reasoning fields // through and to advertise supports_reasoning in /v1/models. Merge it into // any raw metadata the model already carries so other keys survive edits. -export function mergeReasoningFlag( +const DEFAULT_REASONING_EFFORTS = ["low", "medium", "high"]; + +function normalizeReasoningEffort(value: string): string { + return value.trim(); +} + +function dedupeReasoningEfforts(values: Iterable): string[] { + const seen = new Set(); + const result: string[] = []; + for (const value of values) { + const normalized = normalizeReasoningEffort(value); + if (!normalized || seen.has(normalized)) continue; + seen.add(normalized); + result.push(normalized); + } + return result; +} + +export function parseReasoningEffortsInput(value: string): string[] { + return dedupeReasoningEfforts(value.split(/[\n,]/)); +} + +function normalizeDefaultReasoningEffort( + reasoningEfforts: string[], + defaultReasoningEffort: string, +): string { + const normalizedDefault = normalizeReasoningEffort(defaultReasoningEffort); + if (normalizedDefault && reasoningEfforts.includes(normalizedDefault)) { + return normalizedDefault; + } + return reasoningEfforts[0] ?? ""; +} + +export function mergeReasoningMetadata( existing: string | null | undefined, supportsReasoning: boolean, + reasoningEfforts: string[] = [], + defaultReasoningEffort = "", ): string | null { let metadata: Record = {}; if (existing) { @@ -101,8 +144,20 @@ export function mergeReasoningFlag( } if (supportsReasoning) { metadata.supports_reasoning = true; + if (reasoningEfforts.length > 0) { + metadata.supported_reasoning_levels = reasoningEfforts; + metadata.default_reasoning_level = normalizeDefaultReasoningEffort( + reasoningEfforts, + defaultReasoningEffort, + ); + } else { + delete metadata.supported_reasoning_levels; + delete metadata.default_reasoning_level; + } } else { delete metadata.supports_reasoning; + delete metadata.supported_reasoning_levels; + delete metadata.default_reasoning_level; } return Object.keys(metadata).length > 0 ? JSON.stringify(metadata) : null; } @@ -135,7 +190,12 @@ export function modelInputsFromForm( cachedInputPer1M: cachedInputPer1M ?? null, outputPer1M: outputPer1M ?? null, audioPerMinute: audioPerMinute ?? null, - rawMetadataJson: mergeReasoningFlag(existingRawMetadata[model], draft.supportsReasoning), + rawMetadataJson: mergeReasoningMetadata( + existingRawMetadata[model], + draft.supportsReasoning, + draft.reasoningEfforts, + draft.defaultReasoningEffort, + ), isEnabled: existingEnabledByModel[model] ?? true, })); } @@ -147,30 +207,70 @@ function numberToInput(value: number | null | undefined): string { // Derive the shared draft from an existing source. The create UI applies one // set of per-model settings to every model, so editing mirrors that by reading // the first model's values as the representative settings. -function rawMetadataHasReasoning(rawMetadataJson: string | null | undefined): boolean { - if (!rawMetadataJson) return false; +function parseReasoningMetadata(rawMetadataJson: string | null | undefined): { + supportsReasoning: boolean; + reasoningEffortsInput: string; + reasoningEfforts: string[]; + defaultReasoningEffort: string; +} { + const fallback = { + supportsReasoning: false, + reasoningEffortsInput: "", + reasoningEfforts: [], + defaultReasoningEffort: "", + }; + if (!rawMetadataJson) return fallback; try { const parsed: unknown = JSON.parse(rawMetadataJson); - return ( - typeof parsed === "object" && - parsed !== null && - (parsed as Record).supports_reasoning === true - ); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return fallback; + } + const metadata = parsed as Record; + const supportsReasoning = metadata.supports_reasoning === true; + const declaredLevels = Array.isArray(metadata.supported_reasoning_levels) + ? dedupeReasoningEfforts( + metadata.supported_reasoning_levels.flatMap((value): string[] => { + if (typeof value === "string") return [value]; + if (typeof value !== "object" || value === null || Array.isArray(value)) return []; + const effort = (value as Record).effort; + return typeof effort === "string" ? [effort] : []; + }), + ) + : []; + const reasoningEfforts = + supportsReasoning && declaredLevels.length === 0 ? DEFAULT_REASONING_EFFORTS : declaredLevels; + + return { + supportsReasoning, + reasoningEffortsInput: reasoningEfforts.join(", "), + reasoningEfforts, + defaultReasoningEffort: normalizeDefaultReasoningEffort( + reasoningEfforts, + typeof metadata.default_reasoning_level === "string" + ? metadata.default_reasoning_level + : "", + ), + }; } catch { - return false; + return fallback; } } export function draftFromSource(source: ModelSource): ModelSourceDraft { const firstModel = source.models[0]; + const reasoningMetadata = parseReasoningMetadata(firstModel?.rawMetadataJson); return { supportsChatCompletions: source.supportsChatCompletions, supportsResponses: source.supportsResponses, supportsAudioTranscriptions: source.supportsAudioTranscriptions, + supportsEmbeddings: source.supportsEmbeddings, supportsStreaming: firstModel?.supportsStreaming ?? true, supportsTools: firstModel?.supportsTools ?? false, supportsVision: firstModel?.supportsVision ?? false, - supportsReasoning: rawMetadataHasReasoning(firstModel?.rawMetadataJson), + supportsReasoning: reasoningMetadata.supportsReasoning, + reasoningEffortsInput: reasoningMetadata.reasoningEffortsInput, + reasoningEfforts: reasoningMetadata.reasoningEfforts, + defaultReasoningEffort: reasoningMetadata.defaultReasoningEffort, contextWindow: numberToInput(firstModel?.contextWindow), maxOutputTokens: numberToInput(firstModel?.maxOutputTokens), inputPer1M: numberToInput(firstModel?.inputPer1M), diff --git a/frontend/src/features/model-sources/components/model-source-multi-select.tsx b/frontend/src/features/model-sources/components/model-source-multi-select.tsx index ec469f4c98..204424fbc4 100644 --- a/frontend/src/features/model-sources/components/model-source-multi-select.tsx +++ b/frontend/src/features/model-sources/components/model-source-multi-select.tsx @@ -26,6 +26,7 @@ function sourceSubtitle(source: ModelSource, t: ReturnType value !== null); } diff --git a/frontend/src/features/model-sources/schemas.test.ts b/frontend/src/features/model-sources/schemas.test.ts index d3043a8630..bb8f1e3058 100644 --- a/frontend/src/features/model-sources/schemas.test.ts +++ b/frontend/src/features/model-sources/schemas.test.ts @@ -9,10 +9,8 @@ import { const ISO = "2026-01-01T00:00:00+00:00"; -describe("ModelSourceSchema", () => { - it("parses model source payload", () => { - const parsed = ModelSourceSchema.parse({ - id: "src_vllm", +const BASE_SOURCE = { + id: "src_vllm", name: "vLLM", kind: "openai_compatible", baseUrl: "http://localhost:8000/v1", @@ -21,6 +19,7 @@ describe("ModelSourceSchema", () => { supportsChatCompletions: true, supportsResponses: false, supportsAudioTranscriptions: true, + supportsEmbeddings: true, timeoutSeconds: null, maxConcurrency: null, createdAt: ISO, @@ -44,13 +43,27 @@ describe("ModelSourceSchema", () => { createdAt: ISO, updatedAt: ISO, }, - ], - }); + ], +}; + +describe("ModelSourceSchema", () => { + it("parses model source payload", () => { + const parsed = ModelSourceSchema.parse(BASE_SOURCE); expect(parsed.id).toBe("src_vllm"); expect(parsed.supportsAudioTranscriptions).toBe(true); + expect(parsed.supportsEmbeddings).toBe(true); expect(parsed.models[0].model).toBe("local-coder"); }); + + it("defaults supportsEmbeddings to false when the field is absent", () => { + const withoutEmbeddings: Record = { ...BASE_SOURCE }; + delete withoutEmbeddings.supportsEmbeddings; + + const parsed = ModelSourceSchema.parse(withoutEmbeddings); + + expect(parsed.supportsEmbeddings).toBe(false); + }); }); describe("ModelSourcesResponseSchema", () => { @@ -70,10 +83,12 @@ describe("ModelSourceCreateRequestSchema", () => { supportsChatCompletions: true, supportsResponses: true, supportsAudioTranscriptions: true, + supportsEmbeddings: true, models: [{ model: "deepseek-v4-flash" }], }); expect(parsed.supportsAudioTranscriptions).toBe(true); + expect(parsed.supportsEmbeddings).toBe(true); expect(parsed.models[0].model).toBe("deepseek-v4-flash"); }); }); diff --git a/frontend/src/features/model-sources/schemas.ts b/frontend/src/features/model-sources/schemas.ts index 666f6ada89..c5fab121b2 100644 --- a/frontend/src/features/model-sources/schemas.ts +++ b/frontend/src/features/model-sources/schemas.ts @@ -30,6 +30,7 @@ export const ModelSourceSchema = z.object({ supportsChatCompletions: z.boolean(), supportsResponses: z.boolean(), supportsAudioTranscriptions: z.boolean().default(false), + supportsEmbeddings: z.boolean().default(false), timeoutSeconds: z.number().int().positive().nullable().default(null), maxConcurrency: z.number().int().positive().nullable().default(null), createdAt: z.iso.datetime({ offset: true }), @@ -64,6 +65,7 @@ export const ModelSourceCreateRequestSchema = z.object({ supportsChatCompletions: z.boolean().optional(), supportsResponses: z.boolean().optional(), supportsAudioTranscriptions: z.boolean().optional(), + supportsEmbeddings: z.boolean().optional(), timeoutSeconds: z.number().int().positive().nullable().optional(), maxConcurrency: z.number().int().positive().nullable().optional(), models: z.array(ModelSourceModelInputSchema).default([]), @@ -77,6 +79,7 @@ export const ModelSourceUpdateRequestSchema = z.object({ supportsChatCompletions: z.boolean().optional(), supportsResponses: z.boolean().optional(), supportsAudioTranscriptions: z.boolean().optional(), + supportsEmbeddings: z.boolean().optional(), timeoutSeconds: z.number().int().positive().nullable().optional(), maxConcurrency: z.number().int().positive().nullable().optional(), models: z.array(ModelSourceModelInputSchema).optional(), diff --git a/frontend/src/features/reports/api.ts b/frontend/src/features/reports/api.ts index 0f7ae68037..db953a6540 100644 --- a/frontend/src/features/reports/api.ts +++ b/frontend/src/features/reports/api.ts @@ -5,6 +5,7 @@ export type ReportsParams = { startDate?: string; endDate?: string; accountId?: string[]; + apiKeyId?: string[]; model?: string; useragent?: string; timezone?: string; @@ -22,6 +23,11 @@ export function getReports(params: ReportsParams = {}) { query.append("account_id", id); } } + if (params.apiKeyId) { + for (const id of params.apiKeyId) { + query.append("api_key_id", id); + } + } const suffix = query.size > 0 ? `?${query.toString()}` : ""; return get(`/api/reports${suffix}`, ReportsResponseSchema); } diff --git a/frontend/src/features/reports/components/cost-per-day-chart.test.tsx b/frontend/src/features/reports/components/cost-per-day-chart.test.tsx index 8e999b79c5..94ed30a1a7 100644 --- a/frontend/src/features/reports/components/cost-per-day-chart.test.tsx +++ b/frontend/src/features/reports/components/cost-per-day-chart.test.tsx @@ -1,10 +1,28 @@ -import type { ReactNode } from "react"; -import { render } from "@testing-library/react"; +import type { ReactElement, ReactNode } from "react"; +import { render, screen } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { CostPerDayChart } from "./cost-per-day-chart"; -let capturedProps: { margin?: unknown; data?: unknown } | null = null; +type ChartProps = { children: ReactNode; margin?: unknown; data?: unknown }; + +let capturedProps: ChartProps | null = null; +let capturedYAxisProps: { tickFormatter?: (value: number) => string } | null = null; + +function findTooltipContent(node: ReactNode): ReactElement<{ formatValue?: (value: number) => string }> | null { + if (!node || typeof node !== "object") return null; + if (Array.isArray(node)) { + for (const child of node) { + const found = findTooltipContent(child); + if (found) return found; + } + return null; + } + + const element = node as ReactElement<{ content?: ReactElement<{ formatValue?: (value: number) => string }>; children?: ReactNode }>; + if (element.props.content) return element.props.content; + return findTooltipContent(element.props.children); +} vi.mock("@/components/lazy-recharts", async (importOriginal) => { const actual = await importOriginal(); @@ -14,11 +32,14 @@ vi.mock("@/components/lazy-recharts", async (importOriginal) => { ResponsiveContainer: ({ children }: { children: ReactNode }) =>
{children}
, AreaChart: (props: { children: ReactNode; margin?: unknown; data?: unknown }) => { capturedProps = props; - return
; + return
{props.children}
; }, Area: () => null, XAxis: () => null, - YAxis: () => null, + YAxis: (props: { tickFormatter?: (value: number) => string }) => { + capturedYAxisProps = props; + return null; + }, CartesianGrid: () => null, Tooltip: () => null, }; @@ -27,6 +48,7 @@ vi.mock("@/components/lazy-recharts", async (importOriginal) => { describe("CostPerDayChart", () => { beforeEach(() => { capturedProps = null; + capturedYAxisProps = null; }); it("uses equal left and right chart margins", () => { @@ -41,9 +63,11 @@ describe("CostPerDayChart", () => { conversations: 0, inputTokens: 5_400_000, outputTokens: 59_000, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 3.77, activeAccounts: 2, + cancelledCount: 0, errorCount: 0, }, ]} @@ -53,6 +77,34 @@ describe("CostPerDayChart", () => { expect(capturedProps?.margin).toEqual({ top: 5, right: 10, left: 10, bottom: 0 }); }); + it("formats full-value Cost axis and tooltip amounts with grouping separators", () => { + render( + , + ); + + const tooltip = findTooltipContent(capturedProps?.children); + expect(capturedYAxisProps?.tickFormatter?.(1400)).toBe("$1,400.00"); + expect(tooltip?.props.formatValue?.(1400)).toBe("$1,400.00"); + }); + it("fills missing selected days with zero-value rows", () => { render( { conversations: 0, inputTokens: 5_400_000, outputTokens: 59_000, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 3.77, activeAccounts: 2, + cancelledCount: 0, errorCount: 0, }, { @@ -76,9 +130,11 @@ describe("CostPerDayChart", () => { conversations: 0, inputTokens: 6_800_000, outputTokens: 73_000, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 4.54, activeAccounts: 2, + cancelledCount: 0, errorCount: 0, }, ]} @@ -91,4 +147,13 @@ describe("CostPerDayChart", () => { { date: "06-07", cost: 4.54 }, ]); }); + + it("shows no-data instead of a zero-filled series when daily rows are absent", () => { + render(); + + expect(screen.getByText("No data")).toBeInTheDocument(); + expect(screen.getByText("No usage recorded for the selected range.")).toBeInTheDocument(); + expect(capturedProps).toBeNull(); + expect(screen.queryByTestId("cost-area-chart")).not.toBeInTheDocument(); + }); }); diff --git a/frontend/src/features/reports/components/cost-per-day-chart.tsx b/frontend/src/features/reports/components/cost-per-day-chart.tsx index 23d8758fb9..2236c48357 100644 --- a/frontend/src/features/reports/components/cost-per-day-chart.tsx +++ b/frontend/src/features/reports/components/cost-per-day-chart.tsx @@ -11,7 +11,9 @@ import { } from "@/components/lazy-recharts"; import type { DailyReportRow } from "../schemas"; import { buildContinuousDailyRows } from "../daily-series"; +import { formatCurrency } from "@/utils/formatters"; import { ChartTooltip } from "./chart-tooltip"; +import { ReportChartCard } from "./report-chart-card"; export type CostPerDayChartProps = { startDate: string; @@ -27,9 +29,7 @@ export function CostPerDayChart({ startDate, endDate, data }: CostPerDayChartPro })); return ( -
-
{t("reports.charts.costByDay")}
-
+ @@ -49,10 +49,10 @@ export function CostPerDayChart({ startDate, endDate, data }: CostPerDayChartPro tick={{ fontSize: 10, fill: "var(--muted-foreground)" }} axisLine={false} tickLine={false} - tickFormatter={(v: number) => `$${v}`} + tickFormatter={formatCurrency} /> `$${v.toFixed(2)}`} />} + content={} /> -
-
+ ); } diff --git a/frontend/src/features/reports/components/daily-detail-table.test.tsx b/frontend/src/features/reports/components/daily-detail-table.test.tsx index 2968522c7f..9f79df2eaa 100644 --- a/frontend/src/features/reports/components/daily-detail-table.test.tsx +++ b/frontend/src/features/reports/components/daily-detail-table.test.tsx @@ -3,8 +3,33 @@ import { act, cleanup, render, screen, within } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useDateDisplayFormatStore } from "@/hooks/use-date-format"; import { formatReportBucketDate } from "../date"; - -import { DailyDetailTable } from "./daily-detail-table"; +import { buildContinuousDailyRows } from "../daily-series"; +import type { DailyReportRow } from "../schemas"; + +import { + DailyDetailTable as DailyDetailTableImpl, + type DailyDetailTableProps, +} from "./daily-detail-table"; + +type DailyDetailTableFixtureRow = Omit< + DailyReportRow, + "cancelledCount" | "reasoningTokens" +> & { + cancelledCount?: number; + reasoningTokens?: number | null; +}; + +function DailyDetailTable({ + data, + ...props +}: Omit & { data: DailyDetailTableFixtureRow[] }) { + return ( + ({ cancelledCount: 0, reasoningTokens: 0, ...row }))} + /> + ); +} beforeEach(() => { useDateDisplayFormatStore.setState({ dateDisplayFormat: "default" }); @@ -98,6 +123,55 @@ describe("DailyDetailTable", () => { ); }); + it("renders grouped currency in full-value Cost cells", () => { + render( + , + ); + + expect(within(screen.getByTestId("daily-breakdown-row-2026-06-05")).getByText("$1,400.00")).toBeInTheDocument(); + }); + + it("zero-fills cancelled counts for dates missing from the response", () => { + const rows = buildContinuousDailyRows("2026-06-05", "2026-06-06", [ + { + date: "2026-06-05", + requests: 4, + conversations: 0, + inputTokens: 100, + outputTokens: 20, + reasoningTokens: 12, + cachedInputTokens: 0, + costUsd: 1, + activeAccounts: 1, + errorCount: 1, + cancelledCount: 2, + }, + ]); + + expect(Reflect.get(rows[0] ?? {}, "cancelledCount")).toBe(2); + expect(Reflect.get(rows[1] ?? {}, "cancelledCount")).toBe(0); + expect(rows[0]?.reasoningTokens).toBe(12); + expect(rows[1]?.reasoningTokens).toBe(0); + expect(rows[0]?.requests).toBe(4); + expect(rows[0]?.errorCount).toBe(1); + }); + it("renders existing rows when a date bound is cleared", () => { render( { ]); }); - it("exports csv rows in chronological order regardless of visible sort", async () => { + it("renders requests, cancelled, and errors as distinct daily values", () => { + render( + , + ); + + expect.soft(screen.queryByRole("columnheader", { name: "Reqs" })).toBeInTheDocument(); + expect.soft(screen.queryByRole("columnheader", { name: "Cancelled" })).toBeInTheDocument(); + expect.soft(screen.queryByRole("columnheader", { name: "Errors" })).toBeInTheDocument(); + + const row = screen.getByTestId("daily-breakdown-row-2026-06-05"); + const cells = Array.from(row.querySelectorAll("td"), (cell) => cell.textContent?.trim()); + expect.soft(cells).toContain("4"); + expect.soft(cells).toContain("2"); + expect.soft(cells).toContain("1"); + }); + + it("exports localized cancellation values and preserves requests and errors", async () => { const user = userEvent.setup(); const blobText = vi.fn(async () => ""); const createObjectURL = vi.spyOn(URL, "createObjectURL").mockImplementation((blob) => { @@ -246,17 +353,67 @@ describe("DailyDetailTable", () => { render( , + ); + + await user.click(screen.getByRole("button", { name: /csv/i })); + + expect(createObjectURL).toHaveBeenCalledOnce(); + expect(clickSpy).toHaveBeenCalledOnce(); + expect(revokeObjectURL).toHaveBeenCalledWith("blob:daily-breakdown"); + await expect(blobText()).resolves.toBe( + [ + "Date,Requests,Conversations,Input Tokens,Output Tokens,Reported Reasoning Tokens,Cached Tokens,Cost USD,Active Accounts,Cancelled,Errors", + "2026-06-05,4,0,100,20,12,1,1.0000,3,2,1", + "2026-06-06,0,0,0,0,0,0,0.0000,0,0,0", + ].join("\n"), + ); + }); + + it("renders and exports unknown reasoning separately from known zero and sorts unknown last", async () => { + const user = userEvent.setup(); + const blobText = vi.fn(async () => ""); + vi.spyOn(URL, "createObjectURL").mockImplementation((blob) => { + if (!(blob instanceof Blob)) { + throw new TypeError("expected Blob export payload"); + } + blobText.mockImplementation(() => blob.text()); + return "blob:nullable-reasoning"; + }); + vi.spyOn(URL, "revokeObjectURL").mockImplementation(() => {}); + vi.spyOn(HTMLAnchorElement.prototype, "click").mockImplementation(() => {}); + + render( + { conversations: 0, inputTokens: 200, outputTokens: 30, - cachedInputTokens: 2, + reasoningTokens: 0, + cachedInputTokens: 0, costUsd: 2, activeAccounts: 1, errorCount: 0, }, { date: "2026-06-07", - requests: 5, + requests: 3, conversations: 0, inputTokens: 300, outputTokens: 40, - cachedInputTokens: 3, + reasoningTokens: 5, + cachedInputTokens: 0, costUsd: 3, - activeAccounts: 2, + activeAccounts: 1, errorCount: 0, }, ]} />, ); - await user.click(screen.getByRole("button", { name: /reqs/i })); - await user.click(screen.getByRole("button", { name: /csv/i })); + const unknownCells = screen + .getByTestId("daily-breakdown-row-2026-06-05") + .querySelectorAll("td"); + const zeroCells = screen + .getByTestId("daily-breakdown-row-2026-06-06") + .querySelectorAll("td"); + expect(unknownCells[5]?.textContent?.trim()).toBe("—"); + expect(zeroCells[5]?.textContent?.trim()).toBe("0"); - expect(createObjectURL).toHaveBeenCalledOnce(); - expect(clickSpy).toHaveBeenCalledOnce(); - expect(revokeObjectURL).toHaveBeenCalledWith("blob:daily-breakdown"); - await expect(blobText()).resolves.toBe( - [ - "Date,Requests,Conversations,Input Tokens,Output Tokens,Cached Tokens,Cost USD,Active Accounts,Errors", - "2026-06-05,8,0,100,20,1,1.0000,3,0", - "2026-06-06,2,0,200,30,2,2.0000,1,0", - "2026-06-07,5,0,300,40,3,3.0000,2,0", - ].join("\n"), - ); + await user.click(screen.getByRole("button", { name: "Reported Reasoning Tokens" })); + expect( + screen.getAllByTestId(/daily-breakdown-row-/).map((row) => row.dataset.testid), + ).toEqual([ + "daily-breakdown-row-2026-06-06", + "daily-breakdown-row-2026-06-07", + "daily-breakdown-row-2026-06-05", + ]); + + await user.click(screen.getByRole("button", { name: /csv/i })); + const csvLines = (await blobText()).split("\n"); + expect(csvLines[1]?.split(",")[5]).toBe(""); + expect(csvLines[2]?.split(",")[5]).toBe("0"); + expect(csvLines[3]?.split(",")[5]).toBe("5"); }); it.each([ @@ -306,6 +474,7 @@ describe("DailyDetailTable", () => { ["Reqs", "daily-breakdown-row-2026-06-06"], ["Input Tokens", "daily-breakdown-row-2026-06-05"], ["Output Tokens", "daily-breakdown-row-2026-06-05"], + ["Reported Reasoning Tokens", "daily-breakdown-row-2026-06-06"], ["Cost", "daily-breakdown-row-2026-06-05"], ["Accounts", "daily-breakdown-row-2026-06-06"], ])("sorts by %s when its header is clicked", async (headerLabel, expectedFirstRow) => { @@ -323,6 +492,7 @@ describe("DailyDetailTable", () => { conversations: 0, inputTokens: 100, outputTokens: 20, + reasoningTokens: 5, cachedInputTokens: 0, costUsd: 1, activeAccounts: 3, @@ -334,6 +504,7 @@ describe("DailyDetailTable", () => { conversations: 0, inputTokens: 200, outputTokens: 30, + reasoningTokens: 1, cachedInputTokens: 0, costUsd: 2, activeAccounts: 1, @@ -345,6 +516,7 @@ describe("DailyDetailTable", () => { conversations: 0, inputTokens: 300, outputTokens: 40, + reasoningTokens: 3, cachedInputTokens: 0, costUsd: 3, activeAccounts: 2, @@ -510,9 +682,9 @@ describe("DailyDetailTable", () => { startDate="2026-06-05" endDate="2026-06-07" data={[ - { date: "2026-06-05", requests: 8, conversations: 1, inputTokens: 100, outputTokens: 20, cachedInputTokens: 0, costUsd: 1, activeAccounts: 1, errorCount: 0 }, - { date: "2026-06-06", requests: 2, conversations: 5, inputTokens: 200, outputTokens: 30, cachedInputTokens: 1, costUsd: 2, activeAccounts: 1, errorCount: 0 }, - { date: "2026-06-07", requests: 5, conversations: 3, inputTokens: 300, outputTokens: 40, cachedInputTokens: 2, costUsd: 3, activeAccounts: 1, errorCount: 0 }, + { date: "2026-06-05", requests: 8, conversations: 1, inputTokens: 100, outputTokens: 20, cachedInputTokens: 0, costUsd: 1, activeAccounts: 1, cancelledCount: 0, errorCount: 0 }, + { date: "2026-06-06", requests: 2, conversations: 5, inputTokens: 200, outputTokens: 30, cachedInputTokens: 1, costUsd: 2, activeAccounts: 1, cancelledCount: 0, errorCount: 0 }, + { date: "2026-06-07", requests: 5, conversations: 3, inputTokens: 300, outputTokens: 40, cachedInputTokens: 2, costUsd: 3, activeAccounts: 1, cancelledCount: 0, errorCount: 0 }, ]} />, ); @@ -530,15 +702,15 @@ describe("DailyDetailTable", () => { const headerRow = screen.getAllByRole("row")[0]; const headerCells = Array.from(headerRow?.querySelectorAll("th") ?? []); const labels = headerCells.map((c) => c.textContent?.trim() ?? ""); - expect(labels).toEqual(["Day", "Reqs", "Conversations", "Input Tokens", "Output Tokens", "Cost", "Accounts"]); + expect(labels).toEqual(["Day", "Reqs", "Conversations", "Input Tokens", "Output Tokens", "Reported Reasoning Tokens", "Cost", "Accounts", "Cancelled", "Errors"]); // CSV: full header + first data row with Conversations between Requests and Input Tokens await user.click(screen.getByRole("button", { name: /csv/i })); const csv = await blobText(); const csvLines = csv.split("\n"); - expect(csvLines[0]).toBe("Date,Requests,Conversations,Input Tokens,Output Tokens,Cached Tokens,Cost USD,Active Accounts,Errors"); + expect(csvLines[0]).toBe("Date,Requests,Conversations,Input Tokens,Output Tokens,Reported Reasoning Tokens,Cached Tokens,Cost USD,Active Accounts,Cancelled,Errors"); // First data row in CSV (chronological: 06-05 first, conversations=1) - expect(csvLines[1]).toMatch(/2026-06-05,8,1,100,20,0,1\.0000,1,0/); + expect(csvLines[1]).toMatch(/2026-06-05,8,1,100,20,0,0,1\.0000,1,0,0/); }); it("zero-filled gap rows have conversations=0 in column 2", () => { @@ -563,7 +735,7 @@ describe("DailyDetailTable", () => { expect(dataCells[2]?.textContent?.trim()).toBe("3"); }); - it("both header and body tables share min-width for mobile overflow", () => { + it("keeps headers and rows in one horizontally scrollable table", () => { render( { />, ); - const tables = document.querySelectorAll("table.min-w-\\[700px\\]"); - expect(tables.length).toBe(2); + const scrollContainer = screen.getByTestId("daily-breakdown-scroll-body"); + const tables = scrollContainer.querySelectorAll("table.min-w-\\[1000px\\]"); + expect(scrollContainer).toHaveClass("overflow-x-auto", "overflow-y-auto"); + expect(tables).toHaveLength(1); + expect(tables[0]?.querySelector("thead")).toBeInTheDocument(); + expect(tables[0]?.querySelector("tbody")).toBeInTheDocument(); }); }); diff --git a/frontend/src/features/reports/components/daily-detail-table.tsx b/frontend/src/features/reports/components/daily-detail-table.tsx index 32a4d08eb2..29d05e0771 100644 --- a/frontend/src/features/reports/components/daily-detail-table.tsx +++ b/frontend/src/features/reports/components/daily-detail-table.tsx @@ -7,6 +7,7 @@ import { useDateDisplayFormatStore } from "@/hooks/use-date-format"; import { buildContinuousDailyRows } from "../daily-series"; import type { DailyReportRow } from "../schemas"; import { formatReportBucketDate } from "../date"; +import { formatCurrency } from "@/utils/formatters"; export type DailyDetailTableProps = { startDate: string; @@ -16,7 +17,7 @@ export type DailyDetailTableProps = { const DAILY_BREAKDOWN_SCROLL_HEIGHT_CLASS = "max-h-[17.5rem]"; -type SortKey = "date" | "requests" | "conversations" | "inputTokens" | "outputTokens" | "costUsd" | "activeAccounts"; +type SortKey = "date" | "requests" | "conversations" | "inputTokens" | "outputTokens" | "reasoningTokens" | "costUsd" | "activeAccounts" | "cancelledCount" | "errorCount"; type SortDirection = "asc" | "desc"; function formatTokens(v: number): string { @@ -58,10 +59,13 @@ export function DailyDetailTable({ startDate, endDate, data }: DailyDetailTableP {t("reports.dailyBreakdown.csv")}
-
-
+
+
- + toggleSort("outputTokens")} /> + toggleSort("reasoningTokens")} + /> toggleSort("activeAccounts")} /> + toggleSort("cancelledCount")} + /> + toggleSort("errorCount")} + /> + + {rows.map((row) => ( + + + + + + + + + + + + + ))} +
+ {formatReportBucketDate(row.date, dateDisplayFormat)} + {row.requests} + {row.conversations} + + {formatTokens(row.inputTokens)}{" "} + + ({formatTokens(row.cachedInputTokens)}) + + + {formatTokens(row.outputTokens)} + + {row.reasoningTokens == null ? "—" : formatTokens(row.reasoningTokens)} + + {formatCurrency(row.costUsd)} + + {row.activeAccounts} + + {row.cancelledCount} + {row.errorCount}
-
- - - - {rows.map((row) => ( - - - - - - - - - - ))} - -
- {formatReportBucketDate(row.date, dateDisplayFormat)} - - {row.requests} - - {row.conversations} - - {formatTokens(row.inputTokens)}{" "} - - ({formatTokens(row.cachedInputTokens)}) - - - {formatTokens(row.outputTokens)} - - ${row.costUsd.toFixed(2)} - - {row.activeAccounts} -
-
); @@ -201,13 +220,16 @@ function SortableHeader({ function ColumnGroup() { return ( - + + + - - - - + + + + + ); } @@ -220,6 +242,15 @@ function sortRows( const leftValue = left[sort.key]; const rightValue = right[sort.key]; + if (leftValue == null && rightValue == null) { + return 0; + } + if (leftValue == null) { + return 1; + } + if (rightValue == null) { + return -1; + } if (leftValue < rightValue) { return sort.direction === "asc" ? -1 : 1; } @@ -239,13 +270,15 @@ function exportCSV(rows: DailyReportRow[], t: TFunction) { t("reports.dailyBreakdown.csvColumns.conversations"), t("reports.dailyBreakdown.csvColumns.inputTokens"), t("reports.dailyBreakdown.csvColumns.outputTokens"), + t("reports.dailyBreakdown.csvColumns.reasoningTokens"), t("reports.dailyBreakdown.csvColumns.cachedTokens"), t("reports.dailyBreakdown.csvColumns.costUsd"), t("reports.dailyBreakdown.csvColumns.activeAccounts"), + t("reports.dailyBreakdown.csvColumns.cancelled"), t("reports.dailyBreakdown.csvColumns.errors"), ]; const lines = rows.map((r) => - [r.date, r.requests, r.conversations, r.inputTokens, r.outputTokens, r.cachedInputTokens, r.costUsd.toFixed(4), r.activeAccounts, r.errorCount].join(","), + [r.date, r.requests, r.conversations, r.inputTokens, r.outputTokens, r.reasoningTokens ?? "", r.cachedInputTokens, r.costUsd.toFixed(4), r.activeAccounts, r.cancelledCount, r.errorCount].join(","), ); const csv = [headers.join(","), ...lines].join("\n"); const blob = new Blob([csv], { type: "text/csv;charset=utf-8;" }); diff --git a/frontend/src/features/reports/components/queue-wait-chart.test.tsx b/frontend/src/features/reports/components/queue-wait-chart.test.tsx index 7de125173a..7965ccf0e1 100644 --- a/frontend/src/features/reports/components/queue-wait-chart.test.tsx +++ b/frontend/src/features/reports/components/queue-wait-chart.test.tsx @@ -29,9 +29,11 @@ const BASE_ROW = { conversations: 0, inputTokens: 1_000, outputTokens: 100, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 0.5, activeAccounts: 1, + cancelledCount: 0, errorCount: 0, medianTtftMs: 200, medianTps: 25, diff --git a/frontend/src/features/reports/components/queue-wait-chart.tsx b/frontend/src/features/reports/components/queue-wait-chart.tsx index 31e6f7574b..2093451937 100644 --- a/frontend/src/features/reports/components/queue-wait-chart.tsx +++ b/frontend/src/features/reports/components/queue-wait-chart.tsx @@ -12,6 +12,7 @@ import { import type { DailyReportRow } from "../schemas"; import { buildContinuousDailyRows } from "../daily-series"; import { ChartTooltip } from "./chart-tooltip"; +import { ReportChartCard } from "./report-chart-card"; export type QueueWaitChartProps = { startDate: string; @@ -34,9 +35,7 @@ export function QueueWaitChart({ startDate, endDate, data }: QueueWaitChartProps })); return ( -
-
{t("reports.charts.queueWait")}
-
+ @@ -74,7 +73,6 @@ export function QueueWaitChart({ startDate, endDate, data }: QueueWaitChartProps /> -
-
+ ); } diff --git a/frontend/src/features/reports/components/report-chart-card.tsx b/frontend/src/features/reports/components/report-chart-card.tsx new file mode 100644 index 0000000000..9d14eb3865 --- /dev/null +++ b/frontend/src/features/reports/components/report-chart-card.tsx @@ -0,0 +1,32 @@ +import { BarChart3 } from "lucide-react"; +import type { ReactNode } from "react"; +import { useTranslation } from "react-i18next"; + +import { EmptyState } from "@/components/empty-state"; + +export type ReportChartCardProps = { + title: string; + empty: boolean; + children: ReactNode; +}; + +export function ReportChartCard({ title, empty, children }: ReportChartCardProps) { + const { t } = useTranslation(); + + return ( +
+
{title}
+ {empty ? ( +
+ +
+ ) : ( +
{children}
+ )} +
+ ); +} diff --git a/frontend/src/features/reports/components/reports-filters.test.tsx b/frontend/src/features/reports/components/reports-filters.test.tsx index f4c253ce2a..fb5efb756e 100644 --- a/frontend/src/features/reports/components/reports-filters.test.tsx +++ b/frontend/src/features/reports/components/reports-filters.test.tsx @@ -9,6 +9,7 @@ const FILTERS: ReportsFiltersState = { startDate: "2026-06-01", endDate: "2026-06-07", accountId: [], + apiKeyId: [], model: "", useragent: "", }; @@ -35,6 +36,7 @@ describe("ReportsFilters", () => { filters={FILTERS} selectedPresetDays={7} accountOptions={[{ value: "acc_one", label: "Primary account", isEmail: false }]} + apiKeyOptions={[]} modelOptions={[]} useragentOptions={[]} visibleChartIds={ALL_CHART_IDS} @@ -50,6 +52,30 @@ describe("ReportsFilters", () => { expect(onFiltersChange).toHaveBeenCalledWith({ ...FILTERS, accountId: ["acc_one"] }); }); + it("updates API key filters from the API key selector", async () => { + const user = userEvent.setup(); + const onFiltersChange = vi.fn(); + render( + , + ); + + await user.click(screen.getByRole("button", { name: /api keys/i })); + await user.click(await screen.findByRole("menuitemcheckbox", { name: /dev key · key-123/i })); + + expect(onFiltersChange).toHaveBeenCalledWith({ ...FILTERS, apiKeyId: ["key_one"] }); + }); + it("keeps the reports model filter as a single selected value", async () => { const user = userEvent.setup(); const onFiltersChange = vi.fn(); @@ -58,6 +84,7 @@ describe("ReportsFilters", () => { filters={{ ...FILTERS, model: "gpt-5.1" }} selectedPresetDays={7} accountOptions={[]} + apiKeyOptions={[]} modelOptions={[ { value: "gpt-5.1", label: "gpt-5.1" }, { value: "gpt-5.2", label: "gpt-5.2" }, @@ -87,6 +114,7 @@ describe("ReportsFilters", () => { filters={{ ...FILTERS, useragent: "CLI" }} selectedPresetDays={7} accountOptions={[]} + apiKeyOptions={[]} modelOptions={[]} useragentOptions={[ { value: "CLI", label: "CLI" }, @@ -117,6 +145,7 @@ describe("ReportsFilters", () => { filters={FILTERS} selectedPresetDays={30} accountOptions={[]} + apiKeyOptions={[]} modelOptions={[]} useragentOptions={[]} visibleChartIds={ALL_CHART_IDS} @@ -148,6 +177,7 @@ describe("ReportsFilters", () => { filters={FILTERS} selectedPresetDays={30} accountOptions={[]} + apiKeyOptions={[]} modelOptions={[]} useragentOptions={[]} visibleChartIds={ALL_CHART_IDS} @@ -173,6 +203,7 @@ describe("ReportsFilters", () => { filters={{ ...FILTERS, endDate: "2026-06-13" }} selectedPresetDays={null} accountOptions={[]} + apiKeyOptions={[]} modelOptions={[]} useragentOptions={[]} visibleChartIds={ALL_CHART_IDS} @@ -192,6 +223,7 @@ describe("ReportsFilters", () => { filters={{ ...FILTERS, startDate: "2026-06-08" }} selectedPresetDays={null} accountOptions={[]} + apiKeyOptions={[]} modelOptions={[]} useragentOptions={[]} visibleChartIds={ALL_CHART_IDS} @@ -219,6 +251,7 @@ describe("ReportsFilters", () => { filters={FILTERS} selectedPresetDays={7} accountOptions={[]} + apiKeyOptions={[]} modelOptions={[]} useragentOptions={[]} visibleChartIds={ALL_CHART_IDS} @@ -237,6 +270,7 @@ describe("ReportsFilters", () => { filters={FILTERS} selectedPresetDays={7} accountOptions={[]} + apiKeyOptions={[]} modelOptions={[]} useragentOptions={[]} visibleChartIds={ALL_CHART_IDS} @@ -262,6 +296,7 @@ describe("ReportsFilters", () => { filters={FILTERS} selectedPresetDays={7} accountOptions={[]} + apiKeyOptions={[]} modelOptions={[]} useragentOptions={[]} visibleChartIds={ALL_CHART_IDS} @@ -286,6 +321,7 @@ describe("ReportsFilters", () => { filters={FILTERS} selectedPresetDays={7} accountOptions={[]} + apiKeyOptions={[]} modelOptions={[]} useragentOptions={[]} visibleChartIds={ALL_CHART_IDS} @@ -310,6 +346,7 @@ describe("ReportsFilters", () => { filters={FILTERS} selectedPresetDays={7} accountOptions={[]} + apiKeyOptions={[]} modelOptions={[]} useragentOptions={[]} visibleChartIds={[]} diff --git a/frontend/src/features/reports/components/reports-filters.tsx b/frontend/src/features/reports/components/reports-filters.tsx index a743edee0a..477e2b71b4 100644 --- a/frontend/src/features/reports/components/reports-filters.tsx +++ b/frontend/src/features/reports/components/reports-filters.tsx @@ -15,6 +15,7 @@ export type ReportsFiltersState = { startDate: string; endDate: string; accountId: string[]; + apiKeyId: string[]; model: string; useragent: string; }; @@ -23,6 +24,7 @@ export type ReportsFiltersProps = { filters: ReportsFiltersState; selectedPresetDays: number | null; accountOptions: MultiSelectOption[]; + apiKeyOptions: MultiSelectOption[]; modelOptions: MultiSelectOption[]; useragentOptions: MultiSelectOption[]; visibleChartIds: ReportChartId[]; @@ -41,6 +43,7 @@ export function ReportsFilters({ filters, selectedPresetDays, accountOptions, + apiKeyOptions, modelOptions, useragentOptions, visibleChartIds, @@ -86,6 +89,12 @@ export function ReportsFilters({ options={accountOptions} onChange={(accountId) => onFiltersChange({ ...filters, accountId })} /> + onFiltersChange({ ...filters, apiKeyId })} + /> ({ listAccounts: vi.fn().mockResolvedValue({ accounts: [] }), })); +vi.mock("@/features/dashboard/api", () => ({ + getRequestLogOptions: vi.fn().mockResolvedValue({ accountIds: [], apiKeys: [], modelOptions: [], statuses: [] }), +})); + vi.mock("@/features/reports/hooks/use-reports", () => ({ useReports: vi.fn(), })); @@ -44,8 +49,11 @@ const EMPTY_REPORT: ReportsResponse = { totalCostUsd: 0, totalInputTokens: 0, totalOutputTokens: 0, + totalReasoningTokens: 0, + reasoningUsageKnownRequests: 0, totalCachedTokens: 0, totalRequests: 0, + totalCancelled: 0, totalErrors: 0, totalConversations: 0, activeAccounts: 0, @@ -68,6 +76,7 @@ const EMPTY_REPORT: ReportsResponse = { const useReportsMock = vi.mocked(useReports); const listAccountsMock = vi.mocked(listAccounts); +const getRequestLogOptionsMock = vi.mocked(getRequestLogOptions); const getBrowserReportsTimeZoneMock = vi.mocked(getBrowserReportsTimeZone); type UseReportsMockResult = ReturnType; const REPORTS_TIMEZONE_STORAGE_KEY = "codex-lb-reports-timezone"; @@ -80,9 +89,11 @@ describe("ReportsPage", () => { beforeEach(() => { useReportsMock.mockReset(); listAccountsMock.mockReset(); + getRequestLogOptionsMock.mockReset(); getBrowserReportsTimeZoneMock.mockReset(); window.localStorage.clear(); listAccountsMock.mockResolvedValue({ accounts: [] }); + getRequestLogOptionsMock.mockResolvedValue({ accountIds: [], apiKeys: [], modelOptions: [], statuses: [] }); getBrowserReportsTimeZoneMock.mockReturnValue("America/Los_Angeles"); }); @@ -712,7 +723,8 @@ describe("ReportsPage", () => { ); }); - it("shows account option load failures instead of hiding empty selector silently", async () => { + it("shows account option load failures with a retry button and recovers when retried", async () => { + const user = userEvent.setup(); useReportsMock.mockImplementation(() => asUseReportsResult({ data: EMPTY_REPORT, @@ -721,21 +733,180 @@ describe("ReportsPage", () => { refetch: vi.fn(), }), ); - listAccountsMock.mockRejectedValueOnce( - new Error("accounts backend timeout"), - ); + listAccountsMock + .mockRejectedValueOnce(new Error("accounts backend timeout")) + .mockResolvedValueOnce({ accounts: [] }); renderWithProviders(); + const accountErrorText = await screen.findByText( + /Failed to load account options: accounts backend timeout/i, + ); + expect(accountErrorText).toBeInTheDocument(); expect( - await screen.findByText( - /Failed to load account options: accounts backend timeout/i, - ), + screen + .getAllByRole("button", { name: /accounts/i }) + .find((button) => button.getAttribute("aria-haspopup") === "menu"), ).toBeInTheDocument(); + + const accountErrorContainer = accountErrorText.parentElement!.parentElement!; + const retryButton = within(accountErrorContainer).getByRole("button", { name: /retry/i }); + await user.click(retryButton); + + expect(listAccountsMock).toHaveBeenCalledTimes(2); + await waitFor(() => { + expect( + screen.queryByText(/Failed to load account options:/i), + ).not.toBeInTheDocument(); + }); + }); + + it("forwards initial apiKeyId filter to useReports hook calls", () => { + useReportsMock.mockReturnValue( + asUseReportsResult({ + data: EMPTY_REPORT, + isLoading: false, + isError: false, + refetch: vi.fn(), + }), + ); + + renderWithProviders( + , + ); + + expect(useReportsMock).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + apiKeyId: ["key-123"], + }), + "America/Los_Angeles", + ); + }); + + it("shows API key option load failures with a retry button and recovers when retried", async () => { + const user = userEvent.setup(); + useReportsMock.mockImplementation(() => + asUseReportsResult({ + data: EMPTY_REPORT, + isLoading: false, + isError: false, + refetch: vi.fn(), + }), + ); + getRequestLogOptionsMock + .mockRejectedValueOnce(new Error("api keys backend timeout")) + .mockResolvedValueOnce({ accountIds: [], apiKeys: [], modelOptions: [], statuses: [] }); + + renderWithProviders(); + + const apiKeyErrorText = await screen.findByText( + /Failed to load API key options: api keys backend timeout/i, + ); + expect(apiKeyErrorText).toBeInTheDocument(); expect( screen - .getAllByRole("button", { name: /accounts/i }) + .getAllByRole("button", { name: /api keys/i }) .find((button) => button.getAttribute("aria-haspopup") === "menu"), ).toBeInTheDocument(); + + const apiKeyErrorContainer = apiKeyErrorText.parentElement!.parentElement!; + const retryButton = within(apiKeyErrorContainer).getByRole("button", { name: /retry/i }); + await user.click(retryButton); + + expect(getRequestLogOptionsMock).toHaveBeenCalledTimes(2); + await waitFor(() => { + expect( + screen.queryByText(/Failed to load API key options:/i), + ).not.toBeInTheDocument(); + }); + }); + + it("populates API key options from request log options including deleted keys with ID fallback", async () => { + const user = userEvent.setup(); + useReportsMock.mockImplementation(() => + asUseReportsResult({ + data: EMPTY_REPORT, + isLoading: false, + isError: false, + refetch: vi.fn(), + }), + ); + getRequestLogOptionsMock.mockResolvedValue({ + accountIds: [], + apiKeys: [ + { id: "key_active", name: "Active Key", keyPrefix: "sk-active" }, + { id: "key_deleted", name: "key_deleted", keyPrefix: null }, + ], + modelOptions: [], + statuses: [], + }); + + renderWithProviders(); + + const trigger = await screen.findByRole("button", { name: /api keys/i }); + await user.click(trigger); + + expect(await screen.findByText(/Active Key/i)).toBeInTheDocument(); + expect(await screen.findByText("key_deleted")).toBeInTheDocument(); + }); + + it("exports CSV based on the filtered report dataset when API key filter is active", async () => { + const user = userEvent.setup(); + const blobText = vi.fn(async () => ""); + const createObjectURL = vi.spyOn(URL, "createObjectURL").mockImplementation((blob) => { + if (!(blob instanceof Blob)) { + throw new TypeError("expected Blob export payload"); + } + blobText.mockImplementation(() => blob.text()); + return "blob:daily-breakdown"; + }); + vi.spyOn(URL, "revokeObjectURL").mockImplementation(() => {}); + vi.spyOn(HTMLAnchorElement.prototype, "click").mockImplementation(() => {}); + + useReportsMock.mockReturnValue( + asUseReportsResult({ + data: { + ...EMPTY_REPORT, + daily: [ + { + date: "2030-01-15", + requests: 42, + conversations: 2, + inputTokens: 1000, + outputTokens: 200, + reasoningTokens: 0, + cachedInputTokens: 50, + costUsd: 0.15, + activeAccounts: 1, + cancelledCount: 0, + errorCount: 0, + medianTtftMs: 0, + medianTps: 0, + medianQueueMs: 0, + }, + ], + }, + isLoading: false, + isError: false, + refetch: vi.fn(), + }), + ); + + renderWithProviders( + , + ); + + await user.click(screen.getByRole("button", { name: /csv/i })); + + expect(createObjectURL).toHaveBeenCalledOnce(); + const csvContent = await blobText(); + expect(csvContent).toContain("2030-01-15,42,2,1000,200,0,50,0.1500,1,0,0"); }); }); diff --git a/frontend/src/features/reports/components/reports-page.tsx b/frontend/src/features/reports/components/reports-page.tsx index 1e354556a0..33a24cd457 100644 --- a/frontend/src/features/reports/components/reports-page.tsx +++ b/frontend/src/features/reports/components/reports-page.tsx @@ -5,6 +5,7 @@ import { useTranslation } from "react-i18next"; import { AlertMessage } from "@/components/alert-message"; import { Button } from "@/components/ui/button"; import { listAccounts } from "@/features/accounts/api"; +import { getRequestLogOptions } from "@/features/dashboard/api"; import { useReports } from "@/features/reports/hooks/use-reports"; import { useReportChartVisibility } from "@/features/reports/hooks/use-report-chart-visibility"; import { getErrorMessageOrNull } from "@/utils/errors"; @@ -70,6 +71,7 @@ const createDefaultFilters = (): ReportsFiltersState => ({ startDate: daysAgoLocalISO(6), endDate: localDateISO(), accountId: [], + apiKeyId: [], model: "", useragent: "", }); @@ -129,6 +131,14 @@ export function ReportsPage({ initialFilters }: ReportsPageProps = {}) { queryKey: ["accounts", "reports-filter"], queryFn: listAccounts, }); + const { + data: apiKeysOptionsData, + error: apiKeysError, + refetch: refetchApiKeys, + } = useQuery({ + queryKey: ["request-log-options", "reports-filter"], + queryFn: () => getRequestLogOptions(), + }); const accountOptions = useMemo( () => @@ -144,6 +154,15 @@ export function ReportsPage({ initialFilters }: ReportsPageProps = {}) { [accountsData], ); + const apiKeyOptions = useMemo( + () => + (apiKeysOptionsData?.apiKeys ?? []).map((key) => ({ + value: key.id, + label: key.keyPrefix ? `${key.name} · ${key.keyPrefix}` : key.name, + })), + [apiKeysOptionsData], + ); + const modelOptions = useMemo( () => (filterCatalogQuery.data?.byModel ?? []).map((entry) => ({ @@ -165,14 +184,15 @@ export function ReportsPage({ initialFilters }: ReportsPageProps = {}) { const mainReportsError = getErrorMessageOrNull(reportsQuery.error); const sharedOptionsError = getErrorMessageOrNull(filterCatalogQuery.error); const accountOptionsError = getErrorMessageOrNull(accountsError); + const apiKeyOptionsError = getErrorMessageOrNull(apiKeysError); const hasAnyError = Boolean( - mainReportsError || sharedOptionsError || accountOptionsError, + mainReportsError || sharedOptionsError || accountOptionsError || apiKeyOptionsError, ); const handleRetry = async () => { if (!isReportDateRangeValid(filters.startDate, filters.endDate)) { - await refetchAccounts(); + await Promise.allSettled([refetchAccounts(), refetchApiKeys()]); return; } @@ -180,6 +200,7 @@ export function ReportsPage({ initialFilters }: ReportsPageProps = {}) { reportsQuery.refetch(), filterCatalogQuery.refetch(), refetchAccounts(), + refetchApiKeys(), ]); }; @@ -217,6 +238,7 @@ export function ReportsPage({ initialFilters }: ReportsPageProps = {}) { filters={filters} selectedPresetDays={selectedPresetDays} accountOptions={accountOptions} + apiKeyOptions={apiKeyOptions} modelOptions={modelOptions} useragentOptions={useragentOptions} visibleChartIds={visibleChartIds} @@ -231,14 +253,55 @@ export function ReportsPage({ initialFilters }: ReportsPageProps = {}) { ) : null} {sharedOptionsError ? ( - - {t("reports.errors.options", { error: sharedOptionsError })} - +
+ + {t("reports.errors.options", { error: sharedOptionsError })} + + +
) : null} {accountOptionsError ? ( - - {t("reports.errors.accounts", { error: accountOptionsError })} - +
+ + {t("reports.errors.accounts", { error: accountOptionsError })} + + +
+ ) : null} + {apiKeyOptionsError ? ( +
+ + {t("reports.errors.apiKeys", { error: apiKeyOptionsError })} + + +
) : null} {reportsQuery.isLoading ? ( diff --git a/frontend/src/features/reports/components/reports-summary-cards.test.tsx b/frontend/src/features/reports/components/reports-summary-cards.test.tsx index d282b10ac1..a7b1219173 100644 --- a/frontend/src/features/reports/components/reports-summary-cards.test.tsx +++ b/frontend/src/features/reports/components/reports-summary-cards.test.tsx @@ -2,7 +2,37 @@ import { render, screen, within } from "@testing-library/react"; import { describe, expect, it } from "vitest"; -import { ReportsSummaryCards } from "./reports-summary-cards"; +import type { ReportSummary } from "../schemas"; +import { + ReportsSummaryCards as ReportsSummaryCardsImpl, + type ReportsSummaryCardsProps, +} from "./reports-summary-cards"; + +type ReportsSummaryFixture = Omit< + ReportSummary, + "totalCancelled" | "totalReasoningTokens" | "reasoningUsageKnownRequests" +> & { + totalCancelled?: number; + totalReasoningTokens?: number; + reasoningUsageKnownRequests?: number; +}; + +function ReportsSummaryCards({ + summary, + ...props +}: Omit & { summary: ReportsSummaryFixture }) { + return ( + + ); +} describe("ReportsSummaryCards", () => { it("renders inline comparison badges for cost, tokens, and requests", () => { @@ -12,6 +42,8 @@ describe("ReportsSummaryCards", () => { totalCostUsd: 15, totalInputTokens: 1_600_000_000, totalOutputTokens: 13_000_000, + totalReasoningTokens: 8_000_000, + reasoningUsageKnownRequests: 1400, totalCachedTokens: 990_000_000, totalRequests: 1500, totalErrors: 0, @@ -50,11 +82,74 @@ describe("ReportsSummaryCards", () => { ); expect( - within(tokensCard).getByText("Input 1.6B · Cache 990M · Output 13.0M"), + within(tokensCard).getByText( + "Input 1.6B · Cache 990M · Output 13.0M", + ), ).toBeInTheDocument(); + expect(within(tokensCard).getByText("Reported reasoning 8.0M (included in output) · 1400/1500 requests")).toBeInTheDocument(); + expect(tokensCard.parentElement).toHaveClass("lg:grid-cols-3", "xl:grid-cols-6"); expect(within(requestsCard).getByText("avg 500/day · 3 accounts")).toBeInTheDocument(); }); + it("shows reported reasoning coverage without adding reasoning to the token total", () => { + render( + , + ); + + const tokensCard = screen.getByTestId("report-summary-card-tokens"); + expect(within(tokensCard).getByText("140")).toBeInTheDocument(); + expect( + within(tokensCard).getByText( + "Input 100 · Cache 10 · Output 40", + ), + ).toBeInTheDocument(); + expect(within(tokensCard).getByText("Reported reasoning 30 (included in output) · 2/4 requests")).toBeInTheDocument(); + expect(within(tokensCard).queryByText("170")).not.toBeInTheDocument(); + }); + + it("renders grouped currency for full-value Cost displays", () => { + render( + , + ); + + const costCard = screen.getByTestId("report-summary-card-total-cost"); + expect(within(costCard).getByText("$1,400.00")).toBeInTheDocument(); + expect(costCard).toHaveTextContent("avg $1,400.00/day"); + }); + it("hides comparison badges when unavailable or previous totals are zero", () => { const { rerender } = render( { expect(requestsCard.nextElementSibling).toBe(conversationsCard); }); + it("renders requests, cancelled, and errors as distinct summary totals", () => { + render( + , + ); + + const requestsCard = screen.getByTestId("report-summary-card-requests"); + expect(within(requestsCard).getByText("Requests")).toBeInTheDocument(); + expect(within(requestsCard).getByText("4")).toBeInTheDocument(); + + const cancelledCard = screen.queryByTestId("report-summary-card-cancelled"); + expect.soft(cancelledCard).toBeInTheDocument(); + if (cancelledCard) { + expect.soft(within(cancelledCard).getByText("Cancelled")).toBeInTheDocument(); + expect.soft(within(cancelledCard).getByText("2")).toBeInTheDocument(); + } + + const errorsCard = screen.queryByTestId("report-summary-card-errors"); + expect.soft(errorsCard).toBeInTheDocument(); + if (errorsCard) { + expect.soft(within(errorsCard).getByText("Errors")).toBeInTheDocument(); + expect.soft(within(errorsCard).getByText("1")).toBeInTheDocument(); + } + }); + it("preserves trailing zeroes for unrelated whole K and B values", () => { render( { const requestsCard = screen.getByTestId("report-summary-card-requests"); expect(within(tokensCard).getByText("100.0B")).toBeInTheDocument(); - expect(within(tokensCard).getByText("Input 100.0B · Cache 0 · Output 0")).toBeInTheDocument(); + expect( + within(tokensCard).getByText( + "Input 100.0B · Cache 0 · Output 0", + ), + ).toBeInTheDocument(); + expect(within(tokensCard).getByText("Reported reasoning 0 (included in output) · 0/100000 requests")).toBeInTheDocument(); expect(within(requestsCard).getByText("100.0K")).toBeInTheDocument(); }); + + it("hides reasoning coverage when there are no requests", () => { + render( + , + ); + + expect(screen.queryByText(/Reported reasoning/)).not.toBeInTheDocument(); + }); }); diff --git a/frontend/src/features/reports/components/reports-summary-cards.tsx b/frontend/src/features/reports/components/reports-summary-cards.tsx index cdef8c88c9..cda71ec200 100644 --- a/frontend/src/features/reports/components/reports-summary-cards.tsx +++ b/frontend/src/features/reports/components/reports-summary-cards.tsx @@ -1,6 +1,7 @@ import { useTranslation } from "react-i18next"; import { cn } from "@/lib/utils"; +import { formatCurrency } from "@/utils/formatters"; import type { ReportComparison, ReportSummary } from "../schemas"; @@ -20,8 +21,8 @@ export function ReportsSummaryCards({ summary, comparison }: ReportsSummaryCards { id: "total-cost", label: t("reports.summary.totalCost"), - value: `$${summary.totalCostUsd.toFixed(2)}`, - sub: t("reports.summary.avgCostPerDay", { cost: `$${summary.avgCostPerDay.toFixed(2)}` }), + value: formatCurrency(summary.totalCostUsd), + sub: t("reports.summary.avgCostPerDay", { cost: formatCurrency(summary.avgCostPerDay) }), comparison: buildComparison(summary.totalCostUsd, comparison.previous.totalCostUsd, comparison.canCompare), }, { @@ -33,6 +34,14 @@ export function ReportsSummaryCards({ summary, comparison }: ReportsSummaryCards cache: formatNumber(summary.totalCachedTokens), output: formatNumber(summary.totalOutputTokens), }), + secondarySub: + summary.totalRequests > 0 + ? t("reports.summary.reasoningSub", { + reasoning: formatNumber(summary.totalReasoningTokens), + known: summary.reasoningUsageKnownRequests, + total: summary.totalRequests, + }) + : undefined, comparison: buildComparison( summary.totalInputTokens + summary.totalOutputTokens, comparison.previous.totalTokens, @@ -54,10 +63,20 @@ export function ReportsSummaryCards({ summary, comparison }: ReportsSummaryCards label: t("reports.summary.conversations"), value: formatNumber(summary.totalConversations), }, + { + id: "cancelled", + label: t("reports.summary.cancelled"), + value: formatNumber(summary.totalCancelled), + }, + { + id: "errors", + label: t("reports.summary.errors"), + value: formatNumber(summary.totalErrors), + }, ]; return ( -
+
{cards.map((card) => (
{card.sub ?
{card.sub}
: null} + {"secondarySub" in card && card.secondarySub ? ( +
{card.secondarySub}
+ ) : null}
))}
diff --git a/frontend/src/features/reports/components/time-to-first-token-chart.tsx b/frontend/src/features/reports/components/time-to-first-token-chart.tsx index 85f6e4a449..1ee2d353b3 100644 --- a/frontend/src/features/reports/components/time-to-first-token-chart.tsx +++ b/frontend/src/features/reports/components/time-to-first-token-chart.tsx @@ -12,6 +12,7 @@ import { import type { DailyReportRow } from "../schemas"; import { buildContinuousDailyRows } from "../daily-series"; import { ChartTooltip } from "./chart-tooltip"; +import { ReportChartCard } from "./report-chart-card"; export type TimeToFirstTokenChartProps = { startDate: string; @@ -31,9 +32,7 @@ export function TimeToFirstTokenChart({ startDate, endDate, data }: TimeToFirstT })); return ( -
-
{t("reports.charts.timeToFirstToken")}
-
+ @@ -69,7 +68,6 @@ export function TimeToFirstTokenChart({ startDate, endDate, data }: TimeToFirstT /> -
-
+ ); } diff --git a/frontend/src/features/reports/components/tokens-per-day-chart.test.tsx b/frontend/src/features/reports/components/tokens-per-day-chart.test.tsx index e8e7b7b9a9..2906898de2 100644 --- a/frontend/src/features/reports/components/tokens-per-day-chart.test.tsx +++ b/frontend/src/features/reports/components/tokens-per-day-chart.test.tsx @@ -41,9 +41,11 @@ describe("TokensPerDayChart", () => { conversations: 0, inputTokens: 5_400_000, outputTokens: 59_000, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 3.77, activeAccounts: 2, + cancelledCount: 0, errorCount: 0, }, ]} @@ -65,9 +67,11 @@ describe("TokensPerDayChart", () => { conversations: 0, inputTokens: 5_400_000, outputTokens: 59_000, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 3.77, activeAccounts: 2, + cancelledCount: 0, errorCount: 0, }, { @@ -76,9 +80,11 @@ describe("TokensPerDayChart", () => { conversations: 0, inputTokens: 6_800_000, outputTokens: 73_000, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 4.54, activeAccounts: 2, + cancelledCount: 0, errorCount: 0, }, ]} diff --git a/frontend/src/features/reports/components/tokens-per-day-chart.tsx b/frontend/src/features/reports/components/tokens-per-day-chart.tsx index 39a3fb5aac..3325e17fbf 100644 --- a/frontend/src/features/reports/components/tokens-per-day-chart.tsx +++ b/frontend/src/features/reports/components/tokens-per-day-chart.tsx @@ -12,6 +12,7 @@ import { import type { DailyReportRow } from "../schemas"; import { buildContinuousDailyRows } from "../daily-series"; import { ChartTooltip } from "./chart-tooltip"; +import { ReportChartCard } from "./report-chart-card"; export type TokensPerDayChartProps = { startDate: string; @@ -35,9 +36,7 @@ export function TokensPerDayChart({ startDate, endDate, data }: TokensPerDayChar })); return ( -
-
{t("reports.charts.tokensByDay")}
-
+ @@ -86,7 +85,6 @@ export function TokensPerDayChart({ startDate, endDate, data }: TokensPerDayChar /> -
-
+ ); } diff --git a/frontend/src/features/reports/components/tokens-per-second-chart.tsx b/frontend/src/features/reports/components/tokens-per-second-chart.tsx index 1fff8e969c..0e91a01af8 100644 --- a/frontend/src/features/reports/components/tokens-per-second-chart.tsx +++ b/frontend/src/features/reports/components/tokens-per-second-chart.tsx @@ -12,6 +12,7 @@ import { import type { DailyReportRow } from "../schemas"; import { buildContinuousDailyRows } from "../daily-series"; import { ChartTooltip } from "./chart-tooltip"; +import { ReportChartCard } from "./report-chart-card"; export type TokensPerSecondChartProps = { startDate: string; @@ -31,9 +32,7 @@ export function TokensPerSecondChart({ startDate, endDate, data }: TokensPerSeco })); return ( -
-
{t("reports.charts.tokensPerSecond")}
-
+ @@ -69,7 +68,6 @@ export function TokensPerSecondChart({ startDate, endDate, data }: TokensPerSeco /> -
-
+ ); } diff --git a/frontend/src/features/reports/daily-series.ts b/frontend/src/features/reports/daily-series.ts index dd539d0435..f76723e8c5 100644 --- a/frontend/src/features/reports/daily-series.ts +++ b/frontend/src/features/reports/daily-series.ts @@ -41,9 +41,11 @@ function createZeroRow(date: string): DailyReportRow { conversations: 0, inputTokens: 0, outputTokens: 0, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 0, activeAccounts: 0, + cancelledCount: 0, errorCount: 0, medianTtftMs: 0, medianTps: 0, diff --git a/frontend/src/features/reports/hooks/use-reports.test.tsx b/frontend/src/features/reports/hooks/use-reports.test.tsx index 3a2de65e3d..01c6a01495 100644 --- a/frontend/src/features/reports/hooks/use-reports.test.tsx +++ b/frontend/src/features/reports/hooks/use-reports.test.tsx @@ -12,6 +12,8 @@ vi.mock("@/lib/api-client", () => ({ totalCostUsd: 0, totalInputTokens: 0, totalOutputTokens: 0, + totalReasoningTokens: 0, + reasoningUsageKnownRequests: 0, totalCachedTokens: 0, totalRequests: 0, totalErrors: 0, @@ -238,4 +240,70 @@ describe("useReports", () => { ), ).toBe("chatgpt-app"); }); + + it("includes api_key_id in reports requests when apiKeyId filter is provided", async () => { + const queryClient = createTestQueryClient(); + + const { result } = renderHook( + () => + useReports( + { + startDate: "2030-01-09", + endDate: "2030-01-15", + accountId: ["acct_123"], + apiKeyId: ["key_456", "key_789"], + model: "gpt-5.1", + }, + "America/Los_Angeles", + ), + { wrapper: createWrapper(queryClient) }, + ); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + + expect(getMock).toHaveBeenCalledTimes(1); + const searchParams = getRequestedSearchParams(); + expect(searchParams.getAll("api_key_id")).toEqual(["key_456", "key_789"]); + }); + + it("refetches when the apiKeyId filter changes", async () => { + const queryClient = createTestQueryClient(); + + const { result, rerender } = renderHook( + ({ apiKeyId }) => + useReports( + { + startDate: "2030-01-09", + endDate: "2030-01-15", + accountId: ["acct_123"], + apiKeyId, + model: "gpt-5.1", + }, + "America/Los_Angeles", + ), + { + wrapper: createWrapper(queryClient), + initialProps: { apiKeyId: ["key_1"] }, + }, + ); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + + rerender({ apiKeyId: ["key_2"] }); + + await waitFor(() => expect(getMock).toHaveBeenCalledTimes(2)); + + const [firstUrl] = getMock.mock.calls[0] ?? []; + const [secondUrl] = getMock.mock.calls[1] ?? []; + expect( + new URL(String(firstUrl), "http://localhost").searchParams.getAll( + "api_key_id", + ), + ).toEqual(["key_1"]); + expect( + new URL(String(secondUrl), "http://localhost").searchParams.getAll( + "api_key_id", + ), + ).toEqual(["key_2"]); + }); }); diff --git a/frontend/src/features/reports/hooks/use-reports.ts b/frontend/src/features/reports/hooks/use-reports.ts index 099381218b..dbf370c501 100644 --- a/frontend/src/features/reports/hooks/use-reports.ts +++ b/frontend/src/features/reports/hooks/use-reports.ts @@ -6,6 +6,7 @@ type ReportsFilterState = { startDate: string | undefined; endDate: string | undefined; accountId: string[]; + apiKeyId?: string[]; model: string | undefined; useragent?: string | undefined; }; @@ -22,6 +23,10 @@ export function useReports( startDate: filters.startDate, endDate: filters.endDate, accountId: filters.accountId.length > 0 ? filters.accountId : undefined, + apiKeyId: + filters.apiKeyId && filters.apiKeyId.length > 0 + ? filters.apiKeyId + : undefined, model: filters.model || undefined, useragent: filters.useragent || undefined, timezone: timeZone, diff --git a/frontend/src/features/reports/schemas.test.ts b/frontend/src/features/reports/schemas.test.ts index 13f66196ad..464ec0932b 100644 --- a/frontend/src/features/reports/schemas.test.ts +++ b/frontend/src/features/reports/schemas.test.ts @@ -2,23 +2,89 @@ import { describe, expect, it } from "vitest"; import { ReportsResponseSchema } from "./schemas"; +function validReportsPayload() { + return { + summary: { + totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, reasoningUsageKnownRequests: 3, + totalCachedTokens: 0, totalRequests: 4, totalCancelled: 2, + totalErrors: 1, totalConversations: 7, activeAccounts: 3, + avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, + }, + comparison: { canCompare: true, previous: { totalCostUsd: 10, totalTokens: 400, totalRequests: 20 } }, + daily: [{ date: "2026-06-05", requests: 4, conversations: 3, inputTokens: 100, outputTokens: 50, reasoningTokens: 35, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, cancelledCount: 2, errorCount: 1 }], + byModel: [{ model: "gpt-5.1", costUsd: 12.5, requests: 4, percentage: 100 }], + byUseragent: [{ useragent: "claude-code", costUsd: 12.5, requests: 4, percentage: 100 }], + byAccount: [], + }; +} + describe("ReportsResponseSchema", () => { - it("parses totalConversations on summary and conversations on daily rows", () => { + it("preserves conversation, cancellation, and reasoning totals from the reports payload", () => { const parsed = ReportsResponseSchema.parse({ summary: { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, - totalCachedTokens: 0, totalRequests: 25, totalErrors: 1, - totalConversations: 7, activeAccounts: 3, + totalReasoningTokens: 70, reasoningUsageKnownRequests: 3, + totalCachedTokens: 0, totalRequests: 4, totalErrors: 1, + totalCancelled: 2, totalConversations: 7, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, }, comparison: { canCompare: true, previous: { totalCostUsd: 10, totalTokens: 400, totalRequests: 20 } }, - daily: [{ date: "2026-06-05", requests: 10, conversations: 3, inputTokens: 100, outputTokens: 50, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, errorCount: 0 }], - byModel: [{ model: "gpt-5.1", costUsd: 12.5, requests: 25, percentage: 100 }], - byUseragent: [{ useragent: "claude-code", costUsd: 12.5, requests: 25, percentage: 100 }], + daily: [{ date: "2026-06-05", requests: 4, conversations: 3, inputTokens: 100, outputTokens: 50, reasoningTokens: 35, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, errorCount: 1, cancelledCount: 2 }], + byModel: [{ model: "gpt-5.1", costUsd: 12.5, requests: 4, percentage: 100 }], + byUseragent: [{ useragent: "claude-code", costUsd: 12.5, requests: 4, percentage: 100 }], byAccount: [], }); + expect(parsed.summary.totalRequests).toBe(4); + expect(parsed.summary.totalErrors).toBe(1); + expect.soft(Reflect.get(parsed.summary, "totalCancelled")).toBe(2); expect(parsed.summary.totalConversations).toBe(7); + expect(parsed.summary.totalReasoningTokens).toBe(70); + expect(parsed.summary.reasoningUsageKnownRequests).toBe(3); + expect(parsed.daily[0]?.requests).toBe(4); + expect(parsed.daily[0]?.errorCount).toBe(1); + expect.soft(Reflect.get(parsed.daily[0] ?? {}, "cancelledCount")).toBe(2); expect(parsed.daily[0]?.conversations).toBe(3); + expect(parsed.daily[0]?.reasoningTokens).toBe(35); + }); + + it("rejects omitted totalCancelled on summary", () => { + const payload = validReportsPayload(); + Reflect.deleteProperty(payload.summary, "totalCancelled"); + + expect(() => ReportsResponseSchema.parse(payload)).toThrow(/totalCancelled/i); + }); + + it("rejects omitted cancelledCount on daily rows", () => { + const payload = validReportsPayload(); + Reflect.deleteProperty(payload.daily[0] ?? {}, "cancelledCount"); + + expect(() => ReportsResponseSchema.parse(payload)).toThrow(/cancelledCount/i); + }); + + it("rejects omitted reasoning totals and coverage on summary", () => { + const missingTotal = validReportsPayload(); + Reflect.deleteProperty(missingTotal.summary, "totalReasoningTokens"); + expect(() => ReportsResponseSchema.parse(missingTotal)).toThrow(/totalReasoningTokens/i); + + const missingCoverage = validReportsPayload(); + Reflect.deleteProperty(missingCoverage.summary, "reasoningUsageKnownRequests"); + expect(() => ReportsResponseSchema.parse(missingCoverage)).toThrow(/reasoningUsageKnownRequests/i); + }); + + it("rejects omitted reasoningTokens on daily rows", () => { + const payload = validReportsPayload(); + Reflect.deleteProperty(payload.daily[0] ?? {}, "reasoningTokens"); + + expect(() => ReportsResponseSchema.parse(payload)).toThrow(/reasoningTokens/i); + }); + + it("preserves null reasoningTokens as unknown on daily rows", () => { + const payload = validReportsPayload(); + Reflect.set(payload.daily[0] ?? {}, "reasoningTokens", null); + + const parsed = ReportsResponseSchema.parse(payload); + expect(parsed.daily[0]?.reasoningTokens).toBeNull(); }); it("rejects omitted totalConversations on summary", () => { @@ -26,16 +92,17 @@ describe("ReportsResponseSchema", () => { ReportsResponseSchema.parse({ summary: { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, - totalCachedTokens: 0, totalRequests: 25, totalErrors: 1, + totalReasoningTokens: 70, reasoningUsageKnownRequests: 3, + totalCachedTokens: 0, totalRequests: 25, totalCancelled: 0, totalErrors: 1, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, }, comparison: { canCompare: true, previous: { totalCostUsd: 10, totalTokens: 400, totalRequests: 20 } }, - daily: [{ date: "2026-06-05", requests: 10, conversations: 0, inputTokens: 100, outputTokens: 50, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, errorCount: 0 }], + daily: [{ date: "2026-06-05", requests: 10, conversations: 0, inputTokens: 100, outputTokens: 50, reasoningTokens: 35, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, cancelledCount: 0, errorCount: 0 }], byModel: [{ model: "gpt-5.1", costUsd: 12.5, requests: 25, percentage: 100 }], byUseragent: [{ useragent: "claude-code", costUsd: 12.5, requests: 25, percentage: 100 }], byAccount: [], }), - ).toThrow(); + ).toThrow(/totalConversations/i); }); @@ -45,8 +112,11 @@ describe("ReportsResponseSchema", () => { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, + reasoningUsageKnownRequests: 3, totalCachedTokens: 0, totalRequests: 25, + totalCancelled: 0, totalErrors: 1, totalConversations: 0, activeAccounts: 3, @@ -96,9 +166,13 @@ describe("ReportsResponseSchema", () => { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, + reasoningUsageKnownRequests: 3, totalCachedTokens: 0, totalRequests: 25, + totalCancelled: 0, totalErrors: 1, + totalConversations: 0, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, @@ -118,9 +192,13 @@ describe("ReportsResponseSchema", () => { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, + reasoningUsageKnownRequests: 3, totalCachedTokens: 0, totalRequests: 25, + totalCancelled: 0, totalErrors: 1, + totalConversations: 0, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, @@ -143,9 +221,13 @@ describe("ReportsResponseSchema", () => { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, + reasoningUsageKnownRequests: 3, totalCachedTokens: 0, totalRequests: 25, + totalCancelled: 0, totalErrors: 1, + totalConversations: 0, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, @@ -179,9 +261,13 @@ describe("ReportsResponseSchema", () => { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, + reasoningUsageKnownRequests: 3, totalCachedTokens: 0, totalRequests: 25, + totalCancelled: 0, totalErrors: 1, + totalConversations: 0, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, diff --git a/frontend/src/features/reports/schemas.ts b/frontend/src/features/reports/schemas.ts index 795e671577..07dc9a7fed 100644 --- a/frontend/src/features/reports/schemas.ts +++ b/frontend/src/features/reports/schemas.ts @@ -6,9 +6,11 @@ const DailyReportRowSchema = z.object({ conversations: z.number(), inputTokens: z.number(), outputTokens: z.number(), + reasoningTokens: z.number().nullable(), cachedInputTokens: z.number(), costUsd: z.number(), activeAccounts: z.number(), + cancelledCount: z.number(), errorCount: z.number(), medianTtftMs: z.number().optional().default(0), medianTps: z.number().optional().default(0), @@ -40,8 +42,11 @@ const ReportSummarySchema = z.object({ totalCostUsd: z.number(), totalInputTokens: z.number(), totalOutputTokens: z.number(), + totalReasoningTokens: z.number(), + reasoningUsageKnownRequests: z.number(), totalCachedTokens: z.number(), totalRequests: z.number(), + totalCancelled: z.number(), totalErrors: z.number(), totalConversations: z.number(), activeAccounts: z.number(), diff --git a/frontend/src/features/settings/advanced-settings-deeplink.ts b/frontend/src/features/settings/advanced-settings-deeplink.ts new file mode 100644 index 0000000000..ae72234b84 --- /dev/null +++ b/frontend/src/features/settings/advanced-settings-deeplink.ts @@ -0,0 +1,7 @@ +export function shouldExpandAdvancedSettings(search: string, hash: string): boolean { + const query = search.startsWith("?") ? search.slice(1) : search; + if (new URLSearchParams(query).get("advanced") === "1") { + return true; + } + return hash === "#firewall"; +} diff --git a/frontend/src/features/settings/api.ts b/frontend/src/features/settings/api.ts index 5c39f5d36a..e06f15f299 100644 --- a/frontend/src/features/settings/api.ts +++ b/frontend/src/features/settings/api.ts @@ -4,6 +4,8 @@ import { AccountProxyBindingSchema, DashboardSettingsSchema, SettingsUpdateRequestSchema, + TelemetryConsentSchema, + TelemetryConsentUpdateRequestSchema, UpstreamProxyAdminSchema, UpstreamProxyEndpointCreateRequestSchema, UpstreamProxyEndpointSchema, @@ -15,6 +17,7 @@ import { const SETTINGS_PATH = "/api/settings"; const UPSTREAM_PROXY_PATH = `${SETTINGS_PATH}/upstream-proxy`; +const TELEMETRY_PATH = `${SETTINGS_PATH}/telemetry`; export function getSettings() { return get(SETTINGS_PATH, DashboardSettingsSchema); @@ -27,6 +30,18 @@ export function updateSettings(payload: unknown) { }); } +export function getTelemetryConsent(options: { includePreview?: boolean } = {}) { + const path = options.includePreview ? `${TELEMETRY_PATH}?include_preview=true` : TELEMETRY_PATH; + return get(path, TelemetryConsentSchema); +} + +export function updateTelemetryConsent(payload: unknown) { + const validated = TelemetryConsentUpdateRequestSchema.parse(payload); + return put(TELEMETRY_PATH, TelemetryConsentSchema, { + body: validated, + }); +} + export function getUpstreamProxyAdmin() { return get(UPSTREAM_PROXY_PATH, UpstreamProxyAdminSchema); } diff --git a/frontend/src/features/settings/components/advanced-settings-group.test.tsx b/frontend/src/features/settings/components/advanced-settings-group.test.tsx new file mode 100644 index 0000000000..eee910bf4f --- /dev/null +++ b/frontend/src/features/settings/components/advanced-settings-group.test.tsx @@ -0,0 +1,88 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, render, waitFor } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +import { shouldExpandAdvancedSettings } from "@/features/settings/advanced-settings-deeplink"; +import { AdvancedSettingsGroup } from "@/features/settings/components/advanced-settings-group"; + +describe("shouldExpandAdvancedSettings", () => { + it("stays collapsed for a plain settings URL", () => { + expect(shouldExpandAdvancedSettings("", "")).toBe(false); + expect(shouldExpandAdvancedSettings("?view=guest", "")).toBe(false); + }); + + it("opens for the advanced query or firewall hash", () => { + expect(shouldExpandAdvancedSettings("?advanced=1", "")).toBe(true); + expect(shouldExpandAdvancedSettings("", "#firewall")).toBe(true); + expect(shouldExpandAdvancedSettings("?advanced=1", "#firewall")).toBe(true); + }); +}); + +describe("AdvancedSettingsGroup", () => { + it("scrolls once after preceding layout queries settle", async () => { + let resolveLayoutQuery: ((value: string) => void) | undefined; + let resolveUnrelatedQuery: ((value: string) => void) | undefined; + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + const layoutQuery = queryClient.fetchQuery({ + queryKey: ["advanced-settings-layout"], + queryFn: () => + new Promise((resolve) => { + resolveLayoutQuery = resolve; + }), + }); + const unrelatedQuery = queryClient.fetchQuery({ + queryKey: ["firewall", "list"], + queryFn: () => + new Promise((resolve) => { + resolveUnrelatedQuery = resolve; + }), + }); + const scrollIntoView = vi.fn(); + const elementLookup = vi + .spyOn(document, "getElementById") + .mockReturnValue({ scrollIntoView } as unknown as HTMLElement); + const animationFrame = vi + .spyOn(window, "requestAnimationFrame") + .mockImplementation((callback) => { + callback(0); + return 1; + }); + + const view = render( + + +
Firewall
+ + , + ); + + expect(scrollIntoView).not.toHaveBeenCalled(); + + await act(async () => { + resolveLayoutQuery?.("ready"); + await layoutQuery; + }); + + await waitFor(() => expect(scrollIntoView).toHaveBeenCalledTimes(1)); + + resolveUnrelatedQuery?.("ready"); + await unrelatedQuery; + + await queryClient.fetchQuery({ + queryKey: ["later-refresh"], + queryFn: async () => "ready", + }); + expect(scrollIntoView).toHaveBeenCalledTimes(1); + + view.unmount(); + + animationFrame.mockRestore(); + elementLookup.mockRestore(); + }); +}); diff --git a/frontend/src/features/settings/components/advanced-settings-group.tsx b/frontend/src/features/settings/components/advanced-settings-group.tsx index fe2bf34b8f..679a69beb0 100644 --- a/frontend/src/features/settings/components/advanced-settings-group.tsx +++ b/frontend/src/features/settings/components/advanced-settings-group.tsx @@ -1,12 +1,18 @@ import { ChevronRight } from "lucide-react"; -import { useState, type ReactNode } from "react"; +import { useIsFetching, useQueryClient, type Query, type QueryKey } from "@tanstack/react-query"; +import { useCallback, useEffect, useRef, useState, type ReactNode } from "react"; import { useTranslation } from "react-i18next"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { cn } from "@/lib/utils"; +const EMPTY_QUERY_KEYS: readonly QueryKey[] = []; + export type AdvancedSettingsGroupProps = { children: ReactNode; + defaultOpen?: boolean; + scrollToId?: string; + waitForQueryKeys?: readonly QueryKey[]; }; /** @@ -15,9 +21,48 @@ export type AdvancedSettingsGroupProps = { * Children are unmounted while the group is closed, so section data queries * only fire once the operator expands the group. */ -export function AdvancedSettingsGroup({ children }: AdvancedSettingsGroupProps) { +export function AdvancedSettingsGroup({ + children, + defaultOpen = false, + scrollToId, + waitForQueryKeys = EMPTY_QUERY_KEYS, +}: AdvancedSettingsGroupProps) { const { t } = useTranslation(); - const [open, setOpen] = useState(false); + const [open, setOpen] = useState(defaultOpen); + const queryClient = useQueryClient(); + const isLayoutQuery = useCallback( + (query: Query) => + waitForQueryKeys.some((prefix) => + prefix.every((value, index) => Object.is(query.queryKey[index], value)), + ), + [waitForQueryKeys], + ); + const fetchingQueries = useIsFetching({ predicate: isLayoutQuery }); + const scrolledToIdRef = useRef(undefined); + + useEffect(() => { + if (!open || !scrollToId) { + scrolledToIdRef.current = undefined; + return; + } + if (scrolledToIdRef.current === scrollToId) { + return; + } + const frame = window.requestAnimationFrame(() => { + if (queryClient.isFetching({ predicate: isLayoutQuery }) > 0) { + return; + } + const target = document.getElementById(scrollToId); + if (!target) { + return; + } + target.scrollIntoView({ block: "start" }); + scrolledToIdRef.current = scrollToId; + }); + return () => { + window.cancelAnimationFrame(frame); + }; + }, [fetchingQueries, isLayoutQuery, open, queryClient, scrollToId]); return ( @@ -34,7 +79,9 @@ export function AdvancedSettingsGroup({ children }: AdvancedSettingsGroupProps) {t("settings.advanced.description")} - {children} + + {children} + ); } diff --git a/frontend/src/features/settings/components/routing-settings.test.tsx b/frontend/src/features/settings/components/routing-settings.test.tsx index 8ea58cc946..c7a2873794 100644 --- a/frontend/src/features/settings/components/routing-settings.test.tsx +++ b/frontend/src/features/settings/components/routing-settings.test.tsx @@ -610,6 +610,65 @@ describe("RoutingSettings", () => { expect(screen.getByText(/No strategy can guarantee account-safety outcomes/i)).toBeInTheDocument(); }); + it("explains soft sticky routing versus hard Codex continuation affinity", () => { + render(); + + expect( + screen.getByText(/does not disable hard Codex continuation affinity/i), + ).toBeInTheDocument(); + expect(screen.getByText(/soft preference, not a guarantee/i)).toBeInTheDocument(); + }); + + it("explains primary versus secondary quota windows and threshold units", () => { + render(); + + expect(screen.getByText("Primary vs secondary quota")).toBeInTheDocument(); + expect( + screen.getByText(/Primary quota is the short 5-hour usage window/i), + ).toBeInTheDocument(); + expect( + screen.getByText(/5-hour \(primary\) window has been used/i), + ).toBeInTheDocument(); + expect( + screen.getByText(/secondary window \(weekly, or monthly on monthly-only plans\) has been used/i), + ).toBeInTheDocument(); + }); + + it("shows the remaining-percent equivalent for sticky thresholds", async () => { + const user = userEvent.setup(); + render(); + + // Defaults: primary 95% used, secondary 100% used. + expect(screen.getByText("95% used · 5% remaining in quota terms")).toBeInTheDocument(); + expect(screen.getByText("100% used · 0% remaining in quota terms")).toBeInTheDocument(); + + const secondary = screen.getByRole("spinbutton", { name: "Sticky secondary threshold" }); + await user.clear(secondary); + await user.type(secondary, "70"); + + expect(screen.getByText("70% used · 30% remaining in quota terms")).toBeInTheDocument(); + + // Decimal thresholds keep the two displayed values complementary. + await user.clear(secondary); + await user.type(secondary, "12.5"); + + expect(screen.getByText("12.5% used · 87.5% remaining in quota terms")).toBeInTheDocument(); + }); + + it("describes prefer-earlier-reset selection behavior", () => { + render(); + + expect( + screen.getByText(/prefer those whose selected quota window resets sooner/i), + ).toBeInTheDocument(); + }); + + it("describes what limit warm-up sends and that probes consume quota", () => { + render(); + + expect(screen.getByText(/consume a small amount of quota/i)).toBeInTheDocument(); + }); + it("saves staggered idle warm-up when limit warm-up is enabled", async () => { const user = userEvent.setup(); const onSave = vi.fn().mockResolvedValue(undefined); diff --git a/frontend/src/features/settings/components/routing-settings.tsx b/frontend/src/features/settings/components/routing-settings.tsx index 06f9e96083..5582c2adcb 100644 --- a/frontend/src/features/settings/components/routing-settings.tsx +++ b/frontend/src/features/settings/components/routing-settings.tsx @@ -144,6 +144,14 @@ function parseNonnegativeInteger(value: string): number | null { return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : null; } +function thresholdHintValues(value: number): { used: string; remaining: string } { + // Derive the remaining percent from the rounded used percent so the two + // displayed values always sum to exactly 100. + const used = Number(value.toFixed(1)); + const remaining = Number((100 - used).toFixed(1)); + return { used: String(used), remaining: String(remaining) }; +} + export function RoutingSettings({ settings, accounts = EMPTY_ACCOUNTS, @@ -796,23 +804,41 @@ export function RoutingSettings({
-
-
-

{t("settings.routing.stickyThreads.label")}

-

{t("settings.routing.stickyThreads.description")}

+
+
+
+

{t("settings.routing.stickyThreads.label")}

+

{t("settings.routing.stickyThreads.description")}

+
+ save({ stickyThreadsEnabled: checked })} + />
- save({ stickyThreadsEnabled: checked })} - /> +

+ {t("settings.routing.stickyThreads.hardAffinityNote")} +

+
+ +
+

{t("settings.routing.quotaWindows.title")}

+

{t("settings.routing.quotaWindows.explainer")}

{t("settings.routing.stickyThresholds.primaryLabel")}

{t("settings.routing.stickyThresholds.primaryDescription")}

+ {stickyPrimaryThresholdValid ? ( +

+ {t( + "settings.routing.stickyThresholds.usedRemainingHint", + thresholdHintValues(parsedStickyPrimaryThreshold), + )} +

+ ) : null}

{t("settings.routing.stickyThresholds.secondaryLabel")}

{t("settings.routing.stickyThresholds.secondaryDescription")}

+ {stickySecondaryThresholdValid ? ( +

+ {t( + "settings.routing.stickyThresholds.usedRemainingHint", + thresholdHintValues(parsedStickySecondaryThreshold), + )} +

+ ) : null}
({ useSettings: () => useSettingsMock(), @@ -76,6 +79,13 @@ vi.mock("@/features/settings/components/data-retention-settings", () => ({ }, })); +vi.mock("@/features/settings/components/telemetry-settings", () => ({ + TelemetrySettings: (props: unknown) => { + telemetrySettingsMock(props); + return
Telemetry Settings
; + }, +})); + vi.mock("@/features/api-keys/components/api-keys-section", () => ({ ApiKeysSection: (props: unknown) => { apiKeysSectionMock(props); @@ -162,15 +172,29 @@ describe("SettingsPage", () => { stickySessionsSectionMock.mockReset(); modelSourcesSettingsMock.mockReset(); dataRetentionSettingsMock.mockReset(); + telemetrySettingsMock.mockReset(); }); + function renderSettings(initialEntry = "/settings") { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return render( + + + + + , + ); + } + async function expandAdvancedSettings() { const user = userEvent.setup({ delay: null }); await user.click(screen.getByRole("button", { name: "Show advanced settings" })); } it("keeps advanced sections collapsed and unmounted by default", () => { - render(); + renderSettings(); expect(screen.getByRole("button", { name: "Show advanced settings" })).toBeInTheDocument(); expect(screen.queryByText("Routing Settings")).not.toBeInTheDocument(); @@ -192,10 +216,11 @@ describe("SettingsPage", () => { expect(screen.getByText("Appearance Settings")).toBeInTheDocument(); expect(screen.getByText("Import Settings")).toBeInTheDocument(); expect(screen.getByText("API Keys Section")).toBeInTheDocument(); + expect(screen.getByText("Telemetry Settings")).toBeInTheDocument(); }); it("mounts every advanced section after one expand interaction", async () => { - render(); + renderSettings(); await expandAdvancedSettings(); @@ -211,7 +236,7 @@ describe("SettingsPage", () => { it("disables write-capable sections for read-only guests", async () => { useAuthStore.setState({ canWrite: false }); - render(); + renderSettings(); expect(screen.getByText("You are viewing the dashboard with read-only guest access. Admin controls are disabled.")).toBeInTheDocument(); expect(screen.queryByText("Guest Access Settings")).not.toBeInTheDocument(); @@ -219,6 +244,7 @@ describe("SettingsPage", () => { expect(screen.queryByText("Session Settings")).not.toBeInTheDocument(); expect(importSettingsMock).toHaveBeenCalledWith(expect.objectContaining({ busy: true })); expect(apiKeysSectionMock).toHaveBeenCalledWith(expect.objectContaining({ disabled: true })); + expect(telemetrySettingsMock).toHaveBeenCalledWith(expect.objectContaining({ disabled: true })); await expandAdvancedSettings(); @@ -231,7 +257,7 @@ describe("SettingsPage", () => { }); it("keeps guest access settings available for writable sessions", async () => { - render(); + renderSettings(); expect(screen.getByText("Guest Access Settings")).toBeInTheDocument(); expect(guestAccessSettingsMock).toHaveBeenCalledWith( @@ -245,4 +271,13 @@ describe("SettingsPage", () => { expect(routingSettingsMock).toHaveBeenCalledWith(expect.objectContaining({ busy: false })); }); + + it("expands Advanced and mounts firewall on the advanced deeplink", () => { + renderSettings("/settings?advanced=1#firewall"); + + expect(screen.getByText("Firewall Section")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Hide advanced settings" })).toBeInTheDocument(); + expect(firewallSectionMock).toHaveBeenCalled(); + }); + }); diff --git a/frontend/src/features/settings/components/settings-page.tsx b/frontend/src/features/settings/components/settings-page.tsx index 9fc5f3a291..caaf066564 100644 --- a/frontend/src/features/settings/components/settings-page.tsx +++ b/frontend/src/features/settings/components/settings-page.tsx @@ -1,6 +1,7 @@ import { Suspense, lazy } from "react"; import { Settings } from "lucide-react"; import { useTranslation } from "react-i18next"; +import { useLocation } from "react-router-dom"; import { AlertMessage } from "@/components/alert-message"; import { LoadingOverlay } from "@/components/layout/loading-overlay"; @@ -10,6 +11,7 @@ import { FirewallSection } from "@/features/firewall/components/firewall-section import { ModelSourcesSettings } from "@/features/model-sources/components/model-sources-settings"; import { QuotaPlannerSection } from "@/features/quota-planner/components/quota-planner-section"; import { buildSettingsUpdateRequest } from "@/features/settings/payload"; +import { shouldExpandAdvancedSettings } from "@/features/settings/advanced-settings-deeplink"; import { AdvancedSettingsGroup } from "@/features/settings/components/advanced-settings-group"; import { AppearanceSettings } from "@/features/settings/components/appearance-settings"; import { DataRetentionSettings } from "@/features/settings/components/data-retention-settings"; @@ -20,6 +22,7 @@ import { ResetCreditSettings } from "@/features/settings/components/reset-credit import { RoutingSettings } from "@/features/settings/components/routing-settings"; import { SessionSettings } from "@/features/settings/components/session-settings"; import { SettingsSkeleton } from "@/features/settings/components/settings-skeleton"; +import { TelemetrySettings } from "@/features/settings/components/telemetry-settings"; import { UpstreamProxySettings } from "@/features/settings/components/upstream-proxy-settings"; import { StickySessionsSection } from "@/features/sticky-sessions/components/sticky-sessions-section"; import { useAuthStore } from "@/features/auth/hooks/use-auth"; @@ -31,8 +34,17 @@ const TotpSettings = lazy(() => import("@/features/settings/components/totp-settings").then((m) => ({ default: m.TotpSettings })), ); +const FIREWALL_LAYOUT_QUERY_KEYS = [ + ["accounts", "list"], + ["settings", "upstream-proxy"], + ["model-sources", "list"], +] as const; + export function SettingsPage() { const { t } = useTranslation(); + const location = useLocation(); + const expandAdvanced = shouldExpandAdvancedSettings(location.search, location.hash); + const advancedScrollToId = location.hash.replace(/^#/, "") || undefined; const { settingsQuery, updateSettingsMutation } = useSettings(); const { accountsQuery } = useAccounts(); const { @@ -136,7 +148,14 @@ export function SettingsPage() { } /> - + + +
+ {/* Telemetry */} +
+
+
+
+ +
+ + +
+
+ +
+ +
+
+ + +
+ +
+
+
+ {/* Firewall */}
diff --git a/frontend/src/features/settings/components/telemetry-consent-dialog.test.tsx b/frontend/src/features/settings/components/telemetry-consent-dialog.test.tsx new file mode 100644 index 0000000000..63a2ff130e --- /dev/null +++ b/frontend/src/features/settings/components/telemetry-consent-dialog.test.tsx @@ -0,0 +1,194 @@ +import { screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { HttpResponse, http } from "msw"; +import { beforeEach, describe, expect, it } from "vitest"; + +import { useAuthStore } from "@/features/auth/hooks/use-auth"; +import { TelemetryConsentDialog } from "@/features/settings/components/telemetry-consent-dialog"; +import i18n from "@/i18n"; +import { createTelemetryConsent, createTelemetrySnapshotEnvelope } from "@/test/mocks/factories"; +import { server } from "@/test/mocks/server"; +import { renderWithProviders } from "@/test/utils"; + +function undecidedConsent() { + return createTelemetryConsent({ state: "undecided", source: "default", active: true }); +} + +describe("TelemetryConsentDialog", () => { + beforeEach(() => { + useAuthStore.setState({ canWrite: true }); + }); + + it("shows the exact transmitted envelope with both decision actions while undecided", async () => { + server.use(http.get("/api/settings/telemetry", () => HttpResponse.json(undecidedConsent()))); + + renderWithProviders(); + + const dialog = await screen.findByRole("dialog", { name: "Anonymous telemetry" }); + // The full envelope is the exact transmitted body: top-level instance_id + // and timestamp plus the snapshot under metrics. + expect( + within(dialog).getByText(/"instance_id": "00000000-0000-4000-8000-000000000000"/), + ).toBeInTheDocument(); + expect(within(dialog).getByText(/"timestamp": "2026-08-06T00:00:00Z"/)).toBeInTheDocument(); + expect(within(dialog).getByText(/"metrics": \{/)).toBeInTheDocument(); + expect(within(dialog).getByText(/"schema_version": 1/)).toBeInTheDocument(); + expect(within(dialog).getByText(/"consent": "undecided"/)).toBeInTheDocument(); + expect( + within(dialog).getByText(i18n.t("settings.telemetry.optOutNotice")), + ).toBeInTheDocument(); + expect(within(dialog).getByRole("button", { name: "Keep enabled" })).toBeInTheDocument(); + expect(within(dialog).getByRole("button", { name: "Disable telemetry" })).toBeInTheDocument(); + expect( + within(dialog).getByRole("link", { name: "Learn what is collected and why" }), + ).toBeInTheDocument(); + }); + + it("persists enabled=true when the operator keeps telemetry enabled", async () => { + const user = userEvent.setup(); + let putBody: unknown = null; + server.use( + http.get("/api/settings/telemetry", () => HttpResponse.json(undecidedConsent())), + http.put("/api/settings/telemetry", async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json( + createTelemetryConsent({ state: "enabled", source: "persisted", active: true }), + ); + }), + ); + + renderWithProviders(); + + await user.click(await screen.findByRole("button", { name: "Keep enabled" })); + + await waitFor(() => expect(putBody).toEqual({ enabled: true })); + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + }); + + it("persists enabled=false when the operator disables telemetry", async () => { + const user = userEvent.setup(); + let putBody: unknown = null; + server.use( + http.get("/api/settings/telemetry", () => HttpResponse.json(undecidedConsent())), + http.put("/api/settings/telemetry", async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json( + createTelemetryConsent({ state: "disabled", source: "persisted", active: false }), + ); + }), + ); + + renderWithProviders(); + + await user.click(await screen.findByRole("button", { name: "Disable telemetry" })); + + await waitFor(() => expect(putBody).toEqual({ enabled: false })); + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + }); + + it("closes without persisting a decision when dismissed with Escape", async () => { + const user = userEvent.setup(); + let putCalled = false; + server.use( + http.get("/api/settings/telemetry", () => HttpResponse.json(undecidedConsent())), + http.put("/api/settings/telemetry", () => { + putCalled = true; + return HttpResponse.json(undecidedConsent()); + }), + ); + + renderWithProviders(); + + await screen.findByRole("dialog"); + await user.keyboard("{Escape}"); + + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + expect(putCalled).toBe(false); + }); + + it("stays hidden once a decision has been persisted", async () => { + // Synthetic preview keeps the preview-null gate open so this test binds + // the state === "undecided" gate alone. + server.use( + http.get("/api/settings/telemetry", () => + HttpResponse.json( + createTelemetryConsent({ + state: "enabled", + source: "persisted", + active: true, + preview: createTelemetrySnapshotEnvelope(), + }), + ), + ), + ); + + const { queryClient } = renderWithProviders(); + + await waitFor(() => + expect(queryClient.getQueryState(["settings", "telemetry"])?.status).toBe("success"), + ); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }); + + it("stays hidden when the response carries no preview envelope", async () => { + server.use( + http.get("/api/settings/telemetry", () => + HttpResponse.json( + createTelemetryConsent({ state: "undecided", source: "default", active: true, preview: null }), + ), + ), + ); + + const { queryClient } = renderWithProviders(); + + await waitFor(() => + expect(queryClient.getQueryState(["settings", "telemetry"])?.status).toBe("success"), + ); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }); + + it("stays hidden while the environment variable controls telemetry", async () => { + // Synthetic preview keeps the preview-null gate open so this test binds + // the source !== "env" gate alone. + server.use( + http.get("/api/settings/telemetry", () => + HttpResponse.json( + createTelemetryConsent({ + state: "undecided", + source: "env", + active: false, + preview: createTelemetrySnapshotEnvelope(), + }), + ), + ), + ); + + const { queryClient } = renderWithProviders(); + + await waitFor(() => + expect(queryClient.getQueryState(["settings", "telemetry"])?.status).toBe("success"), + ); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }); + + it("stays hidden for read-only sessions and never requests the preview aggregation", async () => { + useAuthStore.setState({ canWrite: false }); + let requested = false; + server.use( + http.get("/api/settings/telemetry", () => { + requested = true; + return HttpResponse.json(undecidedConsent()); + }), + ); + + const { queryClient } = renderWithProviders(); + + // Read-only guests can never act on the dialog, so the consent query is + // disabled entirely: no fetch fires and the query stays pending. + await waitFor(() => + expect(queryClient.getQueryState(["settings", "telemetry"])?.fetchStatus).toBe("idle"), + ); + expect(requested).toBe(false); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }); +}); diff --git a/frontend/src/features/settings/components/telemetry-consent-dialog.tsx b/frontend/src/features/settings/components/telemetry-consent-dialog.tsx new file mode 100644 index 0000000000..4a20410aa2 --- /dev/null +++ b/frontend/src/features/settings/components/telemetry-consent-dialog.tsx @@ -0,0 +1,88 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; + +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { useAuthStore } from "@/features/auth/hooks/use-auth"; +import { TelemetryPayloadPreview } from "@/features/settings/components/telemetry-payload-preview"; +import { useTelemetryConsent } from "@/features/settings/hooks/use-settings"; + +// Same published page the backend startup notice points operators to +// (TELEMETRY_FIELDS_DOCUMENTATION in app/modules/telemetry/scheduler.py). +const TELEMETRY_DOCS_URL = "https://soju06.github.io/codex-lb/telemetry/"; + +export function TelemetryConsentDialog() { + const { t } = useTranslation(); + const canWrite = useAuthStore((state) => state.canWrite); + const [dismissed, setDismissed] = useState(false); + // Read-only guests can never act on the dialog, so skip the preview + // aggregation request entirely instead of fetching and discarding it. + const { telemetryConsentQuery, updateTelemetryConsentMutation } = useTelemetryConsent({ enabled: canWrite }); + + const consent = telemetryConsentQuery.data; + // The dialog exists to show the exact payload before the first send, so it + // is skipped when the backend attached no preview envelope. + const preview = consent?.preview ?? null; + const open = + canWrite && + !dismissed && + consent !== undefined && + consent.state === "undecided" && + consent.source !== "env" && + preview !== null; + + if (!open) { + return null; + } + + const busy = updateTelemetryConsentMutation.isPending; + // Dismissing without a decision (ESC, backdrop, close button) persists + // nothing; the dialog may reappear on the next dashboard entry. + const decide = (enabled: boolean) => { + updateTelemetryConsentMutation.mutate({ enabled }, { onSuccess: () => setDismissed(true) }); + }; + + return ( + setDismissed(!nextOpen)}> + + + {t("settings.telemetry.consentDialog.title")} + {t("settings.telemetry.consentDialog.description")} + +
+

+ {t("settings.telemetry.consentDialog.categories")} +

+

{t("settings.telemetry.optOutNotice")}

+

{t("settings.telemetry.consentDialog.payloadLabel")}

+ +

+ + {t("settings.telemetry.consentDialog.docsLink")} + +

+
+ + + + + + + ); +} diff --git a/frontend/src/features/settings/components/telemetry-payload-preview.tsx b/frontend/src/features/settings/components/telemetry-payload-preview.tsx new file mode 100644 index 0000000000..ce983755c2 --- /dev/null +++ b/frontend/src/features/settings/components/telemetry-payload-preview.tsx @@ -0,0 +1,13 @@ +import type { TelemetrySnapshotEnvelope } from "@/features/settings/schemas"; + +export type TelemetryPayloadPreviewProps = { + preview: TelemetrySnapshotEnvelope; +}; + +export function TelemetryPayloadPreview({ preview }: TelemetryPayloadPreviewProps) { + return ( + 
+      {`${JSON.stringify(preview, null, 2)}\n`}
+
+ ); +} diff --git a/frontend/src/features/settings/components/telemetry-settings.test.tsx b/frontend/src/features/settings/components/telemetry-settings.test.tsx new file mode 100644 index 0000000000..643e93ee3a --- /dev/null +++ b/frontend/src/features/settings/components/telemetry-settings.test.tsx @@ -0,0 +1,97 @@ +import { screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { HttpResponse, http } from "msw"; +import { describe, expect, it } from "vitest"; + +import { TelemetrySettings } from "@/features/settings/components/telemetry-settings"; +import i18n from "@/i18n"; +import { createTelemetryConsent, createTelemetrySnapshotEnvelope } from "@/test/mocks/factories"; +import { server } from "@/test/mocks/server"; +import { renderWithProviders } from "@/test/utils"; + +describe("TelemetrySettings", () => { + it("reflects the resolved state and persists a toggle change", async () => { + const user = userEvent.setup(); + let putBody: unknown = null; + server.use( + http.put("/api/settings/telemetry", async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json( + createTelemetryConsent({ state: "disabled", source: "persisted", active: false }), + ); + }), + ); + + // Default mock state is enabled/persisted. + renderWithProviders(); + + const toggle = await screen.findByRole("switch", { name: "Enable anonymous telemetry" }); + await waitFor(() => expect(toggle).toBeChecked()); + expect(toggle).toBeEnabled(); + expect(screen.getByText(i18n.t("settings.telemetry.optOutNotice"))).toBeInTheDocument(); + + await user.click(toggle); + + await waitFor(() => expect(putBody).toEqual({ enabled: false })); + }); + + it("disables the toggle and explains the environment override", async () => { + server.use( + http.get("/api/settings/telemetry", () => + HttpResponse.json(createTelemetryConsent({ state: "disabled", source: "env", active: false })), + ), + ); + + renderWithProviders(); + + const toggle = await screen.findByRole("switch", { name: "Enable anonymous telemetry" }); + await waitFor(() => + expect(screen.getByText(/CODEX_LB_TELEMETRY_ENABLED/)).toBeInTheDocument(), + ); + expect(toggle).toBeDisabled(); + expect(toggle).not.toBeChecked(); + }); + + it("keeps the toggle disabled for read-only sessions", async () => { + renderWithProviders(); + + const toggle = await screen.findByRole("switch", { name: "Enable anonymous telemetry" }); + await waitFor(() => expect(toggle).toBeChecked()); + expect(toggle).toBeDisabled(); + }); + + it("fetches the preview envelope only when the operator opens the dialog", async () => { + const user = userEvent.setup(); + const telemetryRequests: URL[] = []; + server.use( + http.get("/api/settings/telemetry", ({ request }) => { + const url = new URL(request.url); + telemetryRequests.push(url); + if (url.searchParams.get("include_preview") === "true") { + return HttpResponse.json( + createTelemetryConsent({ preview: createTelemetrySnapshotEnvelope() }), + ); + } + return HttpResponse.json(createTelemetryConsent()); + }), + ); + + renderWithProviders(); + + const viewButton = await screen.findByRole("button", { name: "View collected data" }); + await waitFor(() => expect(viewButton).toBeEnabled()); + // The always-on consent query must not carry the expensive preview flag. + expect(telemetryRequests.length).toBeGreaterThan(0); + expect(telemetryRequests.every((url) => !url.searchParams.has("include_preview"))).toBe(true); + + await user.click(viewButton); + + const dialog = await screen.findByRole("dialog", { name: "Collected telemetry data" }); + expect(within(dialog).getByText(/"schema_version": 1/)).toBeInTheDocument(); + expect(within(dialog).getByText(/"consent": "undecided"/)).toBeInTheDocument(); + expect(within(dialog).getByText(/"timestamp": "2026-08-06T00:00:00Z"/)).toBeInTheDocument(); + expect( + telemetryRequests.filter((url) => url.searchParams.get("include_preview") === "true"), + ).toHaveLength(1); + }); +}); diff --git a/frontend/src/features/settings/components/telemetry-settings.tsx b/frontend/src/features/settings/components/telemetry-settings.tsx new file mode 100644 index 0000000000..120b281cff --- /dev/null +++ b/frontend/src/features/settings/components/telemetry-settings.tsx @@ -0,0 +1,108 @@ +import { Activity } from "lucide-react"; +import { useState } from "react"; +import { useTranslation } from "react-i18next"; + +import { AlertMessage } from "@/components/alert-message"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Switch } from "@/components/ui/switch"; +import { TelemetryPayloadPreview } from "@/features/settings/components/telemetry-payload-preview"; +import { useTelemetryConsent, useTelemetryPreview } from "@/features/settings/hooks/use-settings"; + +export type TelemetrySettingsProps = { + disabled: boolean; +}; + +export function TelemetrySettings({ disabled }: TelemetrySettingsProps) { + const { t } = useTranslation(); + const [previewOpen, setPreviewOpen] = useState(false); + const { telemetryConsentQuery, updateTelemetryConsentMutation } = useTelemetryConsent(); + // Building the snapshot is expensive, so the preview is fetched only once + // the operator opens the dialog. + const { telemetryPreviewQuery } = useTelemetryPreview(previewOpen); + + const consent = telemetryConsentQuery.data; + const envControlled = consent?.source === "env"; + const busy = disabled || updateTelemetryConsentMutation.isPending || !consent; + const previewEnvelope = telemetryPreviewQuery.data?.preview ?? null; + + return ( +
+
+
+
+
+
+
+

{t("settings.telemetry.title")}

+

{t("settings.telemetry.description")}

+
+
+ updateTelemetryConsentMutation.mutate({ enabled: checked })} + /> +
+ +

{t("settings.telemetry.optOutNotice")}

+ + {envControlled ? ( +
+ {t("settings.telemetry.envNotice")} +
+ ) : null} + +
+
+

{t("settings.telemetry.collectedData.label")}

+

+ {t("settings.telemetry.collectedData.description")} +

+
+ +
+
+ + + {previewOpen ? ( + + + {t("settings.telemetry.previewDialog.title")} + + {t("settings.telemetry.previewDialog.description")} + + + {previewEnvelope ? ( + + ) : telemetryPreviewQuery.error ? ( + {telemetryPreviewQuery.error.message} + ) : ( + + )} + + + ) : null} + +
+ ); +} diff --git a/frontend/src/features/settings/hooks/use-settings.test.ts b/frontend/src/features/settings/hooks/use-settings.test.ts index 30ecc77e41..1bb6ae9566 100644 --- a/frontend/src/features/settings/hooks/use-settings.test.ts +++ b/frontend/src/features/settings/hooks/use-settings.test.ts @@ -3,7 +3,7 @@ import { renderHook, waitFor } from "@testing-library/react"; import { createElement, type PropsWithChildren } from "react"; import { describe, expect, it, vi } from "vitest"; -import { useSettings } from "@/features/settings/hooks/use-settings"; +import { useSettings, useTelemetryConsent, useTelemetryPreview } from "@/features/settings/hooks/use-settings"; function createTestQueryClient(): QueryClient { return new QueryClient({ @@ -55,3 +55,48 @@ describe("useSettings", () => { }); }); }); + +describe("useTelemetryConsent", () => { + it("loads consent and invalidates cache on decision", async () => { + const queryClient = createTestQueryClient(); + const invalidateSpy = vi.spyOn(queryClient, "invalidateQueries"); + + const { result } = renderHook(() => useTelemetryConsent(), { + wrapper: createWrapper(queryClient), + }); + + await waitFor(() => expect(result.current.telemetryConsentQuery.isSuccess).toBe(true)); + expect(result.current.telemetryConsentQuery.data?.state).toBe("enabled"); + expect(result.current.telemetryConsentQuery.data?.active).toBe(true); + // A persisted decision skips the expensive snapshot build entirely. + expect(result.current.telemetryConsentQuery.data?.preview).toBeNull(); + + await result.current.updateTelemetryConsentMutation.mutateAsync({ enabled: false }); + + await waitFor(() => { + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: ["settings", "telemetry"] }); + }); + }); +}); + +describe("useTelemetryPreview", () => { + it("stays idle until enabled, then loads the preview envelope", async () => { + const queryClient = createTestQueryClient(); + + const { result, rerender } = renderHook(({ enabled }) => useTelemetryPreview(enabled), { + wrapper: createWrapper(queryClient), + initialProps: { enabled: false }, + }); + + expect(result.current.telemetryPreviewQuery.isFetching).toBe(false); + expect(result.current.telemetryPreviewQuery.data).toBeUndefined(); + + rerender({ enabled: true }); + + await waitFor(() => expect(result.current.telemetryPreviewQuery.isSuccess).toBe(true)); + const preview = result.current.telemetryPreviewQuery.data?.preview; + expect(preview?.metrics.schema_version).toBe(1); + expect(preview?.instance_id).toBe("00000000-0000-4000-8000-000000000000"); + expect(preview?.timestamp).toBe("2026-08-06T00:00:00Z"); + }); +}); diff --git a/frontend/src/features/settings/hooks/use-settings.ts b/frontend/src/features/settings/hooks/use-settings.ts index 6934c68d83..6073def607 100644 --- a/frontend/src/features/settings/hooks/use-settings.ts +++ b/frontend/src/features/settings/hooks/use-settings.ts @@ -8,14 +8,17 @@ import { createUpstreamProxyEndpoint, createUpstreamProxyPool, getSettings, + getTelemetryConsent, getUpstreamProxyAdmin, putAccountProxyBinding, testUpstreamProxyEndpoint, updateSettings, + updateTelemetryConsent, } from "@/features/settings/api"; import type { SettingsUpdateRequest } from "@/features/settings/schemas"; import type { AccountProxyBindingRequest, + TelemetryConsentUpdateRequest, UpstreamProxyEndpointCreateRequest, UpstreamProxyPoolCreateRequest, UpstreamProxyPoolMemberRequest, @@ -54,6 +57,49 @@ export function useSettings() { }; } +export function useTelemetryConsent(options?: { enabled?: boolean }) { + const { t } = useTranslation(); + const queryClient = useQueryClient(); + + const { data, error, isFetching, isLoading, isPending, isSuccess, refetch } = useQuery({ + queryKey: ["settings", "telemetry"], + queryFn: () => getTelemetryConsent(), + enabled: options?.enabled ?? true, + }); + const telemetryConsentQuery = { data, error, isFetching, isLoading, isPending, isSuccess, refetch }; + + const updateTelemetryConsentMutation = useMutation({ + mutationFn: (payload: TelemetryConsentUpdateRequest) => updateTelemetryConsent(payload), + onSuccess: () => { + toast.success(t("settings.telemetry.toasts.saved")); + void queryClient.invalidateQueries({ queryKey: ["settings", "telemetry"] }); + }, + onError: (error: Error) => { + toast.error(error.message || t("settings.telemetry.toasts.saveFailed")); + }, + }); + + return { + telemetryConsentQuery, + updateTelemetryConsentMutation, + }; +} + +// On-demand snapshot preview for the settings "View collected data" dialog. +// The snapshot build is expensive, so the query stays idle until `enabled` +// flips true (the dialog opens); consent mutations invalidate it via the +// ["settings", "telemetry"] key prefix. +export function useTelemetryPreview(enabled: boolean) { + const { data, error, isFetching, isLoading, isPending, isSuccess, refetch } = useQuery({ + queryKey: ["settings", "telemetry", "preview"], + queryFn: () => getTelemetryConsent({ includePreview: true }), + enabled, + }); + return { + telemetryPreviewQuery: { data, error, isFetching, isLoading, isPending, isSuccess, refetch }, + }; +} + export function useUpstreamProxyAdmin() { const { t } = useTranslation(); const queryClient = useQueryClient(); diff --git a/frontend/src/features/settings/schemas.test.ts b/frontend/src/features/settings/schemas.test.ts index bcd95f34a2..cacef0ac89 100644 --- a/frontend/src/features/settings/schemas.test.ts +++ b/frontend/src/features/settings/schemas.test.ts @@ -3,8 +3,11 @@ import { describe, expect, it } from "vitest"; import { DashboardSettingsSchema, SettingsUpdateRequestSchema, + TelemetryConsentSchema, + TelemetrySnapshotEnvelopeSchema, UpstreamProxyAdminSchema, } from "@/features/settings/schemas"; +import { createTelemetrySnapshotEnvelope } from "@/test/mocks/factories"; describe("DashboardSettingsSchema", () => { it("parses settings payload", () => { @@ -458,6 +461,93 @@ describe("UpstreamProxyAdminSchema", () => { }); }); +describe("TelemetrySnapshotEnvelopeSchema", () => { + it("parses the exact transmitted envelope", () => { + const parsed = TelemetrySnapshotEnvelopeSchema.parse(createTelemetrySnapshotEnvelope()); + + expect(parsed.instance_id).toBe("00000000-0000-4000-8000-000000000000"); + expect(parsed.timestamp).toBe("2026-08-06T00:00:00Z"); + expect(parsed.metrics.schema_version).toBe(1); + expect(parsed.metrics.deploy.method).toBe("docker"); + expect(parsed.metrics.usage_7d.request_kinds.unknown).toBe(0); + expect(parsed.metrics.usage_7d.models[0]?.reasoning).toEqual({ high: 0.5, medium: 0.5 }); + expect(parsed.metrics.features.dashboard_auth).toBe(true); + }); + + it("rejects unknown extra fields at every object layer so backend drift fails parsing", () => { + // One path per strict object in the envelope tree; loosening any single + // layer back to a non-strict schema fails this test. + const layers: string[][] = [ + [], + ["metrics"], + ["metrics", "deploy"], + ["metrics", "accounts"], + ["metrics", "accounts", "plan_mix"], + ["metrics", "usage_7d"], + ["metrics", "usage_7d", "request_kinds"], + ["metrics", "usage_7d", "transport_mix"], + ["metrics", "usage_7d", "service_tier_mix"], + ["metrics", "usage_7d", "models", "0"], + ["metrics", "features"], + ]; + for (const path of layers) { + const envelope = structuredClone(createTelemetrySnapshotEnvelope()); + let target = envelope as unknown as Record; + for (const key of path) { + target = target[key] as Record; + } + target.drifted_field = true; + expect( + TelemetrySnapshotEnvelopeSchema.safeParse(envelope).success, + `extra field at ${path.join(".") || "envelope root"} must fail parsing`, + ).toBe(false); + } + }); + + it("rejects missing required fields so backend drift fails parsing", () => { + const missingTimestamp = structuredClone(createTelemetrySnapshotEnvelope()) as Record< + string, + unknown + >; + delete missingTimestamp.timestamp; + expect(TelemetrySnapshotEnvelopeSchema.safeParse(missingTimestamp).success).toBe(false); + + const missingNested = structuredClone(createTelemetrySnapshotEnvelope()); + delete (missingNested.metrics.usage_7d.request_kinds as Record).unknown; + expect(TelemetrySnapshotEnvelopeSchema.safeParse(missingNested).success).toBe(false); + }); +}); + +describe("TelemetryConsentSchema", () => { + it("parses consent with and without a preview envelope", () => { + const withPreview = TelemetryConsentSchema.parse({ + state: "undecided", + source: "default", + active: true, + preview: createTelemetrySnapshotEnvelope(), + }); + expect(withPreview.preview?.metrics.schema_version).toBe(1); + + const withoutPreview = TelemetryConsentSchema.parse({ + state: "enabled", + source: "persisted", + active: true, + preview: null, + }); + expect(withoutPreview.preview).toBeNull(); + }); + + it("rejects consent responses that omit the preview field", () => { + expect( + TelemetryConsentSchema.safeParse({ + state: "enabled", + source: "persisted", + active: true, + }).success, + ).toBe(false); + }); +}); + describe("retention fields", () => { it("parses effective values plus overrides, defaulting for older backends", () => { const withValues = DashboardSettingsSchema.parse({ diff --git a/frontend/src/features/settings/schemas.ts b/frontend/src/features/settings/schemas.ts index 245ccd5891..bd97dec03f 100644 --- a/frontend/src/features/settings/schemas.ts +++ b/frontend/src/features/settings/schemas.ts @@ -343,6 +343,133 @@ export const UpstreamProxyAdminSchema = z.object({ bindings: z.array(AccountProxyBindingSchema), }); +export const TelemetryConsentStateSchema = z.enum(["undecided", "enabled", "disabled"]); +export const TelemetryConsentSourceSchema = z.enum(["env", "persisted", "default"]); + +// Wire-format (snake_case) mirror of app/modules/telemetry/schemas.py. Every +// object is strict so backend drift (renamed, added, or removed fields) fails +// schema parsing instead of passing silently. +const TelemetryDeploymentSnapshotSchema = z.strictObject({ + method: z.enum(["docker", "k8s", "pip", "bare"]), + db_backend: z.enum(["sqlite", "postgres"]), + db_size_bucket: z.enum(["unknown", "<100MB", "100MB-1GB", "1-5GB", "5-10GB", "10-50GB", "50GB+"]), + replicas: z.number().int().min(1), + reverse_proxy: z.boolean(), +}); + +const TelemetryPlanMixSnapshotSchema = z.strictObject({ + plus: z.string(), + pro: z.string(), + team: z.string(), + free: z.string(), +}); + +const TelemetryAccountsSnapshotSchema = z.strictObject({ + pool_bucket: z.string(), + plan_mix: TelemetryPlanMixSnapshotSchema, + workspace_accounts: z.boolean(), + routing_policy: z.string(), + limit_warmup_enabled: z.boolean(), + egress_proxy_used: z.boolean(), +}); + +const TelemetryRequestKindsSnapshotSchema = z.strictObject({ + responses: z.number(), + chat: z.number(), + images: z.number(), + unknown: z.number(), +}); + +const TelemetryTransportMixSnapshotSchema = z.strictObject({ + ws: z.number(), + http_bridge: z.number(), +}); + +const TelemetryServiceTierMixSnapshotSchema = z.strictObject({ + default: z.number(), + flex: z.number(), + priority: z.number(), +}); + +const TelemetryModelUsageSnapshotSchema = z.strictObject({ + name: z.string(), + share: z.number(), + reasoning: z.record(z.string(), z.number()), + avg_output_tokens_bucket: z.string(), +}); + +const TelemetryUsageSnapshotSchema = z.strictObject({ + requests: z.number().int().min(0), + success_rate: z.number().min(0).max(1), + tokens_input: z.number().int().min(0), + tokens_output: z.number().int().min(0), + tokens_cached_ratio: z.number().min(0).max(1), + cost_usd_bucket: z.string(), + request_kinds: TelemetryRequestKindsSnapshotSchema, + transport_mix: TelemetryTransportMixSnapshotSchema, + service_tier_mix: TelemetryServiceTierMixSnapshotSchema, + clients: z.record(z.string(), z.number()), + clients_other_ratio: z.number().min(0).max(1), + models: z.array(TelemetryModelUsageSnapshotSchema), + latency_ms_p50: z.number().int().min(0), + ttft_ms_p50: z.number().int().min(0), + ttft_ms_p95: z.number().int().min(0), + rate_limit_429_ratio: z.number().min(0).max(1), + top_upstream_errors: z.array(z.string()).max(5), +}); + +const TelemetryFeaturesSnapshotSchema = z.strictObject({ + api_firewall: z.boolean(), + quota_planner: z.boolean(), + sticky_sessions: z.boolean(), + conversation_archive: z.boolean(), + automations: z.boolean(), + fleet: z.boolean(), + model_sources_count: z.number().int().min(0), + api_keys_bucket: z.string(), + prometheus: z.boolean(), + otel: z.boolean(), + dashboard_auth: z.boolean(), + reset_credits: z.boolean(), + image_api_used: z.boolean(), +}); + +export const TelemetrySnapshotSchema = z.strictObject({ + schema_version: z.literal(1), + consent: z.enum(["undecided", "enabled"]), + instance_id: z.string(), + version: z.string(), + python: z.string(), + os: z.string(), + arch: z.string(), + uptime_hours: z.number().int().min(0), + deploy: TelemetryDeploymentSnapshotSchema, + accounts: TelemetryAccountsSnapshotSchema, + usage_7d: TelemetryUsageSnapshotSchema, + features: TelemetryFeaturesSnapshotSchema, +}); + +// The exact body the instance would transmit; the consent dialog and the +// settings preview render this envelope verbatim. +export const TelemetrySnapshotEnvelopeSchema = z.strictObject({ + instance_id: z.string(), + metrics: TelemetrySnapshotSchema, + timestamp: z.iso.datetime({ offset: true }), +}); + +export const TelemetryConsentSchema = z.object({ + state: TelemetryConsentStateSchema, + source: TelemetryConsentSourceSchema, + active: z.boolean(), + // Present only when the backend built a snapshot: undecided consent with + // default source (the dialog case) or an explicit include_preview request. + preview: TelemetrySnapshotEnvelopeSchema.nullable(), +}); + +export const TelemetryConsentUpdateRequestSchema = z.object({ + enabled: z.boolean(), +}); + export type UpstreamProxyEndpoint = z.infer; export type UpstreamProxyEndpointCreateRequest = z.infer; export type UpstreamProxyEndpointTestResponse = z.infer; @@ -352,3 +479,7 @@ export type UpstreamProxyPoolMemberRequest = z.infer; export type AccountProxyBindingRequest = z.infer; export type UpstreamProxyAdmin = z.infer; +export type TelemetrySnapshot = z.infer; +export type TelemetrySnapshotEnvelope = z.infer; +export type TelemetryConsent = z.infer; +export type TelemetryConsentUpdateRequest = z.infer; diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index ad0282a21a..dc6837fbe7 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -42,7 +42,10 @@ "accounts.list.addAccount": "Add account", "accounts.list.adjustFilters": "Adjust filters", "accounts.list.allStatuses": "All statuses", + "accounts.list.emptyDescription": "Add an account to start routing.", + "accounts.list.emptyTitle": "No accounts yet", "accounts.list.needHelp": "Need help?", + "accounts.list.statusEligibilityNote": "Status shows each account's displayed state. Individual requests can still skip an Active account due to cooldown, error backoff, quota thresholds, model compatibility, or thread-affinity ownership.", "accounts.list.noMatches": "No matching accounts", "accounts.list.searchPlaceholder": "Search accounts...", "accounts.list.sortAria": "Sort accounts", @@ -57,6 +60,7 @@ "accounts.listItem.noAttempts": "No attempts", "accounts.listItem.quotaRemainingAria": "{{label}} quota remaining", "accounts.listItem.resetAt": "Reset {{label}}", + "accounts.listItem.statusActiveHint": "Active is the account's displayed status. Individual requests can still skip this account due to cooldown, error backoff, quota thresholds, model compatibility, or thread-affinity ownership.", "accounts.listItem.warmupOff": "Warm-up off", "accounts.listItem.warmupOn": "Warm-up on", "accounts.oauth.authorizationUrl": "Authorization URL", @@ -209,6 +213,7 @@ "apiKeys.expiry.presets.sevenDays": "7 days", "apiKeys.expiry.presets.thirtyDays": "30 days", "apiKeys.form.allowedModels": "Allowed models", + "apiKeys.form.allowedReasoningEfforts": "Allowed efforts", "apiKeys.form.applyToCodexModel": "Apply to codex /model", "apiKeys.form.assignedAccounts": "Assigned accounts", "apiKeys.form.assignedModelSources": "Assigned model sources", @@ -256,6 +261,10 @@ "apiKeys.modelSelect.selected": "{{count}} model selected", "apiKeys.modelSelect.selected_one": "{{count}} model selected", "apiKeys.modelSelect.selected_other": "{{count}} models selected", + "apiKeys.reasoningEfforts.all": "All efforts", + "apiKeys.reasoningEfforts.selected": "{{count}} effort selected", + "apiKeys.reasoningEfforts.selected_one": "{{count}} effort selected", + "apiKeys.reasoningEfforts.selected_other": "{{count}} efforts selected", "apiKeys.overview.activeKeys": "Active keys", "apiKeys.overview.apiKeys": "API Keys", "apiKeys.overview.costByKey": "Lifetime Cost by API Key", @@ -332,6 +341,8 @@ "apis.keyInfo.noUsageRecorded": "No usage recorded", "apis.keyInfo.title": "Key Details", "apis.list.createKey": "Create API Key", + "apis.list.emptyDescription": "Create an API key to authenticate clients.", + "apis.list.emptyTitle": "No API keys yet", "apis.list.noMatches": "No matching API keys", "apis.list.searchPlaceholder": "Search API keys...", "apis.listItem.apiLimit": "API Limit", @@ -548,6 +559,7 @@ "common.serviceTier.default": "Default", "common.serviceTier.flex": "Flex", "common.serviceTier.priority": "Priority", + "common.serviceTier.ultrafast": "Ultrafast", "common.states.active": "Active", "common.states.disabled": "Disabled", "common.states.enabled": "Enabled", @@ -597,6 +609,7 @@ "dashboard.accountList.ascending": "ascending", "dashboard.accountList.descending": "descending", "dashboard.accountList.disableWarmupAria": "Disable limit warm-up for {{account}}", + "dashboard.accountList.emptyAction": "Add accounts", "dashboard.accountList.emptyDescription": "Import or authenticate an account to get started.", "dashboard.accountList.emptyTitle": "No accounts connected yet", "dashboard.accountList.enableWarmupAria": "Enable limit warm-up for {{account}}", @@ -618,6 +631,7 @@ "dashboard.accountList.sortedAria": "{{label}}, sorted {{direction}}", "dashboard.accountList.viewDetailsAria": "View details for {{account}}", "dashboard.accounts.disableWarmupFor": "Disable limit warm-up for {{account}}", + "dashboard.accounts.emptyAction": "Add accounts", "dashboard.accounts.emptyDescription": "Import or authenticate an account to get started.", "dashboard.accounts.emptyTitle": "No accounts connected yet", "dashboard.accounts.enableWarmupFor": "Enable limit warm-up for {{account}}", @@ -660,6 +674,7 @@ "dashboard.quotaLabels.weekly": "Weekly", "dashboard.requestStatus.error": "Error", "dashboard.requestStatus.ok": "OK", + "dashboard.requestStatus.cancelled": "Cancelled", "dashboard.requestStatus.quota": "Quota", "dashboard.requestStatus.rate_limit": "Rate limit", "dashboard.requestDetails.clientIp": "Client IP", @@ -674,8 +689,16 @@ "dashboard.requestDetails.fullError": "Full Error", "dashboard.requestDetails.noErrorDetail": "No error detail recorded.", "dashboard.requestDetails.queue": "Queue", + "dashboard.requestDetails.reasoningTokensIncluded": "Reasoning tokens (included in output)", "dashboard.requestDetails.requestId": "Request ID", "dashboard.requestDetails.requestKind": "Request kind", + "dashboard.requestDetails.routeEndpoint": "Proxy endpoint", + "dashboard.requestDetails.routeFailClosedReason": "Fail-closed reason", + "dashboard.requestDetails.routeFallback": "Same-pool fallback", + "dashboard.requestDetails.routeFallbackNotUsed": "Not used", + "dashboard.requestDetails.routeFallbackUsed": "Used", + "dashboard.requestDetails.routeMode": "Route mode", + "dashboard.requestDetails.routePool": "Proxy pool", "dashboard.requestDetails.title": "Request Details", "dashboard.requestDetails.userAgent": "User Agent", "dashboard.requests.columns.account": "Account", @@ -688,10 +711,19 @@ "dashboard.requests.columns.time": "Time", "dashboard.requests.columns.tokens": "Tokens", "dashboard.requests.columns.transport": "Transport", + "dashboard.requests.columns.tps": "TPS", + "dashboard.requests.columns.ttft": "TTFT", + "dashboard.requests.columnLayout.columns": "Columns ({{count}})", + "dashboard.requests.columnLayout.restoreDefault": "Restore default column layout", + "dashboard.requests.columnLayout.visibleColumns": "Visible columns", "dashboard.requests.downstreamTransport": "Downstream client transport", - "dashboard.requests.emptyDescription": "No request logs match the current filters.", + "dashboard.requests.emptyDescription": "Requests will appear here after clients start using the proxy.", + "dashboard.requests.emptyFilteredDescription": "No request logs match the current filters.", + "dashboard.requests.emptyFilteredTitle": "No matching requests", "dashboard.requests.emptyTitle": "No requests yet", "dashboard.requests.requestedTier": "Requested {{tier}}", + "dashboard.requests.reasoningTokensShort": "{{count}} reasoning", + "dashboard.requests.resizeColumn": "Resize {{column}} column", "dashboard.requests.title": "Request Logs", "dashboard.requests.unassigned": "Unassigned", "dashboard.requests.upstreamTransport": "Up {{transport}}", @@ -838,6 +870,7 @@ "modelSources.editDialog.title": "Edit model source", "modelSources.empty": "No model sources configured.", "modelSources.capabilities.audioTranscriptions": "Audio transcriptions", + "modelSources.capabilities.embeddings": "Embeddings", "modelSources.capabilities.chatCompletions": "Chat completions", "modelSources.capabilities.reasoning": "Reasoning", "modelSources.capabilities.responses": "Responses", @@ -926,6 +959,8 @@ "quotaPlanner.toasts.warmupFailed": "Failed to request quota warmup", "quotaPlanner.toasts.warmupResult": "Warmup {{status}}: {{reason}}", "reports.charts.costByDay": "Cost by Day", + "reports.charts.emptyDescription": "No usage recorded for the selected range.", + "reports.charts.emptyTitle": "No data", "reports.charts.input": "Input", "reports.charts.medianQueueWait": "Median queue wait", "reports.charts.medianTps": "Median TPS", @@ -936,14 +971,18 @@ "reports.charts.tokensByDay": "Tokens by Day", "reports.charts.tokensPerSecond": "Tokens per Second", "reports.dailyBreakdown.columns.accounts": "Accounts", + "reports.dailyBreakdown.columns.cancelled": "Cancelled", "reports.dailyBreakdown.columns.cost": "Cost", "reports.dailyBreakdown.columns.day": "Day", + "reports.dailyBreakdown.columns.errors": "Errors", "reports.dailyBreakdown.columns.inputTokens": "Input Tokens", "reports.dailyBreakdown.columns.outputTokens": "Output Tokens", + "reports.dailyBreakdown.columns.reasoningTokens": "Reported Reasoning Tokens", "reports.dailyBreakdown.columns.reqs": "Reqs", "reports.dailyBreakdown.csv": "CSV", "reports.dailyBreakdown.csvColumns.activeAccounts": "Active Accounts", "reports.dailyBreakdown.csvColumns.cachedTokens": "Cached Tokens", + "reports.dailyBreakdown.csvColumns.cancelled": "Cancelled", "reports.dailyBreakdown.columns.conversations": "Conversations", "reports.dailyBreakdown.csvColumns.conversations": "Conversations", "reports.dailyBreakdown.csvColumns.costUsd": "Cost USD", @@ -951,6 +990,7 @@ "reports.dailyBreakdown.csvColumns.errors": "Errors", "reports.dailyBreakdown.csvColumns.inputTokens": "Input Tokens", "reports.dailyBreakdown.csvColumns.outputTokens": "Output Tokens", + "reports.dailyBreakdown.csvColumns.reasoningTokens": "Reported Reasoning Tokens", "reports.dailyBreakdown.csvColumns.requests": "Requests", "reports.dailyBreakdown.title": "Daily Breakdown", "reports.distribution.byModel": "Distribution by Model", @@ -958,6 +998,7 @@ "reports.distribution.missingUserAgent": "Missing User-Agent", "reports.distribution.total": "Total", "reports.errors.accounts": "Failed to load account options: {{error}}", + "reports.errors.apiKeys": "Failed to load API key options: {{error}}", "reports.errors.data": "Failed to load report data: {{error}}", "reports.errors.options": "Failed to load model and user-agent options: {{error}}", "reports.errors.partial": "Some report data could not be loaded. Try reloading.", @@ -969,10 +1010,13 @@ "reports.page.subtitle": "Usage history by date range", "reports.page.title": "Cost Report", "reports.summary.avgCostPerDay": "avg {{cost}}/day", + "reports.summary.cancelled": "Cancelled", + "reports.summary.errors": "Errors", "reports.summary.requests": "Requests", "reports.summary.requestsSub": "avg {{requests}}/day · {{accounts}} accounts", "reports.summary.tokens": "Tokens", "reports.summary.tokensSub": "Input {{input}} · Cache {{cache}} · Output {{output}}", + "reports.summary.reasoningSub": "Reported reasoning {{reasoning}} (included in output) · {{known}}/{{total}} requests", "reports.summary.totalCost": "Total Cost", "reports.summary.conversations": "Active Conversations", "settings.page.title": "Settings", @@ -1071,10 +1115,11 @@ "settings.routing.strategy.guide.singleAccount": "Pins all routed traffic to one selected active account.", "settings.routing.strategy.safetyNote": "No strategy can guarantee account-safety outcomes. For low-volume policy-compliant use, prefer capacity weighted or relative availability with sticky threads enabled so traffic stays steady and session-local.", "settings.routing.stickyThreads.label": "Sticky threads", - "settings.routing.stickyThreads.description": "Keep related requests on the same account.", + "settings.routing.stickyThreads.description": "Keep normal requests for the same session on the same account when possible. This is a soft preference, not a guarantee.", + "settings.routing.stickyThreads.hardAffinityNote": "Turning this off does not disable hard Codex continuation affinity: a request that carries Codex continuation state (turn state, a previous response, or uploaded files) may still require its original owner account unless codex-lb can safely migrate or replay it.", "settings.routing.stickyThreads.ariaLabel": "Enable sticky threads", "settings.routing.preferEarlier.label": "Prefer earlier reset", - "settings.routing.preferEarlier.description": "Bias traffic to accounts with earlier quota reset.", + "settings.routing.preferEarlier.description": "When several accounts are otherwise eligible, prefer those whose selected quota window resets sooner. Weekly resets are compared by day. Applies to the capacity weighted, usage weighted, and fill first strategies.", "settings.routing.preferEarlier.ariaLabel": "Prefer earlier reset accounts", "settings.routing.preferEarlier.windowAria": "Reset preference window", "settings.routing.promptCacheTtl.label": "Prompt-cache affinity TTL", @@ -1118,11 +1163,14 @@ "settings.routing.accountCapacity.fairShareThresholdDescription": "When in-flight streams reach this percentage of pool stream capacity, keys holding more than their fair share are throttled until the pool decongests. 0 disables.", "settings.routing.accountCapacity.save": "Save capacity limits", "settings.routing.stickyThresholds.primaryLabel": "Sticky primary threshold", - "settings.routing.stickyThresholds.primaryDescription": "Reallocate sticky sessions above this primary usage percent.", + "settings.routing.stickyThresholds.primaryDescription": "Move sticky sessions off an account once more than this percent of its 5-hour (primary) window has been used.", "settings.routing.stickyThresholds.savePrimary": "Save primary", "settings.routing.stickyThresholds.secondaryLabel": "Sticky secondary threshold", - "settings.routing.stickyThresholds.secondaryDescription": "Reallocate sticky sessions above this secondary usage percent.", + "settings.routing.stickyThresholds.secondaryDescription": "Move sticky sessions off an account once more than this percent of its secondary window (weekly, or monthly on monthly-only plans) has been used.", "settings.routing.stickyThresholds.saveSecondary": "Save secondary", + "settings.routing.stickyThresholds.usedRemainingHint": "{{used}}% used · {{remaining}}% remaining in quota terms", + "settings.routing.quotaWindows.title": "Primary vs secondary quota", + "settings.routing.quotaWindows.explainer": "Primary quota is the short 5-hour usage window; secondary quota is the longer window (weekly, or monthly on plans without a weekly window). Account pages show each window as percent remaining, while the sticky thresholds below are percent used. Routing also counts in-flight work as temporary extra usage, so thresholds can trigger slightly before the account page shows the matching remaining percent.", "settings.routing.quotaWindows.weekly": "Weekly quota", "settings.routing.quotaWindows.fiveHour": "5h quota", "settings.routing.workingDays.label": "Weekly pace working days", @@ -1138,7 +1186,7 @@ "settings.routing.paceSmoothing.label": "Pace gap average", "settings.routing.paceSmoothing.description": "Smooth the displayed weekly pace gap over recent quota samples.", "settings.routing.limitWarmup.label": "Limit warm-up", - "settings.routing.limitWarmup.description": "Send one reset-confirmed warm-up for opted-in accounts.", + "settings.routing.limitWarmup.description": "When an opted-in account's quota window is confirmed to have newly reset, send one small probe request to verify the account responds. Probes are real requests using the model and prompt below and consume a small amount of quota.", "settings.routing.limitWarmup.ariaLabel": "Enable limit warm-up", "settings.routing.limitWarmup.modelAria": "Warm-up model", "settings.routing.limitWarmup.cooldownAria": "Warm-up cooldown", @@ -1242,6 +1290,25 @@ "settings.password.validation.required": "This field is required.", "settings.password.validation.minLength": "Password must be at least 8 characters.", "settings.password.validation.maxByteLength": "Password must be at most 72 bytes when encoded as UTF-8.", + "settings.telemetry.title": "Anonymous telemetry", + "settings.telemetry.description": "Share anonymous usage statistics to help improve codex-lb.", + "settings.telemetry.toggleAria": "Enable anonymous telemetry", + "settings.telemetry.envNotice": "Telemetry is controlled by the CODEX_LB_TELEMETRY_ENABLED environment variable. Unset it to manage this setting from the dashboard.", + "settings.telemetry.optOutNotice": "Disabling from the dashboard sends one anonymous opt-out notice to keep aggregate counts accurate.", + "settings.telemetry.collectedData.label": "Collected data", + "settings.telemetry.collectedData.description": "Review the exact anonymous payload this instance would send.", + "settings.telemetry.collectedData.view": "View collected data", + "settings.telemetry.previewDialog.title": "Collected telemetry data", + "settings.telemetry.previewDialog.description": "The exact payload this instance would send. It contains no accounts, no prompts, no API keys, and no IP addresses.", + "settings.telemetry.consentDialog.title": "Anonymous telemetry", + "settings.telemetry.consentDialog.description": "codex-lb collects anonymous usage telemetry by default to help guide development. You can change this choice at any time in Settings.", + "settings.telemetry.consentDialog.categories": "Only the version, deployment shape, and aggregated usage statistics are collected — no accounts, no prompts, no API keys, and no IP addresses.", + "settings.telemetry.consentDialog.payloadLabel": "Exact payload this instance would send:", + "settings.telemetry.consentDialog.docsLink": "Learn what is collected and why", + "settings.telemetry.consentDialog.keepEnabled": "Keep enabled", + "settings.telemetry.consentDialog.disable": "Disable telemetry", + "settings.telemetry.toasts.saved": "Telemetry preference saved", + "settings.telemetry.toasts.saveFailed": "Failed to save telemetry preference", "settings.totp.title": "TOTP", "settings.totp.status.configured": "TOTP is configured.", "settings.totp.status.notConfigured": "No TOTP configured.", @@ -1382,5 +1449,9 @@ "upstreamProxy.toasts.poolUpdateFailed": "Proxy pool update failed", "upstreamProxy.validation.hostRequired": "Host is required", "upstreamProxy.validation.nameRequired": "Name is required", - "upstreamProxy.validation.portInvalid": "Enter a port between 1 and 65535" + "upstreamProxy.validation.portInvalid": "Enter a port between 1 and 65535", + "modelSources.fields.reasoningEfforts": "Supported reasoning efforts", + "modelSources.fields.reasoningEffortsDescription": "List the effort slugs this model source accepts and choose which one should be the default.", + "modelSources.fields.reasoningEffortsPlaceholder": "none, low, provider-specific", + "modelSources.fields.defaultReasoningEffort": "Default reasoning effort" } diff --git a/frontend/src/i18n/locales/ko.json b/frontend/src/i18n/locales/ko.json index 7851821adc..078f7930ab 100644 --- a/frontend/src/i18n/locales/ko.json +++ b/frontend/src/i18n/locales/ko.json @@ -42,11 +42,14 @@ "accounts.list.addAccount": "Account 추가", "accounts.list.adjustFilters": "필터 조정", "accounts.list.allStatuses": "전체 상태", + "accounts.list.emptyDescription": "라우팅을 시작하려면 Account를 추가하세요.", + "accounts.list.emptyTitle": "아직 Account가 없습니다", "accounts.list.needHelp": "도움이 필요하신가요?", "accounts.list.noMatches": "일치하는 Account가 없습니다", "accounts.list.searchPlaceholder": "Account 검색...", "accounts.list.sortAria": "Account 정렬", "accounts.list.sortPlaceholder": "Account 정렬", + "accounts.list.statusEligibilityNote": "상태는 각 Account의 표시 상태입니다. cooldown, error backoff, quota 임계값, model 호환성, thread affinity 소유권 때문에 개별 요청은 Active Account도 건너뛸 수 있습니다.", "accounts.list.statusFilterAria": "상태로 Account 필터링", "accounts.list.statusPlaceholder": "상태로 필터", "accounts.sort.most_reset_credits": "reset credit 많은 순", @@ -57,6 +60,7 @@ "accounts.listItem.noAttempts": "시도 없음", "accounts.listItem.quotaRemainingAria": "{{label}} quota 남음", "accounts.listItem.resetAt": "{{label}} reset", + "accounts.listItem.statusActiveHint": "Active는 Account의 표시 상태입니다. cooldown, error backoff, quota 임계값, model 호환성, thread affinity 소유권 때문에 개별 요청은 이 Account를 건너뛸 수 있습니다.", "accounts.listItem.warmupOff": "Warm-up 꺼짐", "accounts.listItem.warmupOn": "Warm-up 켜짐", "accounts.oauth.authorizationUrl": "Authorization URL", @@ -209,6 +213,7 @@ "apiKeys.expiry.presets.sevenDays": "7일", "apiKeys.expiry.presets.thirtyDays": "30일", "apiKeys.form.allowedModels": "허용 Model", + "apiKeys.form.allowedReasoningEfforts": "허용된 추론 강도", "apiKeys.form.applyToCodexModel": "codex /model에 적용", "apiKeys.form.assignedAccounts": "할당 Account", "apiKeys.form.assignedModelSources": "할당 Model source", @@ -256,6 +261,10 @@ "apiKeys.modelSelect.selected": "{{count}}개 Model 선택됨", "apiKeys.modelSelect.selected_one": "{{count}}개 Model 선택됨", "apiKeys.modelSelect.selected_other": "{{count}}개 Model 선택됨", + "apiKeys.reasoningEfforts.all": "모든 추론 강도", + "apiKeys.reasoningEfforts.selected": "{{count}}개 추론 강도 선택됨", + "apiKeys.reasoningEfforts.selected_one": "{{count}}개 추론 강도 선택됨", + "apiKeys.reasoningEfforts.selected_other": "{{count}}개 추론 강도 선택됨", "apiKeys.overview.activeKeys": "활성 key", "apiKeys.overview.apiKeys": "API Key", "apiKeys.overview.costByKey": "API Key별 누적 비용", @@ -332,6 +341,8 @@ "apis.keyInfo.noUsageRecorded": "기록된 usage 없음", "apis.keyInfo.title": "Key 정보", "apis.list.createKey": "API Key 생성", + "apis.list.emptyDescription": "클라이언트를 인증하려면 API Key를 만드세요.", + "apis.list.emptyTitle": "아직 API Key가 없습니다", "apis.list.noMatches": "일치하는 API Key가 없습니다", "apis.list.searchPlaceholder": "API Key 검색...", "apis.listItem.apiLimit": "API Limit", @@ -548,6 +559,7 @@ "common.serviceTier.default": "Default", "common.serviceTier.flex": "Flex", "common.serviceTier.priority": "Priority", + "common.serviceTier.ultrafast": "Ultrafast", "common.states.active": "활성", "common.states.disabled": "꺼짐", "common.states.enabled": "켜짐", @@ -597,6 +609,7 @@ "dashboard.accountList.ascending": "ascending", "dashboard.accountList.descending": "descending", "dashboard.accountList.disableWarmupAria": "{{account}} limit warm-up 비활성화", + "dashboard.accountList.emptyAction": "Account 추가", "dashboard.accountList.emptyDescription": "시작하려면 Account를 import하거나 인증하세요.", "dashboard.accountList.emptyTitle": "연결된 Account가 없습니다", "dashboard.accountList.enableWarmupAria": "{{account}} limit warm-up 활성화", @@ -618,6 +631,7 @@ "dashboard.accountList.sortedAria": "{{label}}, {{direction}} 정렬", "dashboard.accountList.viewDetailsAria": "{{account}} 상세 보기", "dashboard.accounts.disableWarmupFor": "{{account}}의 limit warm-up 끄기", + "dashboard.accounts.emptyAction": "Account 추가", "dashboard.accounts.emptyDescription": "시작하려면 Account를 import하거나 인증하세요.", "dashboard.accounts.emptyTitle": "연결된 Account가 없습니다", "dashboard.accounts.enableWarmupFor": "{{account}}의 limit warm-up 켜기", @@ -660,6 +674,7 @@ "dashboard.quotaLabels.weekly": "Weekly", "dashboard.requestStatus.error": "오류", "dashboard.requestStatus.ok": "정상", + "dashboard.requestStatus.cancelled": "취소됨", "dashboard.requestStatus.quota": "Quota", "dashboard.requestStatus.rate_limit": "Rate limit", "dashboard.requestDetails.clientIp": "Client IP", @@ -674,8 +689,16 @@ "dashboard.requestDetails.fullError": "전체 오류", "dashboard.requestDetails.noErrorDetail": "기록된 오류 상세가 없습니다.", "dashboard.requestDetails.queue": "Queue", + "dashboard.requestDetails.reasoningTokensIncluded": "추론 token (Output token에 포함)", "dashboard.requestDetails.requestId": "Request ID", "dashboard.requestDetails.requestKind": "요청 종류", + "dashboard.requestDetails.routeEndpoint": "프록시 엔드포인트", + "dashboard.requestDetails.routeFailClosedReason": "실패 종료 사유", + "dashboard.requestDetails.routeFallback": "동일 풀 폴백", + "dashboard.requestDetails.routeFallbackNotUsed": "사용되지 않음", + "dashboard.requestDetails.routeFallbackUsed": "사용됨", + "dashboard.requestDetails.routeMode": "라우팅 모드", + "dashboard.requestDetails.routePool": "프록시 풀", "dashboard.requestDetails.title": "요청 상세", "dashboard.requestDetails.userAgent": "User Agent", "dashboard.requests.columns.account": "Account", @@ -688,10 +711,19 @@ "dashboard.requests.columns.time": "시간", "dashboard.requests.columns.tokens": "token", "dashboard.requests.columns.transport": "Transport", + "dashboard.requests.columns.tps": "TPS", + "dashboard.requests.columns.ttft": "TTFT", + "dashboard.requests.columnLayout.columns": "열 ({{count}})", + "dashboard.requests.columnLayout.restoreDefault": "기본 열 레이아웃 복원", + "dashboard.requests.columnLayout.visibleColumns": "표시할 열", "dashboard.requests.downstreamTransport": "Downstream transport", - "dashboard.requests.emptyDescription": "현재 필터와 일치하는 request log가 없습니다.", + "dashboard.requests.emptyDescription": "클라이언트가 프록시를 사용하면 요청이 여기에 표시됩니다.", + "dashboard.requests.emptyFilteredDescription": "현재 필터와 일치하는 request log가 없습니다.", + "dashboard.requests.emptyFilteredTitle": "일치하는 요청이 없습니다", "dashboard.requests.emptyTitle": "아직 요청이 없습니다", "dashboard.requests.requestedTier": "Requested {{tier}}", + "dashboard.requests.reasoningTokensShort": "추론 {{count}}", + "dashboard.requests.resizeColumn": "{{column}} 열 크기 조절", "dashboard.requests.title": "요청 로그", "dashboard.requests.unassigned": "미할당", "dashboard.requests.upstreamTransport": "Up {{transport}}", @@ -838,6 +870,7 @@ "modelSources.editDialog.title": "Model source 수정", "modelSources.empty": "항목 없음", "modelSources.capabilities.audioTranscriptions": "Audio transcriptions", + "modelSources.capabilities.embeddings": "Embeddings", "modelSources.capabilities.chatCompletions": "Chat completions", "modelSources.capabilities.reasoning": "Reasoning", "modelSources.capabilities.responses": "Responses", @@ -926,6 +959,8 @@ "quotaPlanner.toasts.warmupFailed": "Warm-up 실패", "quotaPlanner.toasts.warmupResult": "Warm-up {{status}}: {{reason}}", "reports.charts.costByDay": "일별 비용", + "reports.charts.emptyDescription": "선택한 기간에 사용 기록이 없습니다.", + "reports.charts.emptyTitle": "데이터 없음", "reports.charts.input": "Input", "reports.charts.medianQueueWait": "Median queue wait", "reports.charts.medianTps": "Median TPS", @@ -936,19 +971,24 @@ "reports.charts.tokensByDay": "일별 token", "reports.charts.tokensPerSecond": "Tokens per Second", "reports.dailyBreakdown.columns.accounts": "Accounts", + "reports.dailyBreakdown.columns.cancelled": "취소됨", "reports.dailyBreakdown.columns.cost": "비용", "reports.dailyBreakdown.columns.day": "일자", + "reports.dailyBreakdown.columns.errors": "오류", "reports.dailyBreakdown.columns.inputTokens": "Input token", "reports.dailyBreakdown.columns.outputTokens": "Output token", + "reports.dailyBreakdown.columns.reasoningTokens": "보고된 추론 token", "reports.dailyBreakdown.columns.reqs": "요청", "reports.dailyBreakdown.csv": "CSV", "reports.dailyBreakdown.csvColumns.activeAccounts": "활성 Accounts", "reports.dailyBreakdown.csvColumns.cachedTokens": "Cached token", + "reports.dailyBreakdown.csvColumns.cancelled": "취소됨", "reports.dailyBreakdown.csvColumns.costUsd": "Cost USD", "reports.dailyBreakdown.csvColumns.date": "날짜", "reports.dailyBreakdown.csvColumns.errors": "오류", "reports.dailyBreakdown.csvColumns.inputTokens": "Input token", "reports.dailyBreakdown.csvColumns.outputTokens": "Output token", + "reports.dailyBreakdown.csvColumns.reasoningTokens": "보고된 추론 token", "reports.dailyBreakdown.csvColumns.requests": "요청", "reports.dailyBreakdown.title": "일별 상세", "reports.distribution.byModel": "Model별 분포", @@ -956,6 +996,7 @@ "reports.distribution.missingUserAgent": "User-Agent 없음", "reports.distribution.total": "합계", "reports.errors.accounts": "Account 옵션을 불러오지 못했습니다: {{error}}", + "reports.errors.apiKeys": "API 키 옵션을 불러오지 못했습니다: {{error}}", "reports.errors.data": "report data를 불러오지 못했습니다: {{error}}", "reports.errors.options": "Model 및 User Agent 옵션을 불러오지 못했습니다: {{error}}", "reports.errors.partial": "일부 report data를 불러오지 못했습니다. 다시 불러오세요.", @@ -967,11 +1008,14 @@ "reports.page.subtitle": "기간별 사용 기록", "reports.page.title": "비용 리포트", "reports.summary.avgCostPerDay": "평균 {{cost}}/일", + "reports.summary.cancelled": "취소됨", "reports.summary.conversations": "활성 대화", + "reports.summary.errors": "오류", "reports.summary.requests": "요청", "reports.summary.requestsSub": "평균 {{requests}}/일 · {{accounts}} Accounts", "reports.summary.tokens": "Token", "reports.summary.tokensSub": "Input {{input}} · Cache {{cache}} · Output {{output}}", + "reports.summary.reasoningSub": "보고된 추론 {{reasoning}} (Output 일부) · {{known}}/{{total}}개 요청", "reports.summary.totalCost": "총 비용", "reports.dailyBreakdown.columns.conversations": "대화", "reports.dailyBreakdown.csvColumns.conversations": "대화", @@ -1136,7 +1180,7 @@ "settings.routing.httpDownstream.policies.smart": "Session-aware", "settings.routing.limitWarmup.ariaLabel": "Limit warm-up 활성화", "settings.routing.limitWarmup.cooldownAria": "Warm-up cooldown", - "settings.routing.limitWarmup.description": "opt-in Account에 reset-confirmed warm-up을 1회 보냅니다.", + "settings.routing.limitWarmup.description": "opt-in한 Account의 quota window가 새로 reset된 것이 확인되면, 작은 probe 요청 1건을 보내 Account가 응답하는지 확인합니다. probe는 아래 model·prompt를 사용하는 실제 요청이며 소량의 quota를 소모합니다.", "settings.routing.limitWarmup.label": "Limit warm-up", "settings.routing.limitWarmup.modelAria": "Warm-up model", "settings.routing.limitWarmup.promptAria": "Warm-up prompt", @@ -1165,13 +1209,15 @@ "settings.routing.paceSmoothing.description": "표시되는 weekly pace gap을 최근 quota sample 기준으로 부드럽게 만듭니다.", "settings.routing.paceSmoothing.label": "Pace gap average", "settings.routing.preferEarlier.ariaLabel": "이른 reset Account 선호", - "settings.routing.preferEarlier.description": "quota reset이 더 이른 Account에 traffic을 우선 배정합니다.", + "settings.routing.preferEarlier.description": "여러 Account가 모두 적격일 때 선택한 quota window가 더 빨리 reset되는 Account를 우선합니다. weekly reset은 일 단위로 비교합니다. capacity weighted, usage weighted, fill first 전략에 적용됩니다.", "settings.routing.preferEarlier.label": "이른 reset 선호", "settings.routing.preferEarlier.windowAria": "Reset preference window", "settings.routing.promptCacheTtl.description": "OpenAI 스타일 prompt-cache mapping을 제한된 초 동안 유지합니다.", "settings.routing.promptCacheTtl.label": "Prompt-cache affinity TTL", "settings.routing.promptCacheTtl.save": "TTL 저장", + "settings.routing.quotaWindows.explainer": "Primary quota는 짧은 5시간 사용량 window이고, secondary quota는 더 긴 window(weekly, weekly window가 없는 plan에서는 monthly)입니다. Account 페이지는 각 window를 남은 percent로 표시하지만, 아래 sticky threshold는 사용된 percent 기준입니다. 라우팅은 in-flight 작업도 일시적인 추가 사용량으로 계산하므로, Account 페이지에 해당 남은 percent가 표시되기 전에 threshold가 먼저 발동할 수 있습니다.", "settings.routing.quotaWindows.fiveHour": "5h quota", + "settings.routing.quotaWindows.title": "Primary vs secondary quota", "settings.routing.quotaWindows.weekly": "Weekly quota", "settings.routing.relativeAvailability.powerDescription": "정규화된 relative-availability score를 weighted selection 전에 이 power로 올립니다.", "settings.routing.relativeAvailability.powerLabel": "Relative availability power", @@ -1185,14 +1231,16 @@ "settings.routing.singleAccount.loading": "Accounts 불러오는 중...", "settings.routing.singleAccount.placeholder": "Account 선택", "settings.routing.stickyThreads.ariaLabel": "Sticky threads 활성화", - "settings.routing.stickyThreads.description": "관련 요청을 같은 Account에 유지합니다.", + "settings.routing.stickyThreads.description": "같은 session의 일반 요청을 가능하면 같은 Account에 유지합니다. 보장이 아닌 soft 선호입니다.", + "settings.routing.stickyThreads.hardAffinityNote": "이 옵션을 꺼도 hard Codex continuation affinity는 비활성화되지 않습니다. Codex continuation 상태(turn state, 이전 response, 업로드된 파일)를 가진 요청은 codex-lb가 안전하게 migrate/replay하지 못하는 한 여전히 원래 owner Account가 필요할 수 있습니다.", "settings.routing.stickyThreads.label": "Sticky threads", - "settings.routing.stickyThresholds.primaryDescription": "이 primary 사용량 percent를 넘으면 sticky session을 재할당합니다.", + "settings.routing.stickyThresholds.primaryDescription": "Account의 5시간(primary) window 사용량이 이 percent를 넘으면 sticky session을 다른 Account로 옮깁니다.", "settings.routing.stickyThresholds.primaryLabel": "Sticky primary threshold", "settings.routing.stickyThresholds.savePrimary": "Primary 저장", "settings.routing.stickyThresholds.saveSecondary": "Secondary 저장", - "settings.routing.stickyThresholds.secondaryDescription": "이 secondary 사용량 percent를 넘으면 sticky session을 재할당합니다.", + "settings.routing.stickyThresholds.secondaryDescription": "Account의 secondary window(weekly, monthly 전용 plan에서는 monthly) 사용량이 이 percent를 넘으면 sticky session을 다른 Account로 옮깁니다.", "settings.routing.stickyThresholds.secondaryLabel": "Sticky secondary threshold", + "settings.routing.stickyThresholds.usedRemainingHint": "{{used}}% 사용 · quota 기준 남은 {{remaining}}%", "settings.routing.strategy.capacityWeighted": "Capacity weighted", "settings.routing.strategy.description": "요청이 Accounts 사이에 분배되는 방식을 선택합니다.", "settings.routing.strategy.fillFirst": "Fill first", @@ -1242,6 +1290,25 @@ "settings.session.lifetime.longWarning": "30일을 넘는 유지 시간은 admin session을 오래 유지합니다. 개인 laptop에서는 괜찮을 수 있지만 browser profile이나 cookie가 유출되면 영향이 커집니다.", "settings.session.lifetime.save": "유지 시간 저장", "settings.session.title": "Session", + "settings.telemetry.title": "익명 텔레메트리", + "settings.telemetry.description": "익명 사용 통계를 공유해 codex-lb 개선에 도움을 줍니다.", + "settings.telemetry.toggleAria": "익명 텔레메트리 사용", + "settings.telemetry.envNotice": "텔레메트리는 CODEX_LB_TELEMETRY_ENABLED 환경 변수로 제어되고 있습니다. 대시보드에서 이 설정을 관리하려면 해당 변수를 해제하세요.", + "settings.telemetry.optOutNotice": "대시보드에서 텔레메트리를 비활성화하면 집계 수치의 정확성을 유지하기 위해 익명 비활성화 알림을 한 번 전송합니다.", + "settings.telemetry.collectedData.label": "수집 데이터", + "settings.telemetry.collectedData.description": "이 인스턴스가 전송할 익명 payload 원문을 확인할 수 있습니다.", + "settings.telemetry.collectedData.view": "수집 데이터 보기", + "settings.telemetry.previewDialog.title": "수집되는 텔레메트리 데이터", + "settings.telemetry.previewDialog.description": "이 인스턴스가 전송할 payload 원문입니다. 계정, 프롬프트, API 키, IP 주소는 포함되지 않습니다.", + "settings.telemetry.consentDialog.title": "익명 텔레메트리", + "settings.telemetry.consentDialog.description": "codex-lb는 개발 방향 결정에 도움이 되도록 기본적으로 익명 사용 텔레메트리를 수집합니다. 이 선택은 언제든지 Settings에서 변경할 수 있습니다.", + "settings.telemetry.consentDialog.categories": "버전, 배포 형태, 집계된 사용 통계만 수집됩니다 — 계정, 프롬프트, API 키, IP 주소는 수집되지 않습니다.", + "settings.telemetry.consentDialog.payloadLabel": "이 인스턴스가 전송할 payload 원문:", + "settings.telemetry.consentDialog.docsLink": "수집 항목과 이유 알아보기", + "settings.telemetry.consentDialog.keepEnabled": "계속 사용", + "settings.telemetry.consentDialog.disable": "텔레메트리 비활성화", + "settings.telemetry.toasts.saved": "텔레메트리 설정이 저장되었습니다", + "settings.telemetry.toasts.saveFailed": "텔레메트리 설정 저장에 실패했습니다", "settings.toasts.saved": "설정 저장됨", "settings.toasts.saveFailed": "설정 저장 실패", "settings.totp.actions.disable": "비활성화", @@ -1382,5 +1449,9 @@ "upstreamProxy.toasts.poolUpdateFailed": "Pool 업데이트 실패", "upstreamProxy.validation.hostRequired": "Host는 필수입니다", "upstreamProxy.validation.nameRequired": "이름은 필수입니다", - "upstreamProxy.validation.portInvalid": "1부터 65535 사이의 port를 입력하세요" + "upstreamProxy.validation.portInvalid": "1부터 65535 사이의 port를 입력하세요", + "modelSources.fields.reasoningEfforts": "지원되는 추론 수준", + "modelSources.fields.reasoningEffortsDescription": "이 모델 소스가 허용하는 추론 수준 슬러그를 적고 기본값을 고르세요.", + "modelSources.fields.reasoningEffortsPlaceholder": "none, low, provider-specific", + "modelSources.fields.defaultReasoningEffort": "기본 추론 수준" } diff --git a/frontend/src/i18n/locales/zh-CN.json b/frontend/src/i18n/locales/zh-CN.json index a51cc0b0c5..8f32848bd5 100644 --- a/frontend/src/i18n/locales/zh-CN.json +++ b/frontend/src/i18n/locales/zh-CN.json @@ -42,11 +42,14 @@ "accounts.list.addAccount": "添加账户", "accounts.list.adjustFilters": "调整筛选", "accounts.list.allStatuses": "全部状态", + "accounts.list.emptyDescription": "添加账户以开始路由。", + "accounts.list.emptyTitle": "暂无账户", "accounts.list.needHelp": "需要帮助？", "accounts.list.noMatches": "没有匹配账户", "accounts.list.searchPlaceholder": "搜索账户...", "accounts.list.sortAria": "账户排序", "accounts.list.sortPlaceholder": "账户排序", + "accounts.list.statusEligibilityNote": "状态显示的是各账户的显示状态。个别请求仍可能因冷却、错误退避、配额阈值、模型兼容性或线程亲和所有权而跳过 Active 账户。", "accounts.list.statusFilterAria": "按状态筛选账户", "accounts.list.statusPlaceholder": "按状态筛选", "accounts.sort.most_reset_credits": "reset credit 最多", @@ -57,6 +60,7 @@ "accounts.listItem.noAttempts": "无尝试", "accounts.listItem.quotaRemainingAria": "{{label}} quota 剩余", "accounts.listItem.resetAt": "{{label}} reset", + "accounts.listItem.statusActiveHint": "Active 表示账户的显示状态。个别请求仍可能因冷却、错误退避、配额阈值、模型兼容性或线程亲和所有权而跳过该账户。", "accounts.listItem.warmupOff": "Warm-up 已关闭", "accounts.listItem.warmupOn": "Warm-up 已开启", "accounts.oauth.authorizationUrl": "授权 URL", @@ -209,6 +213,7 @@ "apiKeys.expiry.presets.sevenDays": "7 天", "apiKeys.expiry.presets.thirtyDays": "30 天", "apiKeys.form.allowedModels": "允许的 Model", + "apiKeys.form.allowedReasoningEfforts": "允许的推理强度", "apiKeys.form.applyToCodexModel": "应用到 codex /model", "apiKeys.form.assignedAccounts": "分配账户", "apiKeys.form.assignedModelSources": "分配 Model source", @@ -256,6 +261,10 @@ "apiKeys.modelSelect.selected": "已选择 {{count}} 个 Model", "apiKeys.modelSelect.selected_one": "已选择 {{count}} 个 Model", "apiKeys.modelSelect.selected_other": "已选择 {{count}} 个 Model", + "apiKeys.reasoningEfforts.all": "所有推理强度", + "apiKeys.reasoningEfforts.selected": "已选择 {{count}} 个推理强度", + "apiKeys.reasoningEfforts.selected_one": "已选择 {{count}} 个推理强度", + "apiKeys.reasoningEfforts.selected_other": "已选择 {{count}} 个推理强度", "apiKeys.overview.activeKeys": "活跃 key", "apiKeys.overview.apiKeys": "API Key", "apiKeys.overview.costByKey": "按 API Key 统计的累计费用", @@ -332,6 +341,8 @@ "apis.keyInfo.noUsageRecorded": "尚未记录用量", "apis.keyInfo.title": "Key 信息", "apis.list.createKey": "创建 API Key", + "apis.list.emptyDescription": "创建 API Key 以认证客户端。", + "apis.list.emptyTitle": "暂无 API Key", "apis.list.noMatches": "没有匹配的 API Key", "apis.list.searchPlaceholder": "搜索 API Key...", "apis.listItem.apiLimit": "API 限制", @@ -548,6 +559,7 @@ "common.serviceTier.default": "默认", "common.serviceTier.flex": "Flex", "common.serviceTier.priority": "优先", + "common.serviceTier.ultrafast": "Ultrafast", "common.states.active": "活跃", "common.states.disabled": "已禁用", "common.states.enabled": "已启用", @@ -597,6 +609,7 @@ "dashboard.accountList.ascending": "升序", "dashboard.accountList.descending": "降序", "dashboard.accountList.disableWarmupAria": "为 {{account}} 禁用 limit warm-up", + "dashboard.accountList.emptyAction": "添加账户", "dashboard.accountList.emptyDescription": "导入或认证账户即可开始。", "dashboard.accountList.emptyTitle": "尚未连接账户", "dashboard.accountList.enableWarmupAria": "为 {{account}} 启用 limit warm-up", @@ -618,6 +631,7 @@ "dashboard.accountList.sortedAria": "{{label}}，按{{direction}}排序", "dashboard.accountList.viewDetailsAria": "查看 {{account}} 详情", "dashboard.accounts.disableWarmupFor": "为 {{account}} 禁用 limit warm-up", + "dashboard.accounts.emptyAction": "添加账户", "dashboard.accounts.emptyDescription": "导入或认证账户即可开始。", "dashboard.accounts.emptyTitle": "尚未连接账户", "dashboard.accounts.enableWarmupFor": "为 {{account}} 启用 limit warm-up", @@ -660,6 +674,7 @@ "dashboard.quotaLabels.weekly": "每周", "dashboard.requestStatus.error": "错误", "dashboard.requestStatus.ok": "正常", + "dashboard.requestStatus.cancelled": "已取消", "dashboard.requestStatus.quota": "Quota", "dashboard.requestStatus.rate_limit": "Rate limit", "dashboard.requestDetails.clientIp": "客户端 IP", @@ -674,8 +689,16 @@ "dashboard.requestDetails.fullError": "完整错误", "dashboard.requestDetails.noErrorDetail": "未记录错误详情。", "dashboard.requestDetails.queue": "队列", + "dashboard.requestDetails.reasoningTokensIncluded": "推理 token（包含在输出 token 中）", "dashboard.requestDetails.requestId": "请求 ID", "dashboard.requestDetails.requestKind": "请求类型", + "dashboard.requestDetails.routeEndpoint": "代理端点", + "dashboard.requestDetails.routeFailClosedReason": "失败关闭原因", + "dashboard.requestDetails.routeFallback": "同池回退", + "dashboard.requestDetails.routeFallbackNotUsed": "未使用", + "dashboard.requestDetails.routeFallbackUsed": "已使用", + "dashboard.requestDetails.routeMode": "路由模式", + "dashboard.requestDetails.routePool": "代理池", "dashboard.requestDetails.title": "请求详情", "dashboard.requestDetails.userAgent": "User Agent", "dashboard.requests.columns.account": "账户", @@ -688,10 +711,19 @@ "dashboard.requests.columns.time": "时间", "dashboard.requests.columns.tokens": "token", "dashboard.requests.columns.transport": "传输", + "dashboard.requests.columns.tps": "TPS", + "dashboard.requests.columns.ttft": "TTFT", + "dashboard.requests.columnLayout.columns": "列（{{count}}）", + "dashboard.requests.columnLayout.restoreDefault": "恢复默认列布局", + "dashboard.requests.columnLayout.visibleColumns": "可见列", "dashboard.requests.downstreamTransport": "下游传输", - "dashboard.requests.emptyDescription": "没有符合当前筛选条件的 request log。", + "dashboard.requests.emptyDescription": "客户端开始使用代理后，请求将显示在此处。", + "dashboard.requests.emptyFilteredDescription": "没有符合当前筛选条件的 request log。", + "dashboard.requests.emptyFilteredTitle": "没有匹配的请求", "dashboard.requests.emptyTitle": "暂无请求", "dashboard.requests.requestedTier": "请求 {{tier}}", + "dashboard.requests.reasoningTokensShort": "推理 {{count}}", + "dashboard.requests.resizeColumn": "调整“{{column}}”列宽", "dashboard.requests.title": "请求日志", "dashboard.requests.unassigned": "未分配", "dashboard.requests.upstreamTransport": "上游 {{transport}}", @@ -838,6 +870,7 @@ "modelSources.editDialog.title": "编辑 Model source", "modelSources.empty": "无项目", "modelSources.capabilities.audioTranscriptions": "Audio transcriptions", + "modelSources.capabilities.embeddings": "Embeddings", "modelSources.capabilities.chatCompletions": "Chat completions", "modelSources.capabilities.reasoning": "Reasoning", "modelSources.capabilities.responses": "Responses", @@ -926,6 +959,8 @@ "quotaPlanner.toasts.warmupFailed": "Warm-up 请求失败", "quotaPlanner.toasts.warmupResult": "Warmup {{status}}：{{reason}}", "reports.charts.costByDay": "按天费用", + "reports.charts.emptyDescription": "所选范围内没有用量记录。", + "reports.charts.emptyTitle": "暂无数据", "reports.charts.input": "输入", "reports.charts.medianQueueWait": "中位队列等待", "reports.charts.medianTps": "中位 TPS", @@ -936,19 +971,24 @@ "reports.charts.tokensByDay": "按天 token", "reports.charts.tokensPerSecond": "每秒 token", "reports.dailyBreakdown.columns.accounts": "账户", + "reports.dailyBreakdown.columns.cancelled": "已取消", "reports.dailyBreakdown.columns.cost": "费用", "reports.dailyBreakdown.columns.day": "日期", + "reports.dailyBreakdown.columns.errors": "错误", "reports.dailyBreakdown.columns.inputTokens": "输入 token", "reports.dailyBreakdown.columns.outputTokens": "输出 token", + "reports.dailyBreakdown.columns.reasoningTokens": "已报告推理 token", "reports.dailyBreakdown.columns.reqs": "请求", "reports.dailyBreakdown.csv": "CSV", "reports.dailyBreakdown.csvColumns.activeAccounts": "活跃账户", "reports.dailyBreakdown.csvColumns.cachedTokens": "缓存 token", + "reports.dailyBreakdown.csvColumns.cancelled": "已取消", "reports.dailyBreakdown.csvColumns.costUsd": "费用 USD", "reports.dailyBreakdown.csvColumns.date": "日期", "reports.dailyBreakdown.csvColumns.errors": "错误", "reports.dailyBreakdown.csvColumns.inputTokens": "输入 token", "reports.dailyBreakdown.csvColumns.outputTokens": "输出 token", + "reports.dailyBreakdown.csvColumns.reasoningTokens": "已报告推理 token", "reports.dailyBreakdown.csvColumns.requests": "请求", "reports.dailyBreakdown.title": "每日明细", "reports.distribution.byModel": "按 Model 分布", @@ -956,6 +996,7 @@ "reports.distribution.missingUserAgent": "缺少 User-Agent", "reports.distribution.total": "总计", "reports.errors.accounts": "加载账户选项失败：{{error}}", + "reports.errors.apiKeys": "加载 API 密钥选项失败：{{error}}", "reports.errors.data": "加载报表数据失败：{{error}}", "reports.errors.options": "加载 Model 和 User Agent 选项失败：{{error}}", "reports.errors.partial": "部分报表数据无法加载。请重试。", @@ -967,11 +1008,14 @@ "reports.page.subtitle": "按日期范围查看使用历史", "reports.page.title": "费用报表", "reports.summary.avgCostPerDay": "平均 {{cost}}/天", + "reports.summary.cancelled": "已取消", "reports.summary.conversations": "活跃对话", + "reports.summary.errors": "错误", "reports.summary.requests": "请求", "reports.summary.requestsSub": "平均 {{requests}}/天 · {{accounts}} 个账户", "reports.summary.tokens": "Token", "reports.summary.tokensSub": "输入 {{input}} · 缓存 {{cache}} · 输出 {{output}}", + "reports.summary.reasoningSub": "已报告推理 {{reasoning}}（输出的一部分）· {{known}}/{{total}} 个请求", "reports.summary.totalCost": "总费用", "reports.dailyBreakdown.columns.conversations": "对话", "reports.dailyBreakdown.csvColumns.conversations": "对话", @@ -1071,10 +1115,11 @@ "settings.routing.strategy.guide.singleAccount": "将所有路由流量固定到一个选定的活跃账户。", "settings.routing.strategy.safetyNote": "任何策略都不能保证账户安全结果。对于低流量、合规使用，优先选择按容量加权或相对可用性，并启用粘性会话，让流量更平稳且保持在同一会话本地。", "settings.routing.stickyThreads.label": "粘性会话", - "settings.routing.stickyThreads.description": "将相关请求保持在同一账户上。", + "settings.routing.stickyThreads.description": "尽可能将同一会话的普通请求保持在同一账户上。这是软性偏好，并非保证。", + "settings.routing.stickyThreads.hardAffinityNote": "关闭此项不会禁用硬性 Codex 续接亲和：携带 Codex 续接状态（turn state、先前响应或已上传文件）的请求可能仍需要其原属账户，除非 codex-lb 能安全地迁移或重放。", "settings.routing.stickyThreads.ariaLabel": "启用粘性会话", "settings.routing.preferEarlier.label": "优先重置较早的账户", - "settings.routing.preferEarlier.description": "倾向于将流量分配给配额更早重置的账户。", + "settings.routing.preferEarlier.description": "当多个账户均符合条件时，优先选择所选配额窗口更早重置的账户。每周重置按天比较。适用于按容量加权、按用量加权和优先填满策略。", "settings.routing.preferEarlier.ariaLabel": "优先选择重置较早的账户", "settings.routing.preferEarlier.windowAria": "重置偏好窗口", "settings.routing.promptCacheTtl.label": "Prompt 缓存亲和 TTL", @@ -1118,11 +1163,14 @@ "settings.routing.accountCapacity.fairShareThresholdDescription": "当进行中的流达到账户池流容量的此百分比时，持有超过公平份额的 API 密钥将被限流，直到池不再拥塞。0 表示禁用。", "settings.routing.accountCapacity.save": "保存容量限制", "settings.routing.stickyThresholds.primaryLabel": "粘性主阈值", - "settings.routing.stickyThresholds.primaryDescription": "当主配额使用率高于此百分比时重新分配粘性会话。", + "settings.routing.stickyThresholds.primaryDescription": "当账户 5 小时（主）窗口的已用量超过此百分比时，将粘性会话迁移到其他账户。", "settings.routing.stickyThresholds.savePrimary": "保存主阈值", "settings.routing.stickyThresholds.secondaryLabel": "粘性次阈值", - "settings.routing.stickyThresholds.secondaryDescription": "当次级配额使用率高于此百分比时重新分配粘性会话。", + "settings.routing.stickyThresholds.secondaryDescription": "当账户次级窗口（每周；仅每月的套餐则为每月）的已用量超过此百分比时，将粘性会话迁移到其他账户。", "settings.routing.stickyThresholds.saveSecondary": "保存次阈值", + "settings.routing.stickyThresholds.usedRemainingHint": "已用 {{used}}% · 按配额计剩余 {{remaining}}%", + "settings.routing.quotaWindows.title": "主配额与次配额", + "settings.routing.quotaWindows.explainer": "主配额（primary）是较短的 5 小时用量窗口；次配额（secondary）是较长的窗口（每周；无每周窗口的套餐则为每月）。账户页面按剩余百分比显示各窗口，而下方粘性阈值按已用百分比计。路由还会将进行中的请求计为临时额外用量，因此阈值可能在账户页面显示相应剩余百分比之前先行触发。", "settings.routing.quotaWindows.weekly": "周配额", "settings.routing.quotaWindows.fiveHour": "5 小时配额", "settings.routing.workingDays.label": "周节奏工作日", @@ -1138,7 +1186,7 @@ "settings.routing.paceSmoothing.label": "节奏差距平均", "settings.routing.paceSmoothing.description": "用最近的配额样本平滑显示的周节奏差距。", "settings.routing.limitWarmup.label": "额度预热", - "settings.routing.limitWarmup.description": "为已选择的账户发送一次确认重置后的预热请求。", + "settings.routing.limitWarmup.description": "当已选择加入的账户的配额窗口被确认刚刚重置时，发送一条小型探测请求以验证账户可正常响应。探测是使用下方模型与提示词的真实请求，会消耗少量配额。", "settings.routing.limitWarmup.ariaLabel": "启用额度预热", "settings.routing.limitWarmup.modelAria": "预热模型", "settings.routing.limitWarmup.cooldownAria": "预热冷却时间", @@ -1242,6 +1290,25 @@ "settings.password.validation.required": "此项不能为空。", "settings.password.validation.minLength": "密码至少需要 8 个字符。", "settings.password.validation.maxByteLength": "密码必须在 UTF-8 编码下最多 72 字节。", + "settings.telemetry.title": "匿名遥测", + "settings.telemetry.description": "分享匿名使用统计，帮助改进 codex-lb。", + "settings.telemetry.toggleAria": "启用匿名遥测", + "settings.telemetry.envNotice": "遥测当前由 CODEX_LB_TELEMETRY_ENABLED 环境变量控制。如需在仪表盘中管理此设置，请取消设置该变量。", + "settings.telemetry.optOutNotice": "从仪表盘中禁用遥测时，系统会发送一次匿名的选择退出通知，以确保汇总计数准确。", + "settings.telemetry.collectedData.label": "收集的数据", + "settings.telemetry.collectedData.description": "查看此实例将发送的匿名数据的完整内容。", + "settings.telemetry.collectedData.view": "查看收集的数据", + "settings.telemetry.previewDialog.title": "收集的遥测数据", + "settings.telemetry.previewDialog.description": "此实例将发送的完整数据内容，不包含账号、提示词、API 密钥或 IP 地址。", + "settings.telemetry.consentDialog.title": "匿名遥测", + "settings.telemetry.consentDialog.description": "codex-lb 默认收集匿名使用遥测，以帮助指导开发方向。您可以随时在设置中更改此选择。", + "settings.telemetry.consentDialog.categories": "仅收集版本、部署形态和聚合使用统计 — 不含账号、提示词、API 密钥和 IP 地址。", + "settings.telemetry.consentDialog.payloadLabel": "此实例将发送的完整数据：", + "settings.telemetry.consentDialog.docsLink": "了解收集哪些数据以及原因", + "settings.telemetry.consentDialog.keepEnabled": "保持启用", + "settings.telemetry.consentDialog.disable": "禁用遥测", + "settings.telemetry.toasts.saved": "遥测偏好已保存", + "settings.telemetry.toasts.saveFailed": "保存遥测偏好失败", "settings.totp.title": "TOTP", "settings.totp.status.configured": "TOTP 已配置。", "settings.totp.status.notConfigured": "未配置 TOTP。", @@ -1382,5 +1449,9 @@ "upstreamProxy.toasts.poolUpdateFailed": "Pool 更新失败", "upstreamProxy.validation.hostRequired": "主机不能为空", "upstreamProxy.validation.nameRequired": "名称必填", - "upstreamProxy.validation.portInvalid": "请输入 1 到 65535 之间的端口" + "upstreamProxy.validation.portInvalid": "请输入 1 到 65535 之间的端口", + "modelSources.fields.reasoningEfforts": "支持的推理级别", + "modelSources.fields.reasoningEffortsDescription": "填写这个模型源接受的推理级别标识，并选择默认值。", + "modelSources.fields.reasoningEffortsPlaceholder": "none, low, provider-specific", + "modelSources.fields.defaultReasoningEffort": "默认推理级别" } diff --git a/frontend/src/test/mocks/factories.ts b/frontend/src/test/mocks/factories.ts index ac3df1d3cf..04d59e4582 100644 --- a/frontend/src/test/mocks/factories.ts +++ b/frontend/src/test/mocks/factories.ts @@ -49,8 +49,18 @@ import { RequestLogSchema, RequestLogsResponseSchema, } from "@/features/dashboard/schemas"; -import type { DashboardSettings, UpstreamProxyAdmin } from "@/features/settings/schemas"; -import { DashboardSettingsSchema, UpstreamProxyAdminSchema } from "@/features/settings/schemas"; +import type { + DashboardSettings, + TelemetryConsent, + TelemetrySnapshotEnvelope, + UpstreamProxyAdmin, +} from "@/features/settings/schemas"; +import { + DashboardSettingsSchema, + TelemetryConsentSchema, + TelemetrySnapshotEnvelopeSchema, + UpstreamProxyAdminSchema, +} from "@/features/settings/schemas"; import type { QuotaPlannerDecision, QuotaPlannerForecast, @@ -82,6 +92,7 @@ export type { RequestLogsResponse, RequestLogFilterOptions, DashboardSettings, + TelemetryConsent, UpstreamProxyAdmin, OauthStartResponse, OauthStatusResponse, @@ -171,6 +182,7 @@ export function createModelSource( supportsChatCompletions: true, supportsResponses: false, supportsAudioTranscriptions: false, + supportsEmbeddings: false, timeoutSeconds: null, maxConcurrency: null, createdAt: offsetIso(-30), @@ -533,6 +545,99 @@ export function createDashboardSettings( }); } +export function createTelemetrySnapshotEnvelope(): TelemetrySnapshotEnvelope { + return TelemetrySnapshotEnvelopeSchema.parse({ + instance_id: "00000000-0000-4000-8000-000000000000", + timestamp: "2026-08-06T00:00:00Z", + metrics: { + schema_version: 1, + consent: "undecided", + instance_id: "00000000-0000-4000-8000-000000000000", + version: "1.23.0", + python: "3.13", + os: "linux", + arch: "x86_64", + uptime_hours: 168, + deploy: { + method: "docker", + db_backend: "sqlite", + db_size_bucket: "<100MB", + replicas: 1, + reverse_proxy: true, + }, + accounts: { + pool_bucket: "2-5", + plan_mix: { plus: "2-5", pro: "0", team: "0", free: "0" }, + workspace_accounts: false, + routing_policy: "usage_weighted", + limit_warmup_enabled: false, + egress_proxy_used: false, + }, + usage_7d: { + requests: 1024, + success_rate: 0.99, + tokens_input: 1000000, + tokens_output: 50000, + tokens_cached_ratio: 0.8, + cost_usd_bucket: "<10", + request_kinds: { responses: 0.97, chat: 0.02, images: 0.01, unknown: 0.0 }, + transport_mix: { ws: 0.6, http_bridge: 0.4 }, + service_tier_mix: { default: 1.0, flex: 0.0, priority: 0.0 }, + clients: { "codex-cli": 0.9, other: 0.1 }, + clients_other_ratio: 0.1, + models: [ + { + name: "gpt-5.4-codex", + share: 1.0, + reasoning: { high: 0.5, medium: 0.5 }, + avg_output_tokens_bucket: "250-1k", + }, + ], + latency_ms_p50: 1200, + ttft_ms_p50: 800, + ttft_ms_p95: 3400, + rate_limit_429_ratio: 0.004, + top_upstream_errors: ["server_overloaded"], + }, + features: { + api_firewall: false, + quota_planner: false, + sticky_sessions: true, + conversation_archive: false, + automations: false, + fleet: false, + model_sources_count: 0, + api_keys_bucket: "2-5", + prometheus: false, + otel: false, + dashboard_auth: true, + reset_credits: true, + image_api_used: false, + }, + }, + }); +} + +export function createTelemetryConsent( + overrides: Partial = {}, +): TelemetryConsent { + const base = { + state: "enabled", + source: "persisted", + active: true, + ...overrides, + }; + // Mirror the backend: the base GET attaches a preview envelope only for + // the undecided/default (consent dialog) case; explicit overrides win. + const preview = + "preview" in overrides + ? overrides.preview + : base.state === "undecided" && base.source === "default" + ? createTelemetrySnapshotEnvelope() + : null; + return TelemetryConsentSchema.parse({ ...base, preview }); +} + export function createQuotaPlannerSettings( overrides: Partial = {}, ): QuotaPlannerSettings { diff --git a/frontend/src/test/mocks/handler-coverage.test.ts b/frontend/src/test/mocks/handler-coverage.test.ts index 4e5bd3702e..1efff9b8aa 100644 --- a/frontend/src/test/mocks/handler-coverage.test.ts +++ b/frontend/src/test/mocks/handler-coverage.test.ts @@ -70,6 +70,8 @@ const EXPECTED_ENDPOINTS = [ // settings "GET /api/settings", "PUT /api/settings", + "GET /api/settings/telemetry", + "PUT /api/settings/telemetry", "GET /api/settings/upstream-proxy", "POST /api/settings/upstream-proxy/endpoints", "POST /api/settings/upstream-proxy/endpoints/:endpointId/test", diff --git a/frontend/src/test/mocks/handlers.ts b/frontend/src/test/mocks/handlers.ts index 9077775ca4..25884bc0a3 100644 --- a/frontend/src/test/mocks/handlers.ts +++ b/frontend/src/test/mocks/handlers.ts @@ -37,6 +37,8 @@ import { createQuotaPlannerSettings, createQuotaPlannerWarmupActionResponse, createRequestLogFilterOptions, + createTelemetryConsent, + createTelemetrySnapshotEnvelope, createUpstreamProxyAdmin, createRequestLogsResponse, type DashboardAuthSession, @@ -46,11 +48,12 @@ import { type QuotaPlannerForecast, type QuotaPlannerSettings, type RequestLogEntry, + type TelemetryConsent, type UpstreamProxyAdmin, } from "@/test/mocks/factories"; const MODEL_OPTION_DELIMITER = ":::"; -const STATUS_ORDER = ["ok", "rate_limit", "quota", "error"] as const; +const STATUS_ORDER = ["ok", "cancelled", "rate_limit", "quota", "error"] as const; // ── Zod schemas for mock request bodies ── @@ -95,6 +98,10 @@ const AccountAliasPayloadSchema = z.object({ alias: z.string().max(255).nullable(), }); +const TelemetryConsentPayloadSchema = z.object({ + enabled: z.boolean(), +}); + const AccountRoutingPolicyPayloadSchema = z.object({ routingPolicy: z.enum(["normal", "burn_first", "preserve"]), }); @@ -155,6 +162,7 @@ const ModelSourceCreatePayloadSchema = z.looseObject({ supportsChatCompletions: z.boolean().optional(), supportsResponses: z.boolean().optional(), supportsAudioTranscriptions: z.boolean().optional(), + supportsEmbeddings: z.boolean().optional(), models: z .array( z.looseObject({ @@ -172,6 +180,7 @@ const ModelSourceCreatePayloadSchema = z.looseObject({ const ModelSourceUpdatePayloadSchema = z.looseObject({ isEnabled: z.boolean().optional(), + supportsEmbeddings: z.boolean().optional(), }); const QuotaPlannerSettingsPayloadSchema = z.looseObject({ @@ -248,6 +257,7 @@ type MockState = { conversationDetails: ConversationDetails[]; authSession: DashboardAuthSession; settings: DashboardSettings; + telemetryConsent: TelemetryConsent; quotaPlannerSettings: QuotaPlannerSettings; quotaPlannerDecisions: QuotaPlannerDecision[]; upstreamProxyAdmin: UpstreamProxyAdmin; @@ -340,6 +350,7 @@ function createInitialState(): MockState { ], authSession: createDashboardAuthSession(), settings: createDashboardSettings(), + telemetryConsent: createTelemetryConsent(), quotaPlannerSettings: createQuotaPlannerSettings(), quotaPlannerDecisions: [createQuotaPlannerDecision()], upstreamProxyAdmin: createUpstreamProxyAdmin(), @@ -1198,7 +1209,31 @@ export const handlers = [ return HttpResponse.json(state.settings); }), + http.get("/api/settings/telemetry", ({ request }) => { + // include_preview=true is the on-demand path: the envelope is attached + // regardless of consent state. + if (new URL(request.url).searchParams.get("include_preview") === "true") { + return HttpResponse.json({ + ...state.telemetryConsent, + preview: createTelemetrySnapshotEnvelope(), + }); + } + return HttpResponse.json(state.telemetryConsent); + }), + http.put("/api/settings/telemetry", async ({ request }) => { + const payload = await parseJsonBody(request, TelemetryConsentPayloadSchema); + if (!payload) { + return HttpResponse.json(state.telemetryConsent); + } + state.telemetryConsent = createTelemetryConsent({ + state: payload.enabled ? "enabled" : "disabled", + source: "persisted", + active: payload.enabled, + preview: null, + }); + return HttpResponse.json(state.telemetryConsent); + }), http.get("/api/settings/upstream-proxy", () => { return HttpResponse.json(state.upstreamProxyAdmin); @@ -2088,6 +2123,7 @@ export const handlers = [ supportsChatCompletions: payload?.supportsChatCompletions ?? true, supportsResponses: payload?.supportsResponses ?? false, supportsAudioTranscriptions: payload?.supportsAudioTranscriptions ?? false, + supportsEmbeddings: payload?.supportsEmbeddings ?? false, models: (payload?.models ?? [{ model: `model-${sequence}` }]).map( (model, index) => ({ id: index + 1, @@ -2127,6 +2163,9 @@ export const handlers = [ const updated = createModelSource({ ...existing, ...(payload?.isEnabled !== undefined ? { isEnabled: payload.isEnabled } : {}), + ...(payload?.supportsEmbeddings !== undefined + ? { supportsEmbeddings: payload.supportsEmbeddings } + : {}), updatedAt: new Date().toISOString(), }); state.modelSources = state.modelSources.map((source) => diff --git a/frontend/src/utils/constants.ts b/frontend/src/utils/constants.ts index 619fde66a8..40e5b393bc 100644 --- a/frontend/src/utils/constants.ts +++ b/frontend/src/utils/constants.ts @@ -90,6 +90,7 @@ export const MESSAGE_TONE_META = { export const REQUEST_STATUS_LABELS: Record = { ok: "OK", + cancelled: "Cancelled", rate_limit: "Rate limit", quota: "Quota", error: "Error", diff --git a/mkdocs.yml b/mkdocs.yml index 01a9453f2e..7f77b01a23 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -59,7 +59,9 @@ nav: - Client Setup: client-setup.md - Live Voice: live-voice.md - Conversations: conversations.md + - Usage Reporting: usage-reporting.md - Configuration: configuration.md + - Anonymous Telemetry: telemetry.md - Authentication: authentication.md - API Keys: api-keys.md - Routing: routing.md diff --git a/openspec/changes/add-api-key-allowed-reasoning-efforts/design.md b/openspec/changes/add-api-key-allowed-reasoning-efforts/design.md new file mode 100644 index 0000000000..be7944a8db --- /dev/null +++ b/openspec/changes/add-api-key-allowed-reasoning-efforts/design.md @@ -0,0 +1,152 @@ +## Context + +Reasoning effort is a client-visible choice with two different operator +controls today: no control, or `enforcedReasoningEffort`, which overwrites the +choice. The latter is useful for a fixed low-cost key but is too coarse for a +corporate key where users may choose `minimal`, `low`, `medium`, `high`, or +`xhigh` and must not select `max` or `ultra`. + +The proxy already normalizes model aliases into `reasoning.effort`, rewrites +unsupported and client-plane wire aliases only immediately before forwarding, +and uses one request-policy module across Responses, compact, and WebSocket +paths. Chat Completions converts to that Responses form before selection; the +source-routed chat path additionally retains a chat-shaped outbound payload. + +## Goals / Non-Goals + +**Goals:** + +- Let an operator persist a per-key, explicit set of client-selectable efforts. +- Preserve existing keys and requests when no policy is configured. +- Reject forbidden work before it consumes quota or reaches an upstream. +- Give dashboard users a clear mutually-exclusive choice between forcing one + effort and allowing a set. +- Keep all proxy route families consistent, including aliases and source chat. + +**Non-Goals:** + +- Infer or change an upstream model's default effort when the client omits an + effort. +- Rank efforts or introduce a generic numeric "maximum effort" setting. +- Change the existing `ultra` to `max` wire alias, model catalog, pricing, or + Fast Mode policy. +- Add global settings, per-model policy, roles, or new dashboard navigation. + +## Decisions + +### Persist an optional explicit allowlist, not a maximum + +Store `allowed_reasoning_efforts` as nullable JSON text alongside the existing +nullable `allowed_models` field. The dashboard API exposes camelCase +`allowedReasoningEfforts`. `null` means no restriction; a supplied list must be +non-empty and contain only canonical client-plane efforts (`minimal`, `low`, +`medium`, `high`, `xhigh`, `max`, `ultra`). Service normalization trims, +lowercases, de-duplicates, and orders values by the catalog's canonical order. + +An explicit list avoids assuming that future effort names are linearly ordered +or that every model supports the same scale. It also expresses the requested +`minimal` through `xhigh` policy exactly. + +### Keep fixed and selectable policies mutually exclusive + +An API key may have either `enforcedReasoningEffort` or +`allowedReasoningEfforts`, never both. The service validates the effective +state on create and patch, so a partial PATCH cannot accidentally preserve the +other policy. The dashboard clears the opposite control before submission. + +Combining them is not useful: enforcement always replaces the client request, +making an allowlist invisible. Rejecting the ambiguous state is clearer than +inventing precedence or silently accepting dead configuration. + +### Validate the client-plane effort before wire normalization + +The shared policy derives the client-plane effort before it mutates the +request: an accepted Cursor alias such as `gpt-5.6-sol-xhigh` is checked as +`xhigh`; otherwise it checks explicit `reasoning.effort`. Only afterwards do +current compatibility transforms run, including `minimal` fallback and +`ultra` to `max` wire aliasing. `xhigh` is likewise lowered to upstream +`high` by accepted model aliases. Authorization remains client-plane and +exact: `xhigh` and `high`, as well as `ultra` and `max`, are separate policy +choices even when their downstream wire forms coincide. + +The policy therefore describes what the client chose rather than an internal +wire representation. A request with no explicit effort remains valid and uses +the existing upstream default; inventing a default would be a separate +behavioural feature and would risk breaking clients. + +### Fail visibly and before side effects + +A forbidden effort raises a typed proxy permission exception with HTTP `403`, +OpenAI error type `permission_error`, code `reasoning_effort_not_allowed`, and +parameter `reasoning.effort`. It records only low-cardinality key id and +effort in the diagnostic log. The check runs before admission and API-key +quota reservation, model-source dispatch, account selection, or upstream I/O. + +This is deliberately a rejection, not a downgrade: silently changing a +developer's requested reasoning depth hides an operator policy and makes +unexpected output quality difficult to diagnose. + +### Use the converted Responses payload as the single enforcement point + +Responses, compact, and WebSocket paths already call the shared enforcement +function. Chat Completions converts to `ResponsesRequest` before selection, +so the same call rejects disallowed effort before either account or source +routing. Source-routed chat also resolves `ultra` to the `max` wire value for +every accepted chat reasoning spelling (`reasoning_effort`, `reasoningEffort`, +`reasoning.effort`, and `thinking`) after the request passes the allowlist. +When a client supplies conflicting spellings, the outbound values are aligned +to the already-authorized client-plane effort so a source cannot select a +disallowed value from an ignored alias. Other accepted client-plane values, +including `minimal` and model-alias-derived `xhigh`, remain unchanged for an +external source that may support them directly. + +Source-routed Responses traffic likewise normalizes all accepted reasoning +aliases before egress. The canonical `reasoning.effort` wins when aliases +conflict, and the aliases are removed so an external source cannot select a +different, unauthorized value. + +The origin route records that policy has already been applied when it forwards +the signed request to an owner instance. The owner still authenticates the +internal request and validates model access, but does not re-authorize or +re-normalize the reasoning effort. This keeps the policy idempotent across the +HTTP bridge and prevents a client-plane alias such as `xhigh` from being +mistaken for its wire value `high` on the second pass. + +The request model keeps the original client-plane effort in a private field +while enforcement mutates the payload. This is intentionally not serialized: +the signed bridge request is trusted only after the origin has completed policy +enforcement, and the owner receives an explicit internal call-site marker. + +## Risks / Trade-offs + +- **Client omits an effort and an upstream default changes**: the request + remains compatible, but the allowlist cannot cap an unexpressed upstream + default. A future "required/default reasoning effort" policy would need its + own explicit contract rather than changing this feature's meaning. +- **A model alias hides an effort**: the policy derives the client-plane + effort from accepted model aliases before they are normalized for upstream, + so an `xhigh` suffix cannot bypass the policy. Source-routed chat traffic + applies the same `ultra` to `max` wire conversion as Responses traffic only + after the exact client-plane policy check. +- **Malformed manually-edited stored JSON**: service deserialization treats it + as an empty restrictive policy, so explicit effort requests fail rather than + silently becoming unrestricted. Normal dashboard/API writes cannot create + that state. +- **Operators switch policies through a partial PATCH**: effective-state + validation rejects a key that would hold both settings; the dashboard sends + the clearing value in the same request. +- **Mixed-version replicas during a rolling upgrade**: this feature does not + add protocol machinery for mixed application versions. Operators running a + multi-replica deployment must complete the application rollout before + creating or changing reasoning allowlists. + +## Migration Plan + +The migration adds one nullable allowlist column. Existing rows retain current +unrestricted behavior because their allowlist remains `NULL`. The database +rejects rows that combine a fixed effort with an allowlist. Rolling back drops +the constraint and column without changing key identity or other stored data. + +## Open Questions + +None. diff --git a/openspec/changes/add-api-key-allowed-reasoning-efforts/proposal.md b/openspec/changes/add-api-key-allowed-reasoning-efforts/proposal.md new file mode 100644 index 0000000000..222e3e7623 --- /dev/null +++ b/openspec/changes/add-api-key-allowed-reasoning-efforts/proposal.md @@ -0,0 +1,45 @@ +## Why + +`enforcedReasoningEffort` lets an operator replace every client choice with +one fixed effort, but it cannot express the common policy "let this key choose +among normal efforts, but do not permit max or ultra". Operators then have to +either force a single value or leave the key fully unrestricted. + +## What Changes + +- Add the nullable API-key field `allowedReasoningEfforts` to the dashboard + API, persistence model, and create/edit dialogs. +- Treat a non-empty list as an explicit client-plane effort allowlist. `null` + keeps the existing unrestricted behaviour; an empty list is rejected. +- Make the allowlist and `enforcedReasoningEffort` mutually exclusive, so the + effective policy is unambiguous. +- Reject a disallowed explicit effort before quota reservation or upstream + forwarding with an OpenAI-compatible `403` `reasoning_effort_not_allowed` + error. +- Apply the shared policy after model-alias normalization and before the + existing unsupported-effort and wire-alias rewrites on Responses, compact, + WebSocket, and Chat Completions paths. + +## Capabilities + +### New Capabilities + +- None. + +### Modified Capabilities + +- `api-keys`: API keys can restrict client-selected reasoning efforts without + forcing one effort. +- `responses-api-compat`: all Responses-compatible routes enforce that + per-key restriction before dispatch. +- `chat-completions-compat`: chat requests, including source-routed traffic, + use the same restriction after conversion to Responses semantics. + +## Impact + +- Database: one nullable `TEXT` column and a mutual-exclusion constraint on + `api_keys`; existing rows remain unrestricted with no policy backfill. +- Backend: API-key schemas, service, repository, cache-facing data shape, and + shared proxy policy. +- Dashboard: API-key create/edit form and request schemas only; no new route, + setting, navigation item, dependency, or README section. diff --git a/openspec/changes/add-api-key-allowed-reasoning-efforts/specs/api-keys/spec.md b/openspec/changes/add-api-key-allowed-reasoning-efforts/specs/api-keys/spec.md new file mode 100644 index 0000000000..16d258a683 --- /dev/null +++ b/openspec/changes/add-api-key-allowed-reasoning-efforts/specs/api-keys/spec.md @@ -0,0 +1,84 @@ +## ADDED Requirements + +### Requirement: API keys can restrict client-selected reasoning efforts + +The dashboard API-key create, update, list, and response surfaces SHALL expose +an optional `allowedReasoningEfforts` list. When absent or `null`, the API key +MUST retain unrestricted reasoning-effort behavior. When present, the list +MUST be non-empty and consist only of the supported client-plane efforts +`minimal`, `low`, `medium`, `high`, `xhigh`, `max`, and `ultra`. The service +MUST trim, case-normalize, de-duplicate, and return entries in canonical +catalog order. + +`allowedReasoningEfforts` MUST be mutually exclusive with +`enforcedReasoningEffort`. Create and PATCH requests MUST validate the +effective persisted state, including an unchanged counterpart field. Existing +API keys whose persisted allowlist is null MUST remain unrestricted. +The persistence layer MUST reject a row that contains both an allowlist and a +fixed reasoning effort. +If legacy or manually edited storage contains a malformed non-null allowlist, +the service MUST remain fail-closed for explicit efforts and the dashboard MUST +NOT clear that sentinel during an unrelated edit. A concurrent update that +loses the mutual-exclusion constraint race MUST return the normal dashboard +validation error instead of an internal server error. + +#### Scenario: Create an effort-selectable key + +- **WHEN** an administrator creates an API key with + `allowedReasoningEfforts: ["XHIGH", "low", "high", "low"]` +- **THEN** the response returns `allowedReasoningEfforts` as + `["low", "high", "xhigh"]` +- **AND** `enforcedReasoningEffort` is null + +#### Scenario: Reject an empty allowlist + +- **WHEN** an administrator creates or updates an API key with + `allowedReasoningEfforts: []` +- **THEN** the dashboard API returns 400 +- **AND** the API key is not changed + +#### Scenario: Reject conflicting reasoning policies on update + +- **GIVEN** an API key has `enforcedReasoningEffort: "low"` +- **WHEN** an administrator updates only `allowedReasoningEfforts` to + `["low", "medium"]` +- **THEN** the dashboard API returns 400 +- **AND** the existing fixed effort remains unchanged + +#### Scenario: Existing key remains unrestricted + +- **GIVEN** an API key created before `allowedReasoningEfforts` existed +- **WHEN** it is read or used without that field configured +- **THEN** its response contains `allowedReasoningEfforts: null` +- **AND** no reasoning-effort allowlist is applied + +#### Scenario: Unrelated edit preserves a malformed fail-closed policy + +- **GIVEN** an API key exposes an empty allowlist sentinel for malformed stored + policy data +- **WHEN** an administrator changes only its name +- **THEN** the dashboard update omits `allowedReasoningEfforts` +- **AND** the malformed persisted policy is not replaced with null + +#### Scenario: Concurrent policy conflict returns a validation error + +- **GIVEN** concurrent updates try to set a fixed effort and an allowlist on + the same unrestricted key +- **WHEN** the database mutual-exclusion constraint rejects the losing update +- **THEN** the dashboard API returns its normal invalid API-key payload error + +### Requirement: Dashboard manages selectable reasoning efforts + +The API-key create and edit dialogs SHALL present the supported reasoning +efforts as an accessible multi-select when no fixed effort is selected. The UI +MUST represent no selected values as `null`, not an empty allowlist. When an +administrator selects a fixed effort, the UI MUST clear and disable the +allowlist; when it selects one or more allowlist values, it MUST clear the +fixed-effort selection. + +#### Scenario: Configure all normal efforts without max or ultra + +- **WHEN** an administrator selects `minimal`, `low`, `medium`, `high`, and + `xhigh` in the API-key dialog +- **THEN** the saved key returns exactly those five allowed efforts +- **AND** the dialog does not show `max` or `ultra` as selected diff --git a/openspec/changes/add-api-key-allowed-reasoning-efforts/specs/chat-completions-compat/spec.md b/openspec/changes/add-api-key-allowed-reasoning-efforts/specs/chat-completions-compat/spec.md new file mode 100644 index 0000000000..47ef67680f --- /dev/null +++ b/openspec/changes/add-api-key-allowed-reasoning-efforts/specs/chat-completions-compat/spec.md @@ -0,0 +1,109 @@ +## ADDED Requirements + +### Requirement: Chat Completions shares API-key reasoning allowlist enforcement + +Before Chat Completions traffic selects a subscription account or an external +model source, the service MUST convert reasoning controls to the internal +Responses representation and apply the authenticated API key's +`allowedReasoningEfforts` policy. A rejected effort MUST produce the same +OpenAI-compatible `403` `reasoning_effort_not_allowed` result as a native +Responses request and MUST NOT call the external source. +The `thinking` string alias MUST recognize every selectable effort, including +`minimal`, before allowlist evaluation. +A snake-case `reasoning_effort` MUST still participate in authorization when a +separate `reasoning` object contains only metadata such as `summary`. +An inactive `thinking` control MUST NOT mask a separate enabled reasoning +alias during authorization. +Reasoning metadata inside `thinking` MUST be merged with enabled controls before +allowlist evaluation and MUST NOT hide their implicit `medium` effort. + +After a source-routed Chat Completions request passes the policy, any accepted +`ultra` value MUST use the upstream wire value `max` regardless of whether the +client expressed it through `reasoning_effort`, `reasoningEffort`, +`reasoning.effort`, or `thinking`. If several reasoning spellings conflict, +every retained outbound spelling MUST be aligned to the authorized client-plane +effort. Other allowed client-plane efforts MUST remain unchanged for the +external source. A sole `enable_thinking: true` control authorized as `medium` +MUST remain enabled on source egress. +When source selection replaces an effort-bearing model alias with a canonical +source model slug and the client supplied no separate reasoning control, the +service MUST materialize that authorized effort as `reasoning_effort`. This +applies whether the alias came from the client model or the API key's enforced +model. + +#### Scenario: Source-routed chat request is rejected before forwarding + +- **GIVEN** a source-routed chat model and an API key with + `allowedReasoningEfforts: ["low", "medium", "high"]` +- **WHEN** a Chat Completions client supplies `reasoning_effort: "ultra"` +- **THEN** the service returns `403` with code `reasoning_effort_not_allowed` +- **AND** the source receives no request + +#### Scenario: Minimal thinking alias is evaluated before forwarding + +- **GIVEN** a source-routed chat model and an API key that allows only `low` +- **WHEN** a Chat Completions client supplies `thinking: "minimal"` +- **THEN** the service returns `403` with code `reasoning_effort_not_allowed` +- **AND** the source receives no request + +#### Scenario: Reasoning metadata does not mask snake-case effort + +- **GIVEN** a source-routed chat model and an API key that allows only `low` +- **WHEN** a Chat Completions client supplies `reasoning_effort: "max"` and + `reasoning: {"summary": "auto"}` +- **THEN** the service returns `403` with code `reasoning_effort_not_allowed` +- **AND** the source receives no request + +#### Scenario: Disabled thinking does not mask an enabled alias + +- **GIVEN** a source-routed chat model and an API key that allows only `low` +- **WHEN** a Chat Completions client supplies `thinking: false` and + `enable_thinking: true` +- **THEN** the service evaluates the enabled alias as `medium` +- **AND** returns `403` with code `reasoning_effort_not_allowed` +- **AND** the source receives no request + +#### Scenario: Thinking metadata does not mask its enabled state + +- **GIVEN** a source-routed chat model and an API key that allows only `low` +- **WHEN** a Chat Completions client supplies + `thinking: {"summary": "auto", "enabled": true}` +- **THEN** the service evaluates the enabled control as `medium` +- **AND** returns `403` with code `reasoning_effort_not_allowed` +- **AND** the source receives no request + +#### Scenario: Source-routed chat aliases use the ultra wire value + +- **GIVEN** a source-routed chat model and an API key that allows `ultra` +- **WHEN** a Chat Completions client supplies `thinking: "ultra"` +- **THEN** the source receives `thinking: "max"` + +#### Scenario: Source-routed chat preserves an allowed client-plane effort + +- **GIVEN** a source-routed chat model and an API key that allows `minimal` +- **WHEN** a Chat Completions client supplies `reasoning_effort: "minimal"` +- **THEN** the source receives `reasoning_effort: "minimal"` + +#### Scenario: Source-routed chat preserves an authorized thinking toggle + +- **GIVEN** a source-routed chat model and an API key that allows `medium` +- **WHEN** a Chat Completions client supplies only `enable_thinking: true` +- **THEN** the source receives `enable_thinking: true` + +#### Scenario: Canonical source retains effort from a model alias + +- **GIVEN** a source registered for `gpt-5.6-sol` and an API key that allows + `xhigh` +- **WHEN** a Chat Completions client requests `gpt-5.6-sol-xhigh` without a + separate reasoning control +- **THEN** the source receives model `gpt-5.6-sol` +- **AND** it receives `reasoning_effort: "xhigh"` + +#### Scenario: Canonical source retains effort from an enforced model alias + +- **GIVEN** a source registered for `gpt-5.6-sol` and an API key that enforces + `gpt-5.6-sol-xhigh` and allows `xhigh` +- **WHEN** a Chat Completions client requests canonical `gpt-5.6-sol` without a + separate reasoning control +- **THEN** the source receives model `gpt-5.6-sol` +- **AND** it receives `reasoning_effort: "xhigh"` diff --git a/openspec/changes/add-api-key-allowed-reasoning-efforts/specs/responses-api-compat/spec.md b/openspec/changes/add-api-key-allowed-reasoning-efforts/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..d0c9f5bb45 --- /dev/null +++ b/openspec/changes/add-api-key-allowed-reasoning-efforts/specs/responses-api-compat/spec.md @@ -0,0 +1,113 @@ +## ADDED Requirements + +### Requirement: API-key reasoning allowlists reject disallowed explicit efforts + +When an authenticated API key has a non-null +`allowedReasoningEfforts` policy, the proxy MUST derive the client-selected +effort from an explicit `reasoning.effort` or a supported model alias before +performing wire-level normalization. Policy values remain exact client-plane +choices: `xhigh` does not authorize `high`, and `ultra` does not authorize +`max`, even where their downstream wire forms coincide. If the client-selected +effort is not in the policy, the +proxy MUST reject the request before quota reservation, account or source +selection, or upstream dispatch. The rejection MUST use HTTP 403, OpenAI error type `permission_error`, code +`reasoning_effort_not_allowed`, and parameter `reasoning.effort`. + +The policy MUST apply to Responses, compact Responses, and WebSocket Responses +requests. Its WebSocket error event MUST preserve the same error code and +parameter. It MUST evaluate the client-plane effort before existing +unsupported-effort fallback and `ultra` to `max` upstream-wire aliasing. A +request that omits an effort MUST retain current default behavior. +Applying the policy more than once to the same request, including across a +signed internal HTTP bridge hop, MUST be idempotent and MUST NOT re-authorize +an already-normalized wire value as though it were the original client choice. +Before source-routed Responses traffic is forwarded, accepted reasoning +aliases MUST be aligned with the authorized canonical `reasoning.effort` or +removed so a conflicting alias cannot select a disallowed effort upstream. +Blank alias strings MUST be treated as absent and MUST NOT mask a later +effort-bearing alias during authorization. +Disabled aliases MUST likewise be treated as inactive rather than masking a +separate enabled reasoning alias. +Provider reasoning metadata MUST be merged with enabled controls before effort +authorization instead of masking their implicit `medium` effort. +When no reasoning policy is active, source egress MUST retain existing +provider-shaped reasoning controls and their source-specific fields. +An allowlist MUST also preserve provider-shaped controls that select no effort; +their unrelated fields do not participate in effort authorization. + +#### Scenario: Reject max before upstream dispatch + +- **GIVEN** an API key with + `allowedReasoningEfforts: ["minimal", "low", "medium", "high", "xhigh"]` +- **WHEN** a Responses request explicitly supplies `reasoning.effort: "max"` +- **THEN** the proxy returns `403` with code `reasoning_effort_not_allowed` +- **AND** no API-key quota reservation or upstream request is created + +#### Scenario: Alias effort is evaluated as the client-selected value + +- **GIVEN** an API key with `allowedReasoningEfforts: ["low", "medium"]` +- **WHEN** a client sends the model alias `gpt-5.6-sol-xhigh` +- **THEN** the proxy rejects the request with code `reasoning_effort_not_allowed` +- **AND** does not forward a request upstream + +#### Scenario: Omitted effort remains compatible + +- **GIVEN** an API key with `allowedReasoningEfforts: ["low", "medium"]` +- **WHEN** a Responses request omits `reasoning.effort` and uses no effort alias +- **THEN** the proxy does not add or replace a reasoning effort +- **AND** the request continues through the existing route + +#### Scenario: Effort-less provider controls remain compatible + +- **GIVEN** a source-routed model and an API key with + `allowedReasoningEfforts: ["low"]` +- **WHEN** a Responses request supplies + `thinking: {"type": "adaptive", "budget_tokens": 2048}` without an effort +- **THEN** the source receives the original `thinking` object + +#### Scenario: Source-routed conflicting alias cannot override policy + +- **GIVEN** a source-routed model and an API key with + `allowedReasoningEfforts: ["low"]` +- **WHEN** a Responses request supplies `reasoning.effort: "low"` and + `thinking: "max"` +- **THEN** the source receives the canonical `reasoning.effort: "low"` +- **AND** it does not receive the conflicting `thinking` alias + +#### Scenario: Blank alias cannot hide a disallowed effort + +- **GIVEN** a source-routed model and an API key with + `allowedReasoningEfforts: ["low"]` +- **WHEN** a Responses request supplies `reasoningEffort: " "` and + `thinking: "max"` +- **THEN** the service returns `403` with code `reasoning_effort_not_allowed` +- **AND** the source receives no request + +#### Scenario: Disabled alias cannot hide an enabled effort + +- **GIVEN** a source-routed model and an API key with + `allowedReasoningEfforts: ["low"]` +- **WHEN** a Responses request supplies `thinking: "disabled"` and + `enable_thinking: true` +- **THEN** the service evaluates the enabled alias as `medium` +- **AND** returns `403` with code `reasoning_effort_not_allowed` +- **AND** the source receives no request + +#### Scenario: Provider metadata cannot hide an enabled effort + +- **GIVEN** a source-routed model and an API key with + `allowedReasoningEfforts: ["low"]` +- **WHEN** a Responses request supplies + `thinking: {"summary": "auto", "enabled": true}` +- **THEN** the service evaluates the enabled control as `medium` +- **AND** returns `403` with code `reasoning_effort_not_allowed` +- **AND** the source receives no request + +#### Scenario: Effort-less provider control survives beside an allowed effort + +- **GIVEN** a source-routed model and an API key with + `allowedReasoningEfforts: ["low"]` +- **WHEN** a Responses request supplies `reasoning.effort: "low"` and + `thinking: {"type": "adaptive", "budget_tokens": 2048}` +- **THEN** the source receives both the authorized canonical effort and the + original `thinking` object diff --git a/openspec/changes/add-api-key-allowed-reasoning-efforts/tasks.md b/openspec/changes/add-api-key-allowed-reasoning-efforts/tasks.md new file mode 100644 index 0000000000..4b69754400 --- /dev/null +++ b/openspec/changes/add-api-key-allowed-reasoning-efforts/tasks.md @@ -0,0 +1,21 @@ +## 1. Contract and persistence + +- [x] 1.1 Add nullable `allowed_reasoning_efforts` persistence and a reversible migration from the current Alembic head. +- [x] 1.2 Extend API-key request/response schemas, service data, repository updates, and cache-facing mapping with normalized list semantics. +- [x] 1.3 Validate non-empty supported values and mutual exclusion with `enforced_reasoning_effort` on create and effective PATCH state. + +## 2. Proxy policy + +- [x] 2.1 Add a typed OpenAI-compatible permission error and enforce the allowlist in the shared Responses policy before wire normalization, including accepted aliases. +- [x] 2.2 Prove the shared policy covers Responses, compact, WebSocket, Chat Completions, aliases, and source-routed chat without reservation or upstream dispatch for rejected requests. + +## 3. Dashboard + +- [x] 3.1 Add a localized accessible multi-select to API-key create/edit dialogs, preserving unrestricted `null` behavior and clearing the conflicting fixed-effort control. +- [x] 3.2 Update frontend API schemas and tests for list serialization, validation, and existing-key compatibility. + +## 4. Verification + +- [x] 4.1 Add focused backend unit and integration tests, including migration upgrade/downgrade and dashboard API effective-state validation. +- [x] 4.2 Run focused frontend tests/build, relevant backend checks, migration graph checks, and OpenSpec validation. +- [x] 4.3 Capture before/after dashboard screenshots and include them with the linked issue and PR test plan. diff --git a/openspec/changes/isolate-request-and-refresh-db-sessions/.openspec.yaml b/openspec/changes/add-daybreak-blue-client-profile/.openspec.yaml similarity index 50% rename from openspec/changes/isolate-request-and-refresh-db-sessions/.openspec.yaml rename to openspec/changes/add-daybreak-blue-client-profile/.openspec.yaml index 44f55ffeea..b6b2d1f67c 100644 --- a/openspec/changes/isolate-request-and-refresh-db-sessions/.openspec.yaml +++ b/openspec/changes/add-daybreak-blue-client-profile/.openspec.yaml @@ -1,2 +1,2 @@ schema: spec-driven -created: 2026-08-23 +created: 2026-08-13 diff --git a/openspec/changes/add-daybreak-blue-client-profile/design.md b/openspec/changes/add-daybreak-blue-client-profile/design.md new file mode 100644 index 0000000000..3fa3b0f7b5 --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/design.md @@ -0,0 +1,88 @@ +## Context + +The direct Responses WebSocket ingress already accepts one authenticated `trusted_cyber` carrier and applies the existing `security_work_authorized` selector constraint before opening an upstream connection. The standard Codex client example defines only the ordinary `codex-lb` provider, so Codex Desktop/CLI orchestration has no explicit opt-in configuration that sends the carrier. Provider and profile selection are machine-local Codex settings; current Codex versions ignore them in project-local `.codex/config.toml` files. Current Codex clients treat `supports_websockets = true` as a preference and can retry the same provider request over HTTP; they expose no stable WebSocket-only or fallback-off provider setting. + +## Goals / Non-Goals + +**Goals:** + +- Publish a machine-local Codex configuration with separate ordinary and Daybreak Blue providers. +- Make `--profile daybreak-blue` select the Daybreak provider and canonical `gpt-5.6-sol` model. +- Send the exact trusted-cyber carrier on every Daybreak provider request, including the first request, while leaving ordinary provider requests unchanged. +- Authenticate and fail closed before routing if the Daybreak carrier reaches any unsupported HTTP route or non-Responses WebSocket, while allowing authenticated model-catalog initialization. +- Prove the checked-in configuration against an installed Codex client, the real direct-WebSocket capability-ingress and first-selection seam, and the HTTP fallback seam without external requests or credentials. + +**Non-Goals:** + +- Granting Trusted Access, discovering approved identities, or deriving authorization from a model slug, prompt, skill, user agent, or task content. +- Adding a global carrier, changing the selector, changing reactive `cyber_policy` replay rules, or automatically editing a user's Codex configuration. +- Extending proactive capability lineage through the HTTP bridge, compact, control, chat, Images, Live WebSocket, retry, or reactive-policy machinery. +- Supporting a Daybreak alias that is absent from the current upstream catalog, or running a live provider canary. + +## Decisions + +### Use a second provider plus a profile file + +The published base `config.toml` defines the existing `codex-lb` provider unchanged and a second `codex-lb-daybreak-blue` provider with `env_key = "CODEX_LB_API_KEY"` and the exact static capability header. A separate `daybreak-blue.config.toml` overlay selects that provider. This matches current Codex profile-file semantics, supplies the API-key principal required to trust the signal, and makes activation explicit. Direct Responses WebSocket ingress treats the presence of the internal capability header as requiring validation of that key even when global proxy API-key auth is disabled. HTTP routes apply that same rule through the existing `validate_proxy_api_key` Security dependency rather than a parallel FastAPI identity, so headerless authentication hooks remain intact. Requests without the header retain the existing deployment-level authentication behavior. + +Alternative: add the header to the ordinary provider. Rejected because provider headers apply to every request and would incorrectly classify all traffic as requiring trusted-cyber routing. + +Alternative: require operators to enable global proxy API-key auth before selecting the Daybreak profile. Rejected because that would change the authentication requirement for ordinary traffic; per-request validation keeps the opt-in boundary narrow. + +Alternative: put provider selection in project-local `.codex/config.toml`. Rejected because Codex ignores machine-local provider and profile keys in project configuration. + +### Reject capability-bearing unsupported transports before routing + +The static provider header follows Codex when a WebSocket attempt falls back to HTTP and on other requests made through the selected provider. External Responses, compact, thread-goal, Codex-control, admission, warmup, files, transcription, Images, and reset-credit consume HTTP routes therefore validate the supplied proxy API key even when deployment-wide authentication is disabled, then return `400 required_capability_transport_unsupported` before non-framework body parsing, model-source lookup, account or ChatGPT usage-identity selection, reservation, fan-out, bridge creation, owner binding, credential decryption, or upstream dispatch. Chat Completions applies the same guard defensively if a provider client reaches that equivalent routing sink. A signed internal bridge request with an appended carrier authenticates and fails closed after signature validation but before legacy-anchor or account routing; legitimate origin forwarding is unchanged because the origin strips the carrier before forwarding. Non-Responses Live WebSockets apply the same authenticate-then-deny contract before owner lookup or upstream connection. Direct Responses WebSocket is the only supported routing transport. Authenticated `/models`, local API-key usage, and reset-credit listing requests remain available because they do not select an upstream account or contact upstream; the carrier-authenticated Codex usage path binds directly to the proxy API-key principal and cannot enter ChatGPT usage validation. The separate WHAM namespace still forwards after the shared `validate_proxy_api_key` gate and does not apply the Responses transport denial, because it is not Codex provider ingress. Headerless requests retain the existing authentication and routing behavior. The HTTP guard uses `400`, not `426`, because Codex treats `426` as a WebSocket-to-HTTP fallback signal. + +Alternative: carry proactive capability intent through HTTP selection and failover. Rejected for this bounded fix because HTTP streaming, bridge forwarding, compact, reservations, ownership, retries, and reactive `cyber_policy` compatibility would all need a new strict-versus-reactive invariant to prove that no ordinary account can be selected. + +Alternative: allow Images or Live traffic to ignore the carrier because they do not use the Responses WebSocket selector. Rejected because the authenticated carrier is an explicit required capability; silently entering those routes' ordinary account pipelines would downgrade the caller's stated security boundary. + +Alternative: rely on `supports_websockets = true` or a feature flag to prevent fallback. Rejected because installed-client verification shows WebSocket attempts can fall back to HTTP and current clients expose no stable fallback-off setting. + +### Classify the complete registered proxy ingress + +The regression inventory is generated from the routes registered by +`create_app()` and partitions all 45 proxy method/path entries into exactly one +policy group: + +- direct carrier routing: the two Responses WebSocket routes; +- authenticate-then-deny: every routing-capable external HTTP route, signed internal bridge defense-in-depth, and the three non-Responses WebSocket routes, including both reset-credit consume surfaces; +- authenticated local handling: Codex and `/v1` models, `/v1/usage`, `GET /v1/reset-credit`, `/api/codex/usage` with and without its trailing slash, and the local Images variations rejection; +- separate namespace: WHAM JWKS. + +The `/backend-api/codex/v1/` middleware alias canonicalizes into those +registered Codex routes and therefore inherits their policy. The inventory +test compares the complete runtime registration against these disjoint groups, +so adding a provider-reachable proxy route without an explicit carrier policy +fails the regression instead of opening another route-by-route review loop. + +### Keep the canonical upstream model slug + +The Daybreak profile selects `gpt-5.6-sol`. Daybreak Blue may resolve to that model, while access is also bound to the approved identity/workspace or API project and product surface. A distinct provider/profile and the authenticated capability carrier express the routing requirement without teaching codex-lb that a model alias is authorization. + +Alternative: infer trusted-cyber intent from `gpt-daybreak-blue-latest` or any other model string. Rejected because model selection alone neither proves Trusted Access nor authenticates routing intent. + +### Treat checked-in examples as the client-integration contract + +The user-facing examples live under `docs/examples/codex/` and are linked from `docs/client-setup.md`. The integration tests parse those exact TOML files, apply the configured provider contract to real direct Responses WebSocket, HTTP, Images, Live WebSocket, middleware-alias, and signed internal bridge routes, and observe the first-selection or pre-routing denial boundary. An opt-in E2E regression runs an installed Codex binary with temporary `HOME` and `CODEX_HOME`, a fake API key, and a loopback-only network sandbox. It proves sibling-profile resolution, environment-key handling, initial WebSocket header emission, retained HTTP-fallback headers, and no request when the environment key is missing. + +Alternative: test a duplicated inline dictionary or documentation prose. Rejected because it could pass after the published configuration drifts. + +## Risks / Trade-offs + +- **A user selects the Daybreak profile without an approved account surface** -> The carrier grants nothing; canonical selection fails closed when no eligible `security_work_authorized` account exists. +- **A user selects the profile without a valid Codex LB API key** -> Capability ingress requires and validates the dedicated key before selection, independent of the deployment-wide API-key-auth toggle. +- **Codex profile or fallback semantics change** -> The checked-in TOML remains parseable and the opt-in installed-client regression exercises the real loader and first transport attempts; future client changes require updating the configuration contract and regression together. +- **WebSocket is unavailable or another provider route is requested** -> Capability-bearing HTTP and non-Responses WebSockets fail closed with a stable error before routing; direct Responses WebSocket availability must be restored rather than dropping the carrier. This intentionally makes `$imagegen` unavailable inside the Daybreak profile until Images routing can enforce the same capability invariant. +- **Two provider blocks duplicate endpoint settings** -> The duplication is deliberate because Codex providers do not inherit headers safely and isolation is the control that preserves ordinary traffic. +- **The inert regression does not prove current upstream provisioning** -> It proves only client configuration and codex-lb routing behavior; live identity/product-surface approval remains an external prerequisite. + +## Migration Plan + +Existing users keep the ordinary provider unchanged. To opt in, they add the second provider to user-level `config.toml`, place `daybreak-blue.config.toml` beside it, and explicitly launch with `--profile daybreak-blue`. Rollback removes the profile file and optional second provider; no server, database, or persisted request state changes are required. + +## Open Questions + +None. diff --git a/openspec/changes/add-daybreak-blue-client-profile/proposal.md b/openspec/changes/add-daybreak-blue-client-profile/proposal.md new file mode 100644 index 0000000000..fc969baa55 --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/proposal.md @@ -0,0 +1,40 @@ +## Why + +Codex clients configured with only the ordinary `codex-lb` provider cannot express a Trusted Access requirement before the first account-selection attempt. A later upstream `cyber_policy` result may arrive after `response.created`, when changing accounts or replaying the request is no longer safe. + +## What Changes + +- Publish a separate, explicitly selected Daybreak Blue Codex provider and profile that authenticates with a Codex LB API key and adds `X-Codex-LB-Required-Capability: trusted_cyber` to every provider request. +- Require a valid proxy API key for a direct Responses WebSocket request that carries the capability header even when global proxy API-key auth is disabled, without changing auth behavior for requests that omit the header. +- Authenticate and reject capability-bearing HTTP requests on provider-bound routing surfaces before account selection or upstream dispatch, including control, warmup, files, transcription, Images, reset-credit consume, and signed internal bridge calls; guard Chat Completions as a defensive equivalent sink. +- Authenticate and reject capability-bearing non-Responses WebSockets before owner lookup or upstream connection; keep authenticated model-catalog initialization available because it performs no account routing. +- Keep the existing ordinary `codex-lb` provider free of the capability carrier and preserve its current model and routing behavior. +- Select the existing `gpt-5.6-sol` model through the Daybreak profile; the profile name and authorized account surface, not a model alias alone, identify the intended use. +- Add inert regressions that load the published client configuration, exercise an installed Codex client in a network-isolated loopback harness, prove the Daybreak profile reaches capability ingress before first selection, and prove middleware aliases, signed bridge forwarding, and unsupported HTTP or WebSocket transports cannot reach ordinary routing. +- Document that the profile narrows routing only to accounts already marked and independently approved for security work; it does not grant Trusted Access. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `api-keys`: Require a valid proxy API key through the existing `validate_proxy_api_key` dependency whenever the capability header is present, even when the global authentication switch is off. +- `responses-api-compat`: Define the opt-in Codex provider/profile contract that carries trusted-cyber intent before the first Responses WebSocket routing decision without changing ordinary client traffic. +- `images-api-compat`: Require capability-bearing Images requests to authenticate and fail closed before the existing ordinary image-routing pipeline. +- `chat-completions-compat`: Defensively reject the authenticated carrier before Chat Completions can enter ordinary routing. +- `realtime-api-compat`: Reject the carrier on call creation and non-Responses Live WebSockets before account or owner resolution. +- `proxy-admission-control`: Reject the carrier before opportunistic admission evaluates ordinary account capacity. +- `model-catalog-compat`: Keep authenticated local model-catalog initialization available without account routing. +- `proxy-warmup`: Reject the carrier before account-pool evaluation or fan-out. +- `files-upload-protocol`: Reject the carrier before reservation, account selection, upload registration, or polling. +- `audio-transcriptions-compat`: Reject the carrier before multipart parsing or transcription routing. +- `rate-limit-reset-credits`: Reject the carrier before reset-credit account or ChatGPT usage-identity routing while preserving authenticated local usage initialization. + +## Impact + +- Capability ingress guards, user-facing Codex client setup documentation, and checked-in inert configuration examples. +- Focused installed-client, direct Responses WebSocket, unsupported provider-route, Images, and Live WebSocket coverage at the client-config-to-capability-ingress seam. +- No proxy selector changes, new settings, environment variables, dependencies, migrations, dashboard changes, or automatic modification of user configuration. Ordinary requests keep their existing authentication and routing behavior. diff --git a/openspec/changes/add-daybreak-blue-client-profile/specs/api-keys/spec.md b/openspec/changes/add-daybreak-blue-client-profile/specs/api-keys/spec.md new file mode 100644 index 0000000000..153d1eb1b4 --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/specs/api-keys/spec.md @@ -0,0 +1,18 @@ +## ADDED Requirements + +### Requirement: Required-capability header authenticates through the existing proxy API-key dependency + +Whenever a protected proxy request carries one or more `X-Codex-LB-Required-Capability` values, the existing `validate_proxy_api_key` Security dependency MUST require a valid proxy API key before the handler runs, even when `api_key_auth_enabled` is false and the caller would otherwise qualify as local or CIDR-allowlisted. Headerless requests MUST retain the existing global-switch behavior. The capability header MUST NOT introduce a second FastAPI authentication dependency identity for ordinary proxy routes. + +#### Scenario: Capability header requires a key while global auth is disabled + +- **WHEN** `api_key_auth_enabled` is false +- **AND** a local or CIDR-allowlisted client sends a protected proxy request with `X-Codex-LB-Required-Capability` +- **THEN** ingress requires a valid proxy API key +- **AND** a missing or invalid key is rejected with the existing `401 invalid_api_key` error + +#### Scenario: Headerless requests keep the global authentication switch + +- **WHEN** `api_key_auth_enabled` is false +- **AND** a local or CIDR-allowlisted client sends a protected proxy request without `X-Codex-LB-Required-Capability` +- **THEN** the request proceeds without a new per-request API-key requirement diff --git a/openspec/changes/add-daybreak-blue-client-profile/specs/audio-transcriptions-compat/spec.md b/openspec/changes/add-daybreak-blue-client-profile/specs/audio-transcriptions-compat/spec.md new file mode 100644 index 0000000000..29b9c6e002 --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/specs/audio-transcriptions-compat/spec.md @@ -0,0 +1,16 @@ +## ADDED Requirements + +### Requirement: Daybreak capability intent fails closed before transcription parsing + +`POST /backend-api/transcribe` and `POST /v1/audio/transcriptions` MUST require a valid proxy API key whenever `X-Codex-LB-Required-Capability` is present, even when deployment-wide API-key authentication is disabled. After authentication they MUST return HTTP 400 with `error.code = "required_capability_transport_unsupported"` before multipart parsing, model-source lookup, usage reservation, account selection, or upstream dispatch. Headerless transcription requests MUST retain their existing behavior. + +#### Scenario: Authenticated carrier is denied before transcription parsing + +- **WHEN** a valid proxy API key sends either transcription route with the Daybreak carrier +- **THEN** the route returns HTTP 400 `required_capability_transport_unsupported` +- **AND** no multipart body is parsed and no model source, reservation, account, or upstream request is selected + +#### Scenario: Headerless transcription behavior remains unchanged + +- **WHEN** a transcription request omits the required-capability carrier +- **THEN** the existing authentication, parsing, policy, account-routing, and response behavior remains in effect diff --git a/openspec/changes/add-daybreak-blue-client-profile/specs/chat-completions-compat/spec.md b/openspec/changes/add-daybreak-blue-client-profile/specs/chat-completions-compat/spec.md new file mode 100644 index 0000000000..ba4877af9e --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/specs/chat-completions-compat/spec.md @@ -0,0 +1,16 @@ +## ADDED Requirements + +### Requirement: Daybreak capability intent fails closed on Chat Completions + +`POST /v1/chat/completions` MUST require a valid proxy API key whenever `X-Codex-LB-Required-Capability` is present, even when deployment-wide API-key authentication is disabled. After authentication it MUST return HTTP 400 with `error.code = "required_capability_transport_unsupported"` before model-source lookup, usage reservation, account selection, Responses conversion, or upstream dispatch. Headerless Chat Completions requests MUST retain their existing behavior. + +#### Scenario: Authenticated carrier is denied before chat routing + +- **WHEN** a valid proxy API key sends a Chat Completions request with the Daybreak capability carrier +- **THEN** the route returns HTTP 400 `required_capability_transport_unsupported` +- **AND** no model source, reservation, account, Responses request, or upstream attempt is selected + +#### Scenario: Headerless chat behavior remains unchanged + +- **WHEN** a Chat Completions request omits the required-capability carrier +- **THEN** the route retains its existing authentication, validation, routing, and response behavior diff --git a/openspec/changes/add-daybreak-blue-client-profile/specs/files-upload-protocol/spec.md b/openspec/changes/add-daybreak-blue-client-profile/specs/files-upload-protocol/spec.md new file mode 100644 index 0000000000..22fba4096e --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/specs/files-upload-protocol/spec.md @@ -0,0 +1,16 @@ +## ADDED Requirements + +### Requirement: Daybreak capability intent fails closed before file routing + +`POST /backend-api/files` and `POST /backend-api/files/{file_id}/uploaded` MUST require a valid proxy API key whenever `X-Codex-LB-Required-Capability` is present, even when deployment-wide API-key authentication is disabled. After authentication they MUST return HTTP 400 with `error.code = "required_capability_transport_unsupported"` before usage reservation, account selection, upload registration, status polling, or upstream dispatch. Headerless file requests MUST retain their existing behavior. + +#### Scenario: Authenticated carrier is denied before file account selection + +- **WHEN** a valid proxy API key sends a file-create or file-finalize request with the Daybreak carrier +- **THEN** the route returns HTTP 400 `required_capability_transport_unsupported` +- **AND** no reservation, account selection, upload registration, polling loop, or upstream request begins + +#### Scenario: Headerless file behavior remains unchanged + +- **WHEN** a file-create or file-finalize request omits the required-capability carrier +- **THEN** the existing authentication, validation, reservation, routing, and response behavior remains in effect diff --git a/openspec/changes/add-daybreak-blue-client-profile/specs/images-api-compat/spec.md b/openspec/changes/add-daybreak-blue-client-profile/specs/images-api-compat/spec.md new file mode 100644 index 0000000000..052523d917 --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/specs/images-api-compat/spec.md @@ -0,0 +1,23 @@ +## ADDED Requirements + +### Requirement: Daybreak capability intent fails closed on Images HTTP routes + +The Codex-base and `/v1` image generation and edit routes MUST require a valid proxy API key whenever `X-Codex-LB-Required-Capability` is present, even when deployment-wide API-key authentication is disabled. After authentication they MUST return HTTP 400 with `error.code = "required_capability_transport_unsupported"` before request-body parsing that is not already required by framework validation, model-source lookup, usage reservation, account selection, internal Responses construction, or upstream dispatch. The rejection MUST emit exactly one bounded `images_route_complete` observation. Headerless Images requests MUST preserve their existing behavior. + +#### Scenario: Authenticated Daybreak image request fails closed + +- **WHEN** the Daybreak provider sends a generation or edit request with its valid proxy API key and capability carrier +- **THEN** the Images route returns HTTP 400 `required_capability_transport_unsupported` +- **AND** no model source, reservation, account, internal Responses request, or upstream attempt is selected +- **AND** exactly one bounded invalid-request route observation is emitted + +#### Scenario: Daybreak image request authenticates before transport denial + +- **WHEN** a capability-bearing generation or edit request omits the proxy API key or supplies an invalid key +- **THEN** the Images route returns the existing HTTP 401 `invalid_api_key` response +- **AND** no image body is decoded and no account or upstream request is selected + +#### Scenario: Ordinary Images behavior remains unchanged + +- **WHEN** an Images request omits the required-capability carrier +- **THEN** the route retains its existing authentication, validation, account-routing, observability, and response behavior diff --git a/openspec/changes/add-daybreak-blue-client-profile/specs/model-catalog-compat/spec.md b/openspec/changes/add-daybreak-blue-client-profile/specs/model-catalog-compat/spec.md new file mode 100644 index 0000000000..c5101dea54 --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/specs/model-catalog-compat/spec.md @@ -0,0 +1,22 @@ +## ADDED Requirements + +### Requirement: Daybreak profile can initialize from the local model catalog + +`GET /backend-api/codex/models` MUST validate the proxy API key whenever `X-Codex-LB-Required-Capability` is present, even when deployment-wide API-key authentication is disabled. With a valid key it MUST return the existing local Codex catalog without applying the unsupported-transport denial, selecting an account, or dispatching an upstream request. Headerless model-catalog requests MUST retain their existing behavior. + +#### Scenario: Authenticated Daybreak catalog request remains available + +- **WHEN** the Daybreak provider requests the Codex-native model catalog with its valid key and capability carrier +- **THEN** the route returns the existing catalog response +- **AND** no account is selected and no upstream request is made + +#### Scenario: Catalog carrier requires authentication + +- **WHEN** a capability-bearing catalog request omits its proxy API key or supplies an invalid key +- **THEN** the route returns the existing HTTP 401 `invalid_api_key` response +- **AND** no catalog response or routing attempt occurs + +#### Scenario: Headerless catalog behavior remains unchanged + +- **WHEN** a model-catalog request omits the required-capability carrier +- **THEN** the existing deployment-level authentication and catalog behavior remains in effect diff --git a/openspec/changes/add-daybreak-blue-client-profile/specs/proxy-admission-control/spec.md b/openspec/changes/add-daybreak-blue-client-profile/specs/proxy-admission-control/spec.md new file mode 100644 index 0000000000..0a177d85aa --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/specs/proxy-admission-control/spec.md @@ -0,0 +1,16 @@ +## ADDED Requirements + +### Requirement: Daybreak capability intent bypasses ordinary opportunistic admission + +`GET /backend-api/codex/opportunistic/admission` MUST require a valid proxy API key whenever `X-Codex-LB-Required-Capability` is present and MUST then return HTTP 400 with `error.code = "required_capability_transport_unsupported"` before model-source or ordinary account-capacity evaluation. Headerless admission requests MUST retain their existing behavior. + +#### Scenario: Authenticated carrier is denied before admission evaluation + +- **WHEN** a valid proxy API key requests opportunistic admission with the Daybreak carrier +- **THEN** the route returns HTTP 400 `required_capability_transport_unsupported` +- **AND** no model source or ordinary account capacity is evaluated + +#### Scenario: Headerless admission behavior remains unchanged + +- **WHEN** an opportunistic admission request omits the required-capability carrier +- **THEN** the existing admission policy remains in effect diff --git a/openspec/changes/add-daybreak-blue-client-profile/specs/proxy-warmup/spec.md b/openspec/changes/add-daybreak-blue-client-profile/specs/proxy-warmup/spec.md new file mode 100644 index 0000000000..21e3a47a4a --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/specs/proxy-warmup/spec.md @@ -0,0 +1,16 @@ +## ADDED Requirements + +### Requirement: Daybreak capability intent fails closed before warmup fan-out + +`POST /v1/warmup` and `POST /v1/warmup/{mode}` MUST require a valid proxy API key whenever `X-Codex-LB-Required-Capability` is present, even when deployment-wide API-key authentication is disabled. After authentication they MUST return HTTP 400 with `error.code = "required_capability_transport_unsupported"` before mode validation, account-pool evaluation, or any upstream warmup submission. Headerless warmup requests MUST retain their existing behavior. + +#### Scenario: Authenticated carrier is denied before warmup routing + +- **WHEN** a valid proxy API key sends either warmup route with the Daybreak carrier +- **THEN** the route returns HTTP 400 `required_capability_transport_unsupported` +- **AND** no account pool is evaluated and no upstream warmup is submitted + +#### Scenario: Headerless warmup behavior remains unchanged + +- **WHEN** a warmup request omits the required-capability carrier +- **THEN** the existing authentication, mode, account-scope, and fan-out behavior remains in effect diff --git a/openspec/changes/add-daybreak-blue-client-profile/specs/rate-limit-reset-credits/spec.md b/openspec/changes/add-daybreak-blue-client-profile/specs/rate-limit-reset-credits/spec.md new file mode 100644 index 0000000000..8d592a46a3 --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/specs/rate-limit-reset-credits/spec.md @@ -0,0 +1,22 @@ +## ADDED Requirements + +### Requirement: Daybreak capability intent cannot downgrade through reset-credit routing + +`POST /v1/reset-credit` and `POST /api/codex/rate-limit-reset-credits/consume` (with or without its trailing slash) MUST require a valid proxy API key whenever `X-Codex-LB-Required-Capability` is present. After authentication they MUST return HTTP 400 with `error.code = "required_capability_transport_unsupported"` before account lookup, ChatGPT usage-identity validation, credential decryption, upstream route resolution, reset-credit fetch, or reset-credit consume. Headerless requests MUST retain their existing authentication and redemption behavior. Capability-bearing reads of `/api/codex/usage`, `/v1/usage`, and `/v1/reset-credit` MAY remain available after proxy API-key authentication because their API-key paths are local and do not select an upstream account or dispatch an upstream request. They MUST NOT enter ChatGPT usage-identity validation while the carrier is present. + +#### Scenario: Authenticated reset-credit carrier fails before account routing + +- **WHEN** a valid proxy API key sends either reset-credit consume surface with the Daybreak carrier +- **THEN** ingress returns HTTP 400 `required_capability_transport_unsupported` +- **AND** no account, ChatGPT identity, credential, route, fetch, or consume operation is reached + +#### Scenario: Local usage initialization authenticates without upstream identity lookup + +- **WHEN** a valid proxy API key reads a local usage or reset-credit listing with the Daybreak carrier +- **THEN** the existing local API-key response remains available +- **AND** no ChatGPT usage-identity request or upstream account routing occurs + +#### Scenario: Headerless reset-credit behavior remains unchanged + +- **WHEN** a reset-credit request omits the required-capability carrier +- **THEN** the existing API-key or ChatGPT identity authentication and redemption behavior remains in effect diff --git a/openspec/changes/add-daybreak-blue-client-profile/specs/realtime-api-compat/spec.md b/openspec/changes/add-daybreak-blue-client-profile/specs/realtime-api-compat/spec.md new file mode 100644 index 0000000000..1b2b4cc53d --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/specs/realtime-api-compat/spec.md @@ -0,0 +1,28 @@ +## ADDED Requirements + +### Requirement: Daybreak capability intent fails closed on private Realtime transports + +Capability-bearing private Realtime requests MUST authenticate and fail closed on unsupported transports. `POST /backend-api/codex/realtime/calls` and non-Responses WebSocket handshakes at `/backend-api/codex/{call_id}`, `/v1/live/{call_id}`, and `/v1/realtime` MUST validate the registered proxy API key before returning HTTP 400 with `error.code = "required_capability_transport_unsupported"`. Call creation MUST deny before account selection or owner binding. WebSocket handshakes MUST deny before acceptance, call-owner lookup, lease acquisition, or upstream connection. Headerless Realtime requests MUST retain their existing required-key and exact-owner behavior. + +#### Scenario: Capability-bearing call creation is denied before selection + +- **WHEN** a valid registered key sends private Realtime call creation with the Daybreak carrier +- **THEN** the route returns HTTP 400 `required_capability_transport_unsupported` +- **AND** no account is selected, no upstream call is created, and no owner is bound + +#### Scenario: Capability-bearing Live WebSocket is denied before owner lookup + +- **WHEN** a valid registered key opens any supported non-Responses Realtime WebSocket with the Daybreak carrier +- **THEN** the handshake receives HTTP 400 `required_capability_transport_unsupported` +- **AND** the route does not accept the WebSocket, resolve a call owner, acquire a lease, or connect upstream + +#### Scenario: Realtime carrier authenticates before transport denial + +- **WHEN** a capability-bearing call-creation request or Live WebSocket omits its key or supplies an invalid key +- **THEN** ingress returns the existing HTTP 401 `invalid_api_key` response +- **AND** no account or owner resolution occurs + +#### Scenario: Headerless Realtime behavior remains unchanged + +- **WHEN** a Realtime call or sideband request omits the required-capability carrier +- **THEN** the existing registered-key, immutable-owner, and transport behavior remains in effect diff --git a/openspec/changes/add-daybreak-blue-client-profile/specs/responses-api-compat/spec.md b/openspec/changes/add-daybreak-blue-client-profile/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..91a96f058e --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/specs/responses-api-compat/spec.md @@ -0,0 +1,71 @@ +## ADDED Requirements + +### Requirement: Codex exposes an explicit Daybreak Blue routing profile + +The published Codex client configuration MUST keep the ordinary `codex-lb` provider free of `X-Codex-LB-Required-Capability` and MUST define a separate `codex-lb-daybreak-blue` provider that sources its proxy API key from `CODEX_LB_API_KEY` and whose static headers contain exactly one `X-Codex-LB-Required-Capability: trusted_cyber` carrier. A machine-local `daybreak-blue` profile file MUST select that provider and the canonical `gpt-5.6-sol` model. Activating the Daybreak profile MUST be explicit and MUST NOT modify the default provider selection. Direct Responses WebSocket ingress MUST require a valid proxy API key whenever the capability header is present, even when deployment-wide API-key auth is disabled, and MUST preserve the existing authentication behavior when the header is absent. Any capability-bearing HTTP request on an external provider-bound route that can select or forward an upstream account, including Responses, compact, thread-goal, Codex-control, opportunistic admission, warmup, files, transcription, chat, Images, and reset-credit consume routes, MUST authenticate the carrier and MUST then fail with `400 required_capability_transport_unsupported` before routing or upstream dispatch. Typed JSON Responses, compact, Chat Completions, Images generations, and reset-credit consume routes MUST apply that authenticate-then-deny check before FastAPI decodes the request body. A capability-bearing non-Responses WebSocket MUST apply the same authenticate-then-deny contract before owner lookup or upstream connection. Authenticated model-catalog, local API-key usage, and reset-credit listing requests MAY remain available because they perform no upstream account routing. The separate WHAM namespace MUST retain ordinary forwarding after the shared capability-header authentication rule and MUST NOT apply the Responses transport denial. Headerless ingress MUST retain its existing behavior. A legitimately forwarded internal bridge request MUST strip the capability carrier. If a carrier is appended to an otherwise valid signed internal bridge request, the target MUST authenticate it and fail closed before legacy-anchor validation, account selection, or upstream dispatch. + +#### Scenario: Daybreak profile constrains the first attempt + +- **WHEN** an authenticated direct Responses WebSocket turn starts through the published `daybreak-blue` profile +- **AND** deployment-wide proxy API-key auth is disabled +- **THEN** capability ingress receives exactly one `trusted_cyber` carrier before the first account-selection call +- **AND** capability ingress validates the profile's proxy API key before accepting the carrier +- **AND** the first and every later selection requires an eligible security-work-authorized account +- **AND** no ordinary account receives an upstream attempt + +#### Scenario: Ordinary provider remains unchanged + +- **WHEN** an authenticated direct Responses WebSocket turn starts through the published ordinary `codex-lb` provider +- **THEN** the request contains no required-capability carrier +- **AND** capability ingress does not impose a new per-request API-key requirement +- **AND** the first account-selection call remains unconstrained by trusted-cyber routing + +#### Scenario: Daybreak HTTP downgrade fails closed before routing + +- **WHEN** Codex retains the Daybreak provider's capability carrier while falling back to an HTTP Responses or compact request +- **AND** the request supplies the profile's valid proxy API key +- **THEN** ingress returns `400 required_capability_transport_unsupported` +- **AND** no model source, account, reservation, bridge, or upstream attempt is selected +- **AND** the request is not replayed through ordinary routing + +#### Scenario: Daybreak HTTP downgrade authenticates before transport denial + +- **WHEN** a capability-bearing HTTP Responses request omits or supplies an invalid proxy API key +- **THEN** ingress returns the existing `401 invalid_api_key` authentication error +- **AND** no routing or upstream attempt occurs + +#### Scenario: Ordinary HTTP routing remains unchanged + +- **WHEN** the published ordinary provider sends an HTTP Responses request without the capability carrier +- **THEN** ingress does not impose the Daybreak per-request authentication or transport denial +- **AND** existing ordinary HTTP routing behavior is preserved + +#### Scenario: Other provider-bound HTTP routes fail closed + +- **WHEN** the authenticated Daybreak carrier reaches a thread-goal, Codex-control, opportunistic-admission, warmup, files, transcription, chat, Images, or reset-credit consume HTTP route +- **THEN** ingress returns `400 required_capability_transport_unsupported` +- **AND** no model source, account, reservation, owner binding, or upstream attempt is selected + +#### Scenario: Non-Responses WebSocket fails closed + +- **WHEN** the authenticated Daybreak carrier reaches a Live or Realtime WebSocket route +- **THEN** ingress returns `400 required_capability_transport_unsupported` during the handshake +- **AND** no call owner is resolved and no upstream connection is opened + +#### Scenario: Signed internal forward cannot reintroduce the carrier + +- **WHEN** an otherwise valid signed internal Responses bridge request arrives with the Daybreak carrier appended +- **THEN** ingress authenticates the proxy API key and returns `400 required_capability_transport_unsupported` +- **AND** no legacy bridge anchor, account selection, or upstream dispatch occurs + +#### Scenario: Model-catalog initialization remains available + +- **WHEN** the authenticated Daybreak provider requests `/backend-api/codex/models` +- **THEN** ingress returns the local model catalog +- **AND** no account is selected and no upstream request is made + +#### Scenario: Profile does not grant authorization + +- **WHEN** the Daybreak profile is selected without an authenticated proxy request or without an eligible security-work-authorized account +- **THEN** the existing capability-ingress or empty-capable-pool contract fails closed +- **AND** routing does not fall back to an ordinary account diff --git a/openspec/changes/add-daybreak-blue-client-profile/tasks.md b/openspec/changes/add-daybreak-blue-client-profile/tasks.md new file mode 100644 index 0000000000..e6dcf3a81f --- /dev/null +++ b/openspec/changes/add-daybreak-blue-client-profile/tasks.md @@ -0,0 +1,23 @@ +## 1. Published client configuration + +- [x] 1.1 Add canonical machine-local Codex examples that preserve the ordinary provider and define a separate Daybreak Blue provider/profile with the exact trusted-cyber header. +- [x] 1.2 Update the client setup guide with explicit profile activation, approved-identity/product-surface prerequisites, rollback guidance, and a link to the owning Responses compatibility spec. +- [x] 1.3 Document the authenticated provider-wide allowlist, unsupported route families, local model-catalog exception, and Daybreak `$imagegen` limitation in their owning specifications and client guide. + +## 2. Inert seam regression + +- [x] 2.1 Add a direct Responses WebSocket integration regression that loads the published TOML examples and proves Daybreak validates its inert API key and routes authorized-only before first selection while ordinary routing remains unconstrained with global API-key auth disabled. +- [x] 2.2 Confirm the existing unauthenticated-signal and empty-capable-pool fail-closed coverage remains applicable without adding external calls or credentials. +- [x] 2.3 Add authenticated HTTP Responses and compact fallback regressions that fail before routing while headerless ordinary HTTP remains unchanged. +- [x] 2.4 Add and run an opt-in installed-Codex loopback regression for real profile resolution, environment-key handling, first WebSocket header emission, and retained HTTP-fallback headers. +- [x] 2.5 Add authenticated fail-closed regressions for provider-bound control, admission, warmup, files, transcription, chat, and Images HTTP routes while preserving local model-catalog initialization and the separate WHAM namespace. +- [x] 2.6 Add a non-Responses WebSocket regression that proves the carrier is authenticated and denied before Live owner lookup or upstream connection while headerless behavior remains unchanged. +- [x] 2.7 Inventory every runtime-registered proxy route, assign one explicit carrier policy, and cover reset-credit, local usage, signed internal-bridge, and middleware-alias seams so new unclassified provider ingress fails the regression. + +## 3. Verification + +- [x] 3.1 Rerun scoped OpenSpec validation, focused capability-routing regressions, affected lint/format checks, documentation build, and `git diff --check` after security-review remediation. +- [x] 3.2 Document the remediated provider-wide transport boundary, Images fail-closed behavior, and `$imagegen` limitation in the change artifacts and client guide. +- [x] 3.3 Inspect the final committed diff with one independent Sensitive review and address every actionable in-scope finding before publication. +- [x] 3.4 Keep capability-header authentication on the existing `validate_proxy_api_key` dependency identity so FastAPI overrides and auth-first upload tests continue to apply. +- [x] 3.5 Deny capability-bearing typed JSON routes before FastAPI decodes the request body. diff --git a/openspec/changes/persist-durable-response-transition-manifests/.openspec.yaml b/openspec/changes/add-model-source-embeddings/.openspec.yaml similarity index 50% rename from openspec/changes/persist-durable-response-transition-manifests/.openspec.yaml rename to openspec/changes/add-model-source-embeddings/.openspec.yaml index 701445b891..41c30bab88 100644 --- a/openspec/changes/persist-durable-response-transition-manifests/.openspec.yaml +++ b/openspec/changes/add-model-source-embeddings/.openspec.yaml @@ -1,2 +1,2 @@ schema: spec-driven -created: 2026-08-26 +created: 2026-08-19 diff --git a/openspec/changes/add-model-source-embeddings/proposal.md b/openspec/changes/add-model-source-embeddings/proposal.md new file mode 100644 index 0000000000..306580eedb --- /dev/null +++ b/openspec/changes/add-model-source-embeddings/proposal.md @@ -0,0 +1,48 @@ +## Why + +Model sources already carry per-protocol capability flags for chat +completions, responses, and audio transcriptions, and the proxy routes each +OpenAI-compatible surface to a source that declares the matching capability. +Embeddings have no such flag and no route, so an operator running a local +embedding model behind an OpenAI-compatible source cannot serve +`POST /v1/embeddings` through the proxy at all. There is also no +subscription-backed upstream to fall back on: unlike chat and responses +traffic, embeddings can only ever be served by a configured model source. + +Adding the capability requires a persisted flag, so the create/read/update +contracts, the dashboard form, request validation, and request-log accounting +all move together. + +## What Changes + +- Add a persisted `supports_embeddings` capability flag to model sources, + defaulting to disabled so existing sources keep their current behavior. +- Expose the flag through the model-source create/read/update API contracts + and the dashboard model-source form. +- Add `POST /v1/embeddings`, routed only to an enabled source that declares + the embeddings capability for the requested model. +- Return `model_not_found` when no enabled source supports embeddings for the + requested model, instead of falling through to subscription accounts. +- Record embeddings requests in the request log with the same success/error + accounting and usage-settlement rules the other model-source routes use, + including failing closed when a limited API key needs usage the source did + not report. + +## Capabilities + +### New Capabilities + +- `model-source-routing`: the embeddings capability flag, its routing rule, + and the `/v1/embeddings` request/accounting contract. + +### Modified Capabilities + +None. + +## Impact + +The change adds one nullable-free boolean column with a `false` server +default, one proxy route, one forwarding helper, and one repository lookup. +It adds no setting, no dependency, and no change to existing routing for chat +completions, responses, or audio transcriptions. Sources that do not opt in +are unaffected. diff --git a/openspec/changes/add-model-source-embeddings/specs/model-source-routing/spec.md b/openspec/changes/add-model-source-embeddings/specs/model-source-routing/spec.md new file mode 100644 index 0000000000..b1772e98ba --- /dev/null +++ b/openspec/changes/add-model-source-embeddings/specs/model-source-routing/spec.md @@ -0,0 +1,108 @@ +# model-source-routing Delta + +## ADDED Requirements + +### Requirement: Model sources declare an embeddings capability + +Each model source MUST carry a persisted `supports_embeddings` boolean +capability flag. The flag MUST default to disabled, so a source created or +migrated without an explicit value MUST NOT be treated as embeddings-capable. +The model-source create, read, and update contracts MUST expose the flag, and +the stored value MUST survive a round trip through those contracts. + +#### Scenario: existing sources default to disabled + +- **GIVEN** a model source row that predates the embeddings capability +- **WHEN** the schema migration runs +- **THEN** the source reports `supports_embeddings` as disabled +- **AND** its existing chat-completions, responses, and audio-transcription + routing is unchanged + +#### Scenario: capability round-trips through the API + +- **WHEN** a client creates or updates a model source with the embeddings + capability enabled +- **THEN** reading the source back reports the capability as enabled + +#### Scenario: omitted capability parses as disabled + +- **WHEN** a model-source payload omits `supports_embeddings` +- **THEN** it parses as disabled rather than failing validation + +### Requirement: Embeddings route only to capable model sources + +The system SHALL expose `POST /v1/embeddings` and MUST serve it only from an +enabled model source of kind `openai_compatible` that declares the embeddings +capability and has the requested model enabled. Embeddings requests MUST NOT +fall back to subscription-backed accounts. When the caller presents an API key +restricted to a set of sources, selection MUST stay inside that set. Beyond +the validated `model` and `input` fields, the request payload MUST be +forwarded to the source verbatim. + +#### Scenario: capable source serves the request + +- **GIVEN** an enabled model source declaring the embeddings capability with + the requested model enabled +- **WHEN** a client posts to `/v1/embeddings` +- **THEN** the proxy forwards the payload to that source's `/embeddings` + endpoint and returns the upstream JSON response + +#### Scenario: no capable source is a model error + +- **GIVEN** no enabled model source declares the embeddings capability for + the requested model +- **WHEN** a client posts to `/v1/embeddings` +- **THEN** the proxy returns 404 with an OpenAI-format error envelope using + code `model_not_found` +- **AND** the request is not routed to a subscription-backed account + +#### Scenario: source-restricted API key cannot escape its set + +- **GIVEN** an API key restricted to a set of model sources +- **WHEN** the only embeddings-capable source for the model is outside that + set +- **THEN** the proxy returns `model_not_found` + +### Requirement: Embeddings requests are accounted like other source routes + +Embeddings responses MUST be inspected for prompt and total token usage. When +the caller's API key requires usage for settlement and the source response +reports none, the proxy MUST fail closed with `usage_unavailable` rather than +serving unmetered traffic. Every embeddings attempt that is dispatched to a +model source MUST produce a request-log entry, with `success` on a forwarded +response and `error` on a forwarding, usage, or settlement failure. That entry +MUST carry the upstream status code when a source returned an HTTP response, +and MUST record the upstream status as absent when the attempt failed before +any response was received. A request rejected before source selection succeeds +is not a dispatched attempt: it MUST NOT produce a request-log entry, because +no source was contacted and no reservation was consumed. + +#### Scenario: missing usage fails closed for a limited key + +- **GIVEN** an API key whose reservation requires reported usage +- **WHEN** the model source returns an embeddings response without a usage + object +- **THEN** the proxy returns an error envelope using code `usage_unavailable` +- **AND** records an error request log + +#### Scenario: forwarding error propagates the upstream status + +- **WHEN** the model source returns an error status for an embeddings request +- **THEN** the proxy returns an OpenAI-format error envelope with that status +- **AND** records an error request log carrying the upstream status code + +#### Scenario: transport failure records an attempt without an upstream status + +- **WHEN** the request to the model source fails before any HTTP response is + received +- **THEN** the proxy records an error request log for the attempt with no + upstream status code + +#### Scenario: unroutable model is not a logged attempt + +- **GIVEN** no enabled model source declares the embeddings capability for + the requested model +- **WHEN** a client posts to `/v1/embeddings` +- **THEN** the proxy returns the `model_not_found` envelope without writing a + request-log entry +- **AND** no reservation is consumed for the rejected request diff --git a/openspec/changes/add-model-source-embeddings/tasks.md b/openspec/changes/add-model-source-embeddings/tasks.md new file mode 100644 index 0000000000..78faa95139 --- /dev/null +++ b/openspec/changes/add-model-source-embeddings/tasks.md @@ -0,0 +1,31 @@ +## 1. Persisted Capability + +- [x] 1.1 Add the `supports_embeddings` column with a `false` server default + and an idempotent migration that tolerates a pre-existing column. +- [x] 1.2 Surface the flag in the model-source create/read/update schemas. +- [x] 1.3 Surface the flag in the dashboard model-source form and locales. + +## 2. Routing + +- [x] 2.1 Add a repository lookup that selects an enabled source declaring + the embeddings capability for the requested model. +- [x] 2.2 Add `POST /v1/embeddings` and forward the payload verbatim beyond + the validated `model` and `input` fields. +- [x] 2.3 Return `model_not_found` when no source qualifies, with no + subscription-account fallback. + +## 3. Accounting + +- [x] 3.1 Parse prompt/total token usage from the embeddings response shape. +- [x] 3.2 Fail closed with `usage_unavailable` when a limited API key needs + usage the source did not report. +- [x] 3.3 Record success and error request logs with the upstream status. + +## 4. Verification + +- [x] 4.1 Add integration coverage for capability routing, the missing-source + path, and usage accounting. +- [x] 4.2 Add frontend coverage for the capability default and the enabled + submit path. +- [x] 4.3 Run Ruff check/format, type checks, and the migration round-trip + test. diff --git a/openspec/changes/recognize-canonical-invalid-previous-response-anchor/.openspec.yaml b/openspec/changes/add-root-file-allowlist-budget/.openspec.yaml similarity index 50% rename from openspec/changes/recognize-canonical-invalid-previous-response-anchor/.openspec.yaml rename to openspec/changes/add-root-file-allowlist-budget/.openspec.yaml index 44f55ffeea..f774115be7 100644 --- a/openspec/changes/recognize-canonical-invalid-previous-response-anchor/.openspec.yaml +++ b/openspec/changes/add-root-file-allowlist-budget/.openspec.yaml @@ -1,2 +1,2 @@ schema: spec-driven -created: 2026-08-23 +created: 2026-08-20 diff --git a/openspec/changes/add-root-file-allowlist-budget/proposal.md b/openspec/changes/add-root-file-allowlist-budget/proposal.md new file mode 100644 index 0000000000..b8138a4297 --- /dev/null +++ b/openspec/changes/add-root-file-allowlist-budget/proposal.md @@ -0,0 +1,25 @@ +## Why + +Tracked files can accumulate at the repository root without review, including one-off agent artifacts that obscure the intended project surface. The existing simplicity-budget mechanism should make additions to that surface explicit and reviewer-visible. + +## What Changes + +- Define the complete set of allowed tracked repository-root entries in the simplicity-budget configuration. +- Extend the simplicity-budget checker to reject tracked root entries outside that allowlist, while preserving the existing PR-label override behavior. +- Relocate the proxy architecture ADR into its owning OpenSpec capability context and remove obsolete root-level agent debris. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `contribution-simplicity`: Budget the tracked repository-root surface with an explicit allowlist and the existing maintainer override. + +## Impact + +- `.github/simplicity-budgets.toml` gains the root-entry allowlist. +- `.github/scripts/check_simplicity_budgets.py` checks the committed root tree. +- Proxy architecture context moves under `openspec/specs/proxy-architecture/`; obsolete root files are removed and ignored against recurrence. diff --git a/openspec/changes/add-root-file-allowlist-budget/specs/contribution-simplicity/spec.md b/openspec/changes/add-root-file-allowlist-budget/specs/contribution-simplicity/spec.md new file mode 100644 index 0000000000..4631d898e7 --- /dev/null +++ b/openspec/changes/add-root-file-allowlist-budget/specs/contribution-simplicity/spec.md @@ -0,0 +1,32 @@ +## ADDED Requirements + +### Requirement: Tracked repository-root entries are allowlisted + +Every tracked repository-root entry (file or directory, as listed by `git ls-tree --name-only HEAD`) MUST appear in the `allowed` list of the `[root_files]` section in `.github/simplicity-budgets.toml`, and the simplicity-budget check SHALL report each unlisted entry as a violation that names the entry and the escape hatch. A PR that adds an unlisted root entry SHALL be blocked from merge unless the entry is added to the allowlist in the same diff or a maintainer applies the `simplicity-budget-approved` label. When the `[root_files]` section is absent from the budget configuration, the check SHALL be skipped rather than fail. + +#### Scenario: Root tree matches the allowlist + +- **WHEN** every tracked repository-root entry appears in the `[root_files]` allowlist +- **THEN** the simplicity-budget check passes with no label required + +#### Scenario: Stray root file without an allowlist update + +- **WHEN** a PR commits a new repository-root file without adding it to the `[root_files]` allowlist +- **AND** no `simplicity-budget-approved` label is present +- **THEN** the simplicity-budget check fails with a violation naming that file and the escape hatch, and the PR is blocked from merge + +#### Scenario: Intentional root entry added with the allowlist in the same diff + +- **WHEN** a PR adds a repository-root entry and adds it to the `[root_files]` allowlist in the same diff +- **THEN** the simplicity-budget check passes, and the allowlist change is visible to the reviewer + +#### Scenario: Stray root entry with maintainer approval + +- **GIVEN** a PR whose tracked root entry is not in the allowlist +- **WHEN** a maintainer applies the `simplicity-budget-approved` label +- **THEN** the violation is downgraded to a warning on the PR run and the check passes + +#### Scenario: Budget configuration without a root-files section + +- **WHEN** the budget configuration has no `[root_files]` section +- **THEN** the simplicity-budget check skips root-entry enforcement and evaluates the remaining budgets unchanged diff --git a/openspec/changes/add-root-file-allowlist-budget/tasks.md b/openspec/changes/add-root-file-allowlist-budget/tasks.md new file mode 100644 index 0000000000..22f5165e2b --- /dev/null +++ b/openspec/changes/add-root-file-allowlist-budget/tasks.md @@ -0,0 +1,14 @@ +## 1. Relocate root documents + +- [x] 1.1 Move the ADR-0001 body from the root `DECISIONS.md` into `openspec/specs/proxy-architecture/context.md` with a short relocation header, then delete `DECISIONS.md`. +- [x] 1.2 Delete the root `SUMMARY.md` agent debris and add `DECISIONS.md` beside the existing `SUMMARY.md` line in the `.gitignore` agent-debris block. + +## 2. Enforce the root-entry allowlist + +- [x] 2.1 Add a `[root_files]` section with a sorted `allowed` list of every tracked repository-root entry to `.github/simplicity-budgets.toml`. +- [x] 2.2 Extend `.github/scripts/check_simplicity_budgets.py` to compare `git ls-tree --name-only HEAD` against the allowlist, reporting each unlisted entry as a violation that names the file and the escape hatch, keeping the override-label and exit-code semantics, and skipping the check when `[root_files]` is absent. + +## 3. Verification + +- [x] 3.1 Run the budget checker on the final tree (exit 0) and demonstrate that a stray committed root file fails the check with a named violation. +- [x] 3.2 Validate the OpenSpec change and run repository lint. diff --git a/openspec/changes/add-rowless-checkpoint-semantic-rebase/design.md b/openspec/changes/add-rowless-checkpoint-semantic-rebase/design.md deleted file mode 100644 index b3b3d07e1c..0000000000 --- a/openspec/changes/add-rowless-checkpoint-semantic-rebase/design.md +++ /dev/null @@ -1,159 +0,0 @@ -# Design: operator-acknowledged rowless semantic rebase - -## Authority record - -`http_bridge_rowless_recovery_authorities` is deliberately independent of -`http_bridge_sessions`; it has no cascading foreign key. Its stable identity -is `(api_key_scope, strong_session_hash, stale_anchor_hash)`. The strong -session hash is derived from a task-authority digest over exact ingress -`session-id`, body `prompt_cache_key`, and `thread-id`; it is independent of -ephemeral turn-state routing. Plain session and response identifiers are -never persisted. - -This release admits only root task requests for which the three identity -values are exactly equal. Spawned child threads can share their root's -session/prompt bridge row, so they fail closed until the durable bridge has a -per-thread replacement-session identity. - -State transitions are monotonic except for a physically proven unsent rollback: - -``` -captured -> approved -> unknown -> consumed - | - +-- proven unsent only --> approved -``` - -`unknown` means the send primitive may have been reached and is never replayed. -Consumed rows are permanent compact no-replay tombstones. Automatic cleanup -may purge only unapproved CAPTURED rows older than seven days; it never purges -APPROVED, UNKNOWN, or CONSUMED authority. -Schema downgrade likewise refuses to drop the authority table while any -APPROVED, UNKNOWN, or CONSUMED replay fence exists. Rolling back an image does -not require erasing these monotonic database fences. -The trusted-operator status endpoint publishes content-free per-state counts -and an explicit minimum rollback capability. After the first non-CAPTURED -authority exists, production MUST NOT roll back to a pre-rowless image because -such an image cannot enforce the retained replay fences. - -Tombstone lookup does not trust the retry's anchor field. For the stable root -task identity, an exact match of captured input count/fingerprint, non-input -contract, retained direct-call ledger, and projected logical payload resolves -one existing authority even when the client omits or changes -`previous_response_id`. CAPTURED, UNKNOWN, and CONSUMED therefore remain local -terminal states; a unique APPROVED match still passes through the same -generation CAS. A changed/new turn does not match this request domain and is -admitted only through ordinary continuity with the replacement anchor. -The database enforces that semantic-turn identity as a second unique fence in -addition to the direct stale-anchor lookup key, so two different rejected -anchors cannot create or approve two generations for the same task turn. - -## Capture - -Capture is currently allowed only for a canonical hard `session_header` key, -the exact official `session-id`/`thread-id` headers, an explicit body -`prompt_cache_key`, an explicit client-provided anchor, the canonical -stale-anchor error, and `response_events_seen=0`. The server stores: - -- raw client input count and canonical SHA-256 fingerprint; -- canonical non-input request-contract SHA-256 fingerprint; -- an ordered content-free ledger digest over direct call/output id, type, and - status, plus unresolved count; -- a versioned digest of the projected anchor-free logical payload and a - separate digest of the exact serialized bytes at the first send boundary; -- selected account intent; -- whether an anchor-free projected request is self-contained and account - neutral; -- generation, random nonce, timestamps, and the stable task-authority digest. - -The response is a stable non-retryable -`previous_response_recovery_authorization_required` error. An existing record -is never overwritten by a different contract. - -## Admin acknowledgement - -List/challenge/approve routes require a nonblank operator identity supplied by -the configured trusted-proxy dashboard mode; DISABLED, standard password, -proxy bearer, spoofed/untrusted header, and guest identities cannot authorize -recovery. An approval request -must present the exact generation and one-time challenge plus the literal -acknowledgement `operator_acknowledged_semantic_rebase`. - -The server canonicalizes the client-owned, content-free receipt and verifies -its declared SHA-256. The receipt binds remote session JSONL SHA-256, byte -size, exact last read offset, task and session identity, stable -task-authority/strong-session hashes, a full-checkpoint ledger digest, and zero -unresolved calls. The full-checkpoint ledger is a distinct domain from the -compacted request-input direct-call ledger; equality between those two digests -is neither expected nor used. Approval records only receipt and identity -digests, never the JSONL or conversation content. - -The receipt contains two explicitly different evidence domains. The JSONL -SHA/size/offset, task identity, full-checkpoint ledger, and composite -`unresolved_count=0` are independent client checkpoint evidence. The four -captured-request count/input/contract/retained-ledger values carry literal -`captured_request_binding_provenance=server_challenge`: the trusted operator -copies and explicitly acknowledges the content-free server challenge, and the -transaction compares it back to the unchanged captured row. These four -values are not described as independently client-verified physical evidence. -Two additional server-challenge values bind the versioned projected logical -payload and the exact anchor-free projection of the first-attempt -post-installation-metadata wire bytes. Approval surviving a deploy cannot -dispatch if projection semantics or any forwarded field changed since -capture. Transformable external image URLs are not eligible for rowless -recovery because refetching could change their bytes. -A future transport-time client receipt may strengthen that provenance without -weakening this operator-acknowledged flow. - -The client full-checkpoint ledger uses domain -`qk-client-full-checkpoint-tool-ledger-v1\0`. From the complete-newline JSONL -prefix it selects ordered `response_item` call/output records, requires an -exact call/output type pairing and globally unique correlation IDs, and emits -only line/ordinal/kind/type plus domain hashes of the correlation ID, canonical -tool identity `{type,name,namespace}`, canonical arguments/input, canonical -output, and canonical payload without the raw ID. Canonical JSON uses sorted -keys, ASCII escaping, and compact separators. The overall digest hashes -`{schema:"qk_client_full_checkpoint_tool_ledger_v1",entries:[...]}` under that -domain. `unresolved_count` is the sum of pending calls, missing-ID events, -orphan outputs, duplicate call IDs, duplicate outputs, and type mismatches; -approval requires zero. Other JSONL records remain physically bound by the -file SHA/size/offset even when excluded from the ledger. - -## Dispatch and terminal settlement - -An approved request must be the exact failed turn: its complete input count and -fingerprint, non-input contract, and direct-call ledger remain exact. This -generation permits no suffix. A later ordinary user/developer follow-up is -admitted only after `response.completed` publishes the replacement anchor. -This avoids inventing an ordered-prefix proof from a single whole-input hash. -The anchor-free projected body must remain self-contained. Account selection -is pinned to the captured account. - -Before account lookup or WebSocket connection, one primary CAS transitions the -non-cascading authority from APPROVED to UNKNOWN with exact generation, -request, wire, and task-authority binding. Concurrent losers stop before -selection/connect. The winner then creates the replacement durable session -and binds its existing FK-backed UNKNOWN journal before physical send. A -crash between stages remains UNKNOWN. A locally observed setup failure before -any possible send, or two typed closed-before-send socket results, may restore -APPROVED after an exact journal delete; a zero-event disconnect never does. -Immediately after the final image/materialization and account-installation -metadata transform, but before journal binding or send, the winner hashes the -actual serialized wire again. A mismatch sends zero bytes and restores -APPROVED only through the exact proven-unsent preflight rollback; an ambiguous -rollback outcome remains UNKNOWN. This second comparison closes the account -metadata read/selection TOCTOU window. - -On `response.completed`, the same terminal persistence transaction must publish -the ordinary response anchor, aliases, complete client input count/fingerprint, -and mark the authority CONSUMED. A terminal failure consumes the dispatch but -does not authorize another attempt. This leaves PR #19's ambiguous receive -fence intact. - -## Operational limitation - -The service uses OpenAI-compatible HTTP 400 invalid-request responses, not -generic 409, for every stable no-progress continuity state. An -authorization-required response carries -`action=retry_same_turn_after_admin_approval`. The operator must regenerate -the receipt after that failed turn, approve through the trusted-proxy API, and -use the client's same-turn Retry action; a new user/developer item is rejected. diff --git a/openspec/changes/add-rowless-checkpoint-semantic-rebase/proposal.md b/openspec/changes/add-rowless-checkpoint-semantic-rebase/proposal.md deleted file mode 100644 index 8483cc4136..0000000000 --- a/openspec/changes/add-rowless-checkpoint-semantic-rebase/proposal.md +++ /dev/null @@ -1,62 +0,0 @@ -# Proposal: add rowless checkpoint semantic rebase - -## Problem - -An upstream can explicitly reject a client-supplied `previous_response_id` -before emitting any response event after the HTTP bridge session row that used -to hold the completed checkpoint has already been purged. The service then -has no durable stored-prefix or pending-call manifest with which to prove an -automatic unanchored replay. Repeating the request only repeats the rejected -anchor, while deleting the client thread loses task identity and context. - -Logs are not continuity authority and the purged row cannot be reconstructed. -The safe recovery is therefore an explicit, dashboard-admin-authorized -**semantic rebase** of the same client thread, not an automatic replay and not -proof that an unknown historical side effect did or did not occur. - -## Change - -- Persist a content-free recovery authority outside the - `http_bridge_sessions` foreign-key graph when an explicit client anchor is - rejected with zero response events. The authority is keyed by API-key - scope, a strong session hash, and the rejected-anchor hash. -- Capture only canonical input count/fingerprint, non-input Responses contract - fingerprint, ordered direct-call closure digest and unresolved count, - projected-logical and exact post-transform wire digests, selected-account - intent, generation/nonce, and eligibility flags. Never persist request - content or the plaintext anchor. Requests with externally refetched image - content are ineligible. -- Expose authenticated dashboard-admin list/challenge/approve endpoints. - Approval binds one exact generation to a client-owned checkpoint receipt: - authoritative session JSONL SHA-256/size/last offset, task/session identity, - ordered tool ledger digest, and `unresolved_count=0`. -- Admit one same-thread retry only when its complete input and non-input - contract exactly match the captured failed turn. This recovery generation - admits no suffix; ordinary follow-ups resume only after the retry publishes - a new anchor. Before account selection or WebSocket connect, atomically - claim the approved non-cascading generation. Bind the existing - replacement-session UNKNOWN journal after durable replacement creation and - before physical send. -- Recompute the exact serialized wire after the final account/image transform; - a mismatch sends no bytes and can restore approval only through the - physically-proven pre-send rollback. -- A proven pre-send failure restores APPROVED. Any ambiguous post-send result - remains UNKNOWN permanently. `response.completed` publishes the ordinary - durable session/aliases/new anchor/full checkpoint and consumes the semantic - rebase generation. - -## Safety classification - -The operator acknowledgement states that the client-owned checkpoint is the -authority for continuing the task. It does **not** assert that purged history -was recovered or that unknown old side effects were absent. The one approved -dispatch is at-most-once within this generation; PR #19's ambiguous-receive -no-replay contract remains unchanged. - -## Non-goals - -- Reconstructing request content or authority from logs. -- Automatically authorizing a rowless recovery. -- Authorizing any account-scoped recovery; only account-neutral requests are eligible. -- Treating a new task/thread as recovery. -- Retrying an UNKNOWN dispatch. diff --git a/openspec/changes/add-rowless-checkpoint-semantic-rebase/specs/responses-api-compat/spec.md b/openspec/changes/add-rowless-checkpoint-semantic-rebase/specs/responses-api-compat/spec.md deleted file mode 100644 index 1dfe598a8a..0000000000 --- a/openspec/changes/add-rowless-checkpoint-semantic-rebase/specs/responses-api-compat/spec.md +++ /dev/null @@ -1,84 +0,0 @@ -# Delta: Responses API compatibility - -## ADDED Requirements - -### Requirement: Rowless stale-anchor recovery is an explicit semantic rebase - -When upstream explicitly rejects a client-supplied `previous_response_id` -before emitting any response event and the durable session checkpoint needed -for automatic proof is absent, the service MUST persist a content-free, -non-cascading recovery authority and MUST return a stable non-retryable -authorization-required error. It MUST NOT retry the request unanchored, infer -authority from logs, or create a different logical thread. - -#### Scenario: Capture survives session cleanup - -- **GIVEN** a hard same-thread request with an explicit stale anchor -- **AND** upstream rejects it with zero response events -- **WHEN** the associated bridge session is absent or later purged -- **THEN** a tombstone keyed by API-key scope, stable task-authority hash, and anchor - hash remains -- **AND** it contains no request content or plaintext anchor -- **AND** the same contract is rejected locally before account selection or - WebSocket connect until an administrator approves its exact generation. - -#### Scenario: Only a dashboard admin can approve - -- **GIVEN** a captured generation -- **WHEN** a proxy bearer, dashboard guest, stale challenge, mismatched - generation, or malformed client-owned receipt attempts approval -- **THEN** no state changes -- **AND** only an authenticated trusted-proxy dashboard operator with the exact challenge and - `operator_acknowledged_semantic_rebase` acknowledgement can approve it. - -#### Scenario: Receipt and request are exact - -- **GIVEN** an approved generation bound to a client-owned checkpoint receipt -- **WHEN** the next same-thread request arrives -- **THEN** its complete captured input, non-input contract, ordered direct-call - ledger, account intent, task/session identity binding, and zero unresolved - calls MUST match -- **AND** this recovery generation MUST reject every suffix or input mutation -- **AND** the versioned projected logical payload and exact post-transform - serialized wire MUST match the captured challenge before dispatch -- **AND** a late account-metadata or other wire drift MUST stop before physical - send and may restore approval only with exact proven-unsent authority -- **AND** later ordinary follow-ups MUST wait for the replacement anchor -- **AND** the projected anchor-free request MUST be self-contained and account - neutral; account-scoped requests are never eligible. - -#### Scenario: One at-most-once dispatch - -- **GIVEN** an approved exact request -- **WHEN** concurrent workers attempt it -- **THEN** exactly one worker transitions the non-cascading authority to - UNKNOWN before account selection or WebSocket connect -- **AND** the winner binds the replacement-session journal before physical send -- **AND** a physically proven pre-send failure may restore APPROVED -- **BUT** an ambiguous post-send result remains UNKNOWN and is never replayed. - -#### Scenario: Completion publishes a normal checkpoint - -- **GIVEN** the one semantic-rebase dispatch reaches `response.completed` -- **WHEN** terminal durable settlement commits -- **THEN** the replacement bridge session, aliases, new response anchor, full - client input count/fingerprint, journal settlement, and CONSUMED authority - are published atomically -- **AND** later requests use ordinary continuity rather than the tombstone. - -#### Scenario: Stable client and retention behavior - -- **GIVEN** an authorization-required, recovery-marker, UNKNOWN, or already - CONSUMED rowless state -- **WHEN** the signed Desktop repeats the same contract -- **THEN** the service returns HTTP 400 with a stable machine code/action and - does not enter the generic reconnect/replay path -- **AND** omitting or changing the anchor cannot bypass an exact stable-task - tombstone match for the captured request domain -- **AND** a unique exact APPROVED match still uses the same one-generation CAS, - while a genuinely changed turn must use ordinary replacement-anchor continuity -- **AND** expired unapproved CAPTURED rows may be purged after seven days -- **BUT** APPROVED, UNKNOWN, and CONSUMED identity fences are never deleted by - automatic bridge cleanup. -- **AND** any non-CAPTURED fence makes a pre-rowless application image an - invalid rollback target, exposed through the trusted-operator status API. diff --git a/openspec/changes/add-rowless-checkpoint-semantic-rebase/tasks.md b/openspec/changes/add-rowless-checkpoint-semantic-rebase/tasks.md deleted file mode 100644 index 65dae6599b..0000000000 --- a/openspec/changes/add-rowless-checkpoint-semantic-rebase/tasks.md +++ /dev/null @@ -1,13 +0,0 @@ -# Tasks - -- [x] Add non-cascading authority schema, migration, repository, and lifecycle tests. -- [x] Add content-free input/contract/direct-call ledger fingerprints and fail-closed matching tests. -- [x] Capture zero-event explicit stale-anchor rejects and locally stop repeated same-contract sends. -- [x] Add trusted-proxy dashboard list/challenge/approve API and content-free operator response contract. -- [x] Bind approval to the exact client-owned checkpoint receipt and account-neutral dispatch intent. -- [x] Claim one generation before selection/connect, bind UNKNOWN journal before send, and restore only - physically proven unsent attempts. -- [x] Atomically publish the new durable checkpoint and consume the authority on completion. -- [x] Cover three sanitized production structures, concurrent claims, restart/cleanup, all drift cases, - PR #19 ambiguous predecessor, and irreversible-effect count. -- [x] Run strict OpenSpec validation, focused unit/integration tests, architecture, Ruff/format, and ty. diff --git a/openspec/changes/recover-proxied-websocket-early-close/.openspec.yaml b/openspec/changes/add-telemetry-optout-signal/.openspec.yaml similarity index 50% rename from openspec/changes/recover-proxied-websocket-early-close/.openspec.yaml rename to openspec/changes/add-telemetry-optout-signal/.openspec.yaml index 44f55ffeea..f774115be7 100644 --- a/openspec/changes/recover-proxied-websocket-early-close/.openspec.yaml +++ b/openspec/changes/add-telemetry-optout-signal/.openspec.yaml @@ -1,2 +1,2 @@ schema: spec-driven -created: 2026-08-23 +created: 2026-08-20 diff --git a/openspec/changes/add-telemetry-optout-signal/design.md b/openspec/changes/add-telemetry-optout-signal/design.md new file mode 100644 index 0000000000..a3befa3143 --- /dev/null +++ b/openspec/changes/add-telemetry-optout-signal/design.md @@ -0,0 +1,85 @@ +## Context + +Snapshot transmission currently resolves consent before building the payload and uses an +instance identity to register, activate, sign, and post through an isolated HTTP client. The +settings PUT persists dashboard decisions through a request-scoped database session. See +`proposal.md` for motivation and `specs/telemetry/spec.md` for the changed contract. + +The opt-out signal is unusual because its triggering decision has already made consent inactive. +It therefore cannot depend on the normal consent-gated sender context, and its asynchronous work +cannot retain request-scoped database or HTTP resources. + +## Goals / Non-Goals + +**Goals:** + +- Preserve one authoritative consent resolution for each snapshot and preview. +- Emit an opt-out only from a dashboard-driven effective active-to-inactive transition. +- Keep the settings response independent from all collector network activity. +- Preserve canonical serialization, signing, bounded retries, and debug-only failure handling. + +**Non-Goals:** + +- No new setting, scheduler cadence, collector implementation, or database migration. +- No historical opt-out backfill or notification when the environment kill switch disables + telemetry. +- No delivery guarantee beyond the existing bounded best-effort telemetry discipline. + +## Decisions + +### Pass resolved consent into snapshot construction + +The scheduler and preview API will pass their already-resolved active consent state into the +snapshot builder. This keeps the envelope deterministic and prevents a second resolution from +observing a different state. Resolving consent again inside the builder was rejected because it +would duplicate policy and could disagree with the caller's send decision. + +### Detect the effective transition around persistence + +The PUT handler will resolve consent before persisting the decision and again afterward. It will +schedule an opt-out only when the first resolution is active and the second is inactive. This +naturally excludes disabled-to-disabled writes and both environment override values, while +allowing a later re-enable/re-disable cycle to produce a new event. Comparing only persisted +values was rejected because it would incorrectly notify while an environment override controls +effective behavior. + +### Give the background task explicit immutable inputs + +Before scheduling, the handler will obtain the instance identity and gather the version and +platform fields needed for registration and activation. The sender will then open and close its +own HTTP client, lazily register and activate, and post the signed canonical opt-out body. A +module-owned task set will retain a strong reference until completion. Reusing the request's +database session or the consent-gated sender context was rejected because those resources and +policy no longer match the task's lifetime. + +### Reuse the sender's bounded delivery discipline + +Opt-out delivery will share the snapshot path's five-second total timeout, at-most-one retry, +canonical JSON, signing, accepted-status handling, and debug-only exception isolation. The +event body is: + +```json +{"app_version":"1.2.3","event":"optout","instance_id":"550e8400-e29b-41d4-a716-446655440000","occurred_at":"2026-08-20T12:00:00+00:00"} +``` + +The route order for an uninitialized process is registration, activation, then opt-out. Waiting +for network completion in the API handler was rejected because collector latency must not affect +the operator's settings response. + +## Risks / Trade-offs + +- **Process exit can cancel the best-effort task** → Server-side idempotency and per-transition + scheduling make retries safe, while avoiding shutdown delay or API coupling. +- **Concurrent duplicate dashboard requests can each observe a transition** → Persisted state + serialization and transition tests constrain ordinary request behavior; the collector remains + idempotent for rare delivery duplication. +- **Registration or activation outage prevents the event** → The sender swallows the bounded + failure at debug level, matching snapshot availability and privacy behavior. +- **A future snapshot caller could pass disabled consent** → The type narrows the payload to the + two wire-valid states, and callers only build while resolved consent is active. + +## Migration Plan + +Deploy the additive client contract together with collector support for `/v1/optout`. Existing +instances require no data migration. Rollback removes the new snapshot field and notification; +the collector's idempotent endpoint can remain deployed without affecting older clients. diff --git a/openspec/changes/add-telemetry-optout-signal/proposal.md b/openspec/changes/add-telemetry-optout-signal/proposal.md new file mode 100644 index 0000000000..012e9e63a2 --- /dev/null +++ b/openspec/changes/add-telemetry-optout-signal/proposal.md @@ -0,0 +1,34 @@ +## Why + +Telemetry currently becomes silent when an operator disables it, so the collector cannot +distinguish an explicit rejection from an instance that stopped running. The wire contract also +needs the effective persisted consent state on snapshots so aggregate interpretation stays +accurate without weakening the environment kill switch. + +## What Changes + +- Add the active consent state (`undecided` or `enabled`) to every snapshot payload. +- Send one final signed opt-out notification for each dashboard-driven transition from active + telemetry to inactive telemetry. +- Preserve absolute silence for the `CODEX_LB_TELEMETRY_ENABLED=false` environment path and + isolate opt-out transmission failures from the settings API response. +- Explain the additive wire behavior in the telemetry settings UI and published telemetry docs. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `telemetry`: Extend the outbound allowlist and consent behavior with snapshot consent and the + decision-time opt-out notification. + +## Impact + +- Telemetry schemas, snapshot construction, scheduler and preview callers, sender, and settings + API transition handling. +- Telemetry unit tests and wire-contract allowlists. +- Dashboard telemetry consent copy and component tests. +- Published telemetry payload documentation. diff --git a/openspec/changes/add-telemetry-optout-signal/specs/telemetry/spec.md b/openspec/changes/add-telemetry-optout-signal/specs/telemetry/spec.md new file mode 100644 index 0000000000..eac1ade3e8 --- /dev/null +++ b/openspec/changes/add-telemetry-optout-signal/specs/telemetry/spec.md @@ -0,0 +1,66 @@ +## ADDED Requirements + +### Requirement: Snapshot payload declares active consent + +Every snapshot payload MUST include a top-level `consent` field whose value is the resolved +persisted consent state `undecided` or `enabled`; `disabled` MUST NOT appear because snapshots +are not transmitted while telemetry is inactive. + +#### Scenario: Undecided snapshot declares consent + +- **WHEN** telemetry is active under the default undecided consent state +- **THEN** the transmitted snapshot and exact payload preview contain `consent: "undecided"` + +#### Scenario: Enabled snapshot declares consent + +- **WHEN** telemetry is active under persisted enabled consent +- **THEN** the transmitted snapshot and exact payload preview contain `consent: "enabled"` + +### Requirement: Dashboard opt-out notification + +The service MUST send one final signed `POST /v1/optout` notification for each +dashboard-driven effective consent transition from active to inactive, and MUST complete any +required instance registration and activation before sending that notification. The notification +MUST use the telemetry instance identity and snapshot signing scheme, MUST be isolated from the +settings API response, and MUST NOT be sent for an environment-controlled consent path. + +#### Scenario: Opt-out fires exactly once per transition + +- **WHEN** dashboard consent transitions from undecided or enabled active telemetry to disabled + inactive telemetry without an environment override +- **THEN** exactly one opt-out notification is attempted for that transition before telemetry + becomes silent + +#### Scenario: Environment kill switch stays silent + +- **WHEN** `CODEX_LB_TELEMETRY_ENABLED=false` makes telemetry inactive or a dashboard decision + is persisted while consent is controlled by either environment override value +- **THEN** no opt-out notification or other telemetry network request is attempted + +#### Scenario: Opt-out failure is isolated + +- **WHEN** registration, activation, or opt-out transmission fails +- **THEN** the failure uses a total timeout of no more than five seconds, retries no more than + once, is logged only at debug level, does not raise to the caller, and does not delay or alter + the successful settings API response + +#### Scenario: A later transition may notify again + +- **WHEN** an operator re-enables telemetry and later disables it again through the dashboard + without an environment override +- **THEN** the later active-to-inactive transition attempts exactly one new opt-out notification + +## MODIFIED Requirements + +### Requirement: Disabled means zero telemetry traffic + +Except for the single decision-time opt-out notification on a dashboard-driven active-to-inactive +transition, when resolved consent is `disabled` the service MUST NOT open any network connection +to the telemetry endpoint. The environment kill-switch path MUST NOT receive this exception and +MUST remain completely silent. + +#### Scenario: No connection attempts when disabled + +- **WHEN** telemetry is disabled and the service runs through startup and a 24-hour scheduler + cycle outside the dashboard decision-time transition +- **THEN** no connection attempt to the telemetry endpoint is made diff --git a/openspec/changes/add-telemetry-optout-signal/tasks.md b/openspec/changes/add-telemetry-optout-signal/tasks.md new file mode 100644 index 0000000000..f45d480b3c --- /dev/null +++ b/openspec/changes/add-telemetry-optout-signal/tasks.md @@ -0,0 +1,22 @@ +## 1. Snapshot Consent Contract + +- [x] 1.1 Add the active consent literal to snapshot schemas and introduce the typed opt-out event schema +- [x] 1.2 Require callers to pass resolved consent into snapshot construction and expose it in sender and preview envelopes +- [x] 1.3 Update schema allowlist and builder, scheduler, and preview regression tests for the consent field + +## 2. Opt-Out Delivery + +- [x] 2.1 Implement signed canonical opt-out delivery with lazy registration, activation, bounded retry, and debug-only failure isolation +- [x] 2.2 Detect dashboard effective active-to-inactive transitions and schedule one resource-owning background send +- [x] 2.3 Add sender and settings API tests for successful delivery, retry/failure isolation, repeated transitions, no-op decisions, and both environment overrides +- [x] 2.4 Preserve transport-level zero-call coverage for disabled scheduler and sender paths + +## 3. Operator Communication + +- [x] 3.1 Add neutral opt-out notice copy to the telemetry consent dialog and settings components with co-located test coverage +- [x] 3.2 Document the snapshot consent field, opt-out wire payload, transition behavior, and environment-path silence + +## 4. Verification + +- [x] 4.1 Validate the OpenSpec change and run focused backend and frontend tests +- [x] 4.2 Run the full unit suite, lint, and type-check gates and confirm the final diff stays within the approved scope diff --git a/openspec/changes/add-timeout-invariant-linter/notes.md b/openspec/changes/add-timeout-invariant-linter/notes.md new file mode 100644 index 0000000000..f6a735edd4 --- /dev/null +++ b/openspec/changes/add-timeout-invariant-linter/notes.md @@ -0,0 +1,55 @@ +# Timeout Invariant Linter Audit Disposition + +Inputs: `LINTER_AUDIT.md` and the 11 PR #1622 inline findings fetched with +`gh api repos/Soju06/codex-lb/pulls/1622/comments`. + +| # | Rule | Audit verdict | Bot finding | Disposition | +|---:|---|---|---|---| +| 1 | `upstream-connect-within-proxy-budget` | CIRCULAR | Wrong connect anchor | Removed; generic client clamps connect to total budget, so this was circular. | +| 2 | `upstream-connect-within-stream-budget` | GROUNDED | Wrong connect anchor family | Deferred; anchor fixed to `app/core/clients/proxy.py:2720` but not enforced by the shipped registry. | +| 3 | `upstream-connect-within-compact-budget` | CIRCULAR | None | Removed; compact passes remaining budget as an override/clamp. | +| 4 | `upstream-connect-within-bridge-budget` | CIRCULAR | None | Removed; bridge request budget does not directly own upstream connect. | +| 5 | `admission-plus-connect-within-proxy-budget` | GROUNDED | Wrong phase/circular deadline | Removed; runtime recomputes remaining absolute budget after admission. | +| 6 | `admission-plus-connect-within-compact-budget` | GROUNDED | Wrong phase/circular deadline family | Removed; compact also passes remaining deadline-derived overrides. | +| 7 | `admission-wait-within-proxy-budget` | GROUNDED | None | Kept. | +| 8 | `admission-wait-within-stream-budget` | GROUNDED | None | Kept. | +| 9 | `admission-wait-within-compact-budget` | GROUNDED | None | Kept. | +| 10 | `admission-wait-within-bridge-budget` | CIRCULAR | None | Removed; bridge admission is clamped to remaining bridge budget. | +| 11 | `stream-idle-within-stream-budget` | GROUNDED | Bot says total may precede idle | Removed; total and idle are independent aiohttp limits. | +| 12 | `stream-idle-within-bridge-budget` | GROUNDED | Bot says same phase family | Removed; outer bridge deadline may validly fire before idle. | +| 13 | `sse-keepalive-before-stream-idle` | GROUNDED | Downstream keepalive cannot reset upstream idle | Removed; it compared independent directions. | +| 14 | `sse-keepalive-within-stream-budget` | GROUNDED | None | Deferred; not enforced by the shipped registry. | +| 15 | `sse-keepalive-within-bridge-budget` | GROUNDED | None | Deferred; not enforced by the shipped registry. | +| 16 | `token-refresh-claim-covers-admission-and-exchange` | GROUNDED | None | Deferred; not enforced by the shipped registry. | +| 17 | `refresh-failure-cooldown-within-claim-ttl` | GROUNDED | Cooldown is process-local cache | Removed; cooldown does not extend claim ownership. | +| 18 | `token-refresh-exchange-within-claim-ttl` | GROUNDED | None | Deferred; not enforced by the shipped registry. | +| 19 | `usage-fetch-within-refresh-interval` | WRONG | Scheduler serializes, cadence may slip | Removed. | +| 20 | `usage-fetch-within-reset-credits-interval` | WRONG | Usage cadence family | Removed. | +| 21 | `compact-budget-within-proxy-budget` | CIRCULAR | Compact lane independent | Removed. | +| 22 | `bridge-idle-ttl-within-bridge-budget` | WRONG | Reuse TTL may exceed request budget | Removed. | +| 23 | `bridge-codex-idle-ttl-within-bridge-budget` | WRONG | Reuse TTL family | Removed. | +| 24 | `bridge-stuck-gate-retire-after-admission` | GROUNDED | Separate phase from admission | Removed; response-created acknowledgement retirement is independent from queue admission. | +| 25 | `bridge-stuck-gate-retire-within-bridge-budget` | GROUNDED | Missing hard-anchor 2x multiplier | Fixed; now compares `2 * retire_after` with bridge budget. | +| 26 | `bridge-clean-close-jitter-within-admission` | CIRCULAR | Jitter/admission independent | Removed. | +| 27 | `bridge-clean-close-jitter-within-bridge-budget` | GROUNDED | None | Deferred; no settings field anchors the clean-close jitter, so the rule has nothing to read. | +| 28 | `account-lease-ttl-covers-proxy-budget` | GROUNDED | None | Kept. | +| 29 | `account-lease-ttl-covers-compact-budget` | GROUNDED | None | Kept. | +| 30 | `model-registry-snapshot-outlives-refresh-interval` | Proposed new | None | Added; `model_registry_snapshot_max_age_seconds > _REFRESH_INTERVAL_SECONDS`. | +| 31 | `durable-bridge-retry-circuit-ttl-covers-backoff-and-half-open` | Proposed new | None | Added; retry-circuit state TTL must outlive max backoff and half-open lease. | + +Shipped registry: `TIMEOUT_INVARIANT_RULES` enforces exactly the eight rules +above marked Kept, Fixed, or Added (rows 7, 8, 9, 25, 28, 29, 30, 31), and +`tests/unit/test_timeout_invariants.py` pins that count. Rows marked Deferred +were judged grounded by the audit but ship unenforced: this validator runs at +startup and can abort the process under +`CODEX_LB_TIMEOUT_INVARIANT_VALIDATION_STRICT`, so a rule enters the registry +only once it has a settings anchor and a default configuration that satisfies +it. + +Validation scope: + +- Validated: startup `Settings` fields and imported constants used by the rule + table. +- Not validated: per-request `ContextVar` overrides, runtime clamps, derived + effective values, and database/API-key/model-source timeout inputs loaded + after startup. diff --git a/openspec/changes/add-timeout-invariant-linter/proposal.md b/openspec/changes/add-timeout-invariant-linter/proposal.md new file mode 100644 index 0000000000..b1c196e24a --- /dev/null +++ b/openspec/changes/add-timeout-invariant-linter/proposal.md @@ -0,0 +1,36 @@ +## Why + +Timeout and TTL mismatches have repeatedly caused healthy codex-lb work to be +killed by a different, shorter budget. The project needs those relationships +encoded as executable configuration policy instead of scattered comments. + +## What Changes + +- Add a declarative timeout-invariant rule table over effective Settings fields. +- Validate the effective startup configuration in non-strict mode by default, + logging CRITICAL for every violated rule. +- Add `timeout_invariant_validation_strict` for deployments that want startup to + fail on violations. +- Add a strict CI entrypoint via `python -m app.core.timeout_invariants`. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `deployment-installation`: defines startup and CI validation for timeout + invariants. +- `proxy-runtime-observability`: defines low-cardinality CRITICAL diagnostics + for timeout-invariant violations. + +## Impact + +- Code: `app/core/timeout_invariants.py`, startup lifespan settings validation, + and the Settings strict-mode flag. +- Tests: focused unit coverage for defaults, inverted config detection, strict + raise, and the CI entrypoint. +- Operators: default deployments continue to start; strict mode opts into + fail-fast behavior. diff --git a/openspec/changes/add-timeout-invariant-linter/specs/deployment-installation/spec.md b/openspec/changes/add-timeout-invariant-linter/specs/deployment-installation/spec.md new file mode 100644 index 0000000000..664283dcc8 --- /dev/null +++ b/openspec/changes/add-timeout-invariant-linter/specs/deployment-installation/spec.md @@ -0,0 +1,41 @@ +## ADDED Requirements + +### Requirement: Timeout invariants are validated at startup and in CI + +The application SHALL define executable timeout-invariant rules over effective +startup `Settings` fields and explicitly imported code constants for verified +relationships between request budgets, TTLs, refresh deadlines, admission +waits, retry jitter, fixed refresh cadence, and durable retry-circuit state. +Each rule SHALL name the compared setting, constant, or expression; the +relation; and a one-line rationale describing the runtime failure prevented. +Unverified timeout inventory entries SHALL NOT be enforced until their code +relationship is verified. + +At startup, the application SHALL validate the effective startup `Settings` +object against the rule table. This validation SHALL NOT claim coverage for +per-request `ContextVar` overrides, runtime clamps, derived effective values +computed after startup, or database/API-key/model-source timeout values loaded +after startup. By default, startup SHALL log every violation at CRITICAL and +continue. When `timeout_invariant_validation_strict` is true, startup SHALL raise +after logging the violations. The project SHALL expose a runnable CI entrypoint +that validates the same rule table, defaults to non-strict reporting, and exits +nonzero only when `--strict` is passed and any rule is violated. + +#### Scenario: Default settings satisfy timeout invariants + +- **WHEN** timeout-invariant validation runs against default settings +- **THEN** every enforced rule passes +- **AND** the CI entrypoint exits successfully + +#### Scenario: Non-strict startup reports violations without failing + +- **WHEN** effective settings violate one or more timeout-invariant rules +- **AND** strict timeout-invariant validation is disabled +- **THEN** startup validation logs every violated rule at CRITICAL +- **AND** startup may continue + +#### Scenario: Strict startup rejects violations + +- **WHEN** effective settings violate one or more timeout-invariant rules +- **AND** `timeout_invariant_validation_strict` is true +- **THEN** startup validation raises an error that includes the violated rule ids diff --git a/openspec/changes/add-timeout-invariant-linter/specs/proxy-runtime-observability/spec.md b/openspec/changes/add-timeout-invariant-linter/specs/proxy-runtime-observability/spec.md new file mode 100644 index 0000000000..d92fb631e2 --- /dev/null +++ b/openspec/changes/add-timeout-invariant-linter/specs/proxy-runtime-observability/spec.md @@ -0,0 +1,18 @@ +## ADDED Requirements + +### Requirement: Timeout-invariant violations are diagnosable + +Timeout-invariant validation diagnostics SHALL include the rule id, left-hand +setting or expression and value, relation, right-hand setting or expression and +value, rationale, and code anchors. Diagnostics SHALL avoid request payloads, +API keys, access tokens, raw affinity keys, account emails, and other +high-cardinality runtime identifiers. Diagnostics SHALL describe startup +validation of `Settings` and imported constants only, not per-request overrides, +runtime clamps, or runtime-derived effective values. + +#### Scenario: Violation log names the invariant + +- **WHEN** startup timeout-invariant validation observes a violated rule +- **THEN** the CRITICAL log includes that rule id and rationale +- **AND** the log contains no request payload, API key, access token, raw + affinity key, or account email diff --git a/openspec/changes/add-timeout-invariant-linter/tasks.md b/openspec/changes/add-timeout-invariant-linter/tasks.md new file mode 100644 index 0000000000..9d6b2fbd40 --- /dev/null +++ b/openspec/changes/add-timeout-invariant-linter/tasks.md @@ -0,0 +1,23 @@ +## 1. Timeout invariant policy + +- [x] 1.1 Verify curated timeout inequalities against current code before + encoding them. +- [x] 1.2 Add a declarative rule table with the accepted 8 verified startup + inequalities and code-anchored rationales. +- [x] 1.3 Leave unverified curated timeout entries as TODOs rather than + enforcing them. + +## 2. Runtime and CI validation + +- [x] 2.1 Validate effective settings during application startup. +- [x] 2.2 Keep startup non-strict by default and log CRITICAL for violations. +- [x] 2.3 Add strict mode that raises on violations. +- [x] 2.4 Add a runnable CI entrypoint that validates settings strictly and + exits nonzero on violations. + +## 3. Regression coverage + +- [x] 3.1 Prove defaults satisfy all enforced rules. +- [x] 3.2 Prove an inverted configuration names the specific violated rule. +- [x] 3.3 Prove strict mode raises. +- [x] 3.4 Run focused tests and OpenSpec validation before commit. diff --git a/openspec/changes/apply-context-window-override-to-v1-input-budget/proposal.md b/openspec/changes/apply-context-window-override-to-v1-input-budget/proposal.md new file mode 100644 index 0000000000..97eb37aca7 --- /dev/null +++ b/openspec/changes/apply-context-window-override-to-v1-input-budget/proposal.md @@ -0,0 +1,23 @@ +## Why + +`model_context_window_overrides` is documented as the highest-priority reported-context override, but on `/v1/models` it only reaches `metadata.context_window`. The fields generic OpenAI-compatible clients actually read — `context_length`, `contextLength`, `capabilities.context_length`, and `metadata.input_context_window` — keep reporting the un-overridden upstream `context_window`, so an operator who raises a model's window sees Codex-native clients use the wider window from `/backend-api/codex/models` while every OpenAI-compatible client silently caps itself at the old value. A single catalog then advertises two different budgets for one model. + +The split was introduced by `2026-06-02-report-v1-model-full-context` to stop over-advertising context to generic clients, because the backend really did reject inputs above its `context_window` with `context_length_exceeded`. That reasoning still holds for the *default*, but not for an explicit operator override: current frontier models publish a `max_context_window` well above `context_window` (for example `gpt-5.6-sol` at `272000` / `872000`), the backend accepts input up to that ceiling, and the Codex client itself treats a config override as the session window after clamping it to `max_context_window`. + +## What Changes + +- An explicit `model_context_window_overrides` entry is reported as the input budget too, so `context_length`, `contextLength`, `capabilities.context_length`, and `metadata.input_context_window` agree with `metadata.context_window` instead of contradicting it. +- The reported input budget is clamped to the upstream-declared `max_context_window` when upstream declares one above the backend `context_window`, so an override can never advertise more input than the backend sanctions — the same clamp the Codex client applies to `model_context_window` in `config.toml`. This preserves the original protection against over-advertising while removing the under-advertising. A `max_context_window` equal to `context_window` never clamps: bootstrap subscription models and source-catalog models synthesize that value purely so Codex clients can parse the entry, and treating it as a real ceiling would silently disable raise overrides for those models. +- The override is resolved (and clamped) once per model and that single value feeds `metadata.context_window`, every input-budget field, and the Codex-native `context_window`/`max_context_window` rewrite. An override above the ceiling therefore reports the ceiling everywhere instead of splitting one model into two budgets again (previously `metadata.context_window=1000000` next to `context_length=872000`). +- Behavior is unchanged when no override is configured for the model: the reported input budget stays the upstream `context_window`. +- `/backend-api/codex/models` is affected in two ways, both consistency-preserving: the Codex-native `models` list already reported the override on `context_window` (and rewrote `max_context_window` to match) and now uses the same clamped value, and the endpoint's OpenAI-compatible `data` alias is built from the same list-item shape as `/v1/models`, so its `context_length`-family fields pick up the corrected values too. Both views of one model advertise one budget. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `model-catalog-compat`: an operator context-window override applies to the OpenAI-compatible input-budget fields, clamped to the upstream `max_context_window` when upstream declares one above the backend `context_window`. The Codex-native catalog requirement now specifies the same single resolved value for the native `context_window`/`max_context_window` rewrite and the endpoint's OpenAI-compatible `data` alias. diff --git a/openspec/changes/apply-context-window-override-to-v1-input-budget/specs/model-catalog-compat/spec.md b/openspec/changes/apply-context-window-override-to-v1-input-budget/specs/model-catalog-compat/spec.md new file mode 100644 index 0000000000..de80363dce --- /dev/null +++ b/openspec/changes/apply-context-window-override-to-v1-input-budget/specs/model-catalog-compat/spec.md @@ -0,0 +1,89 @@ +## MODIFIED Requirements + +### Requirement: OpenAI-compatible model metadata uses backend context windows + +When serving `GET /v1/models`, the system SHALL expose `metadata.context_window` as the upstream backend `context_window` budget by default. The system MUST NOT promote raw `max_context_window` values or hard-coded full-context guesses into `metadata.context_window`. Explicit operator context-window overrides remain the highest-priority reported-context value, clamped to the upstream-declared `max_context_window` when upstream declares one above the backend `context_window`. + +#### Scenario: GPT-5 Codex models are reported with the backend context window on /v1/models + +- **WHEN** the upstream model catalog contains `gpt-5.5`, `gpt-5.4-mini`, `gpt-5.3-codex`, or `gpt-5.4` with `context_window=272000` +- **THEN** `GET /v1/models` returns each entry with `metadata.context_window=272000` + +#### Scenario: raw max_context_window does not inflate /v1/models context_window + +- **WHEN** the upstream model catalog contains a model with `context_window=272000` and `max_context_window=900000` +- **THEN** `GET /v1/models` returns that entry with `metadata.context_window=272000` + +### Requirement: OpenAI-compatible model metadata preserves the backend input budget explicitly + +When serving `GET /v1/models`, the system SHALL expose the upstream backend input/context budget in `metadata.input_context_window`. When an explicit operator context-window override applies to a model, that override SHALL be the reported input budget as well, clamped to the upstream-declared `max_context_window` when upstream declares one above the backend `context_window`, so `metadata.input_context_window` and the OpenAI-compatible `context_length`, `contextLength`, and `capabilities.context_length` fields never contradict `metadata.context_window` and never advertise more input than the backend sanctions. A `max_context_window` equal to the backend `context_window` — the parseability default synthesized for bootstrap and source-catalog models — MUST NOT clamp an override, so raise overrides for those models keep working. For models whose reported `metadata.context_window` is not operator-overridden, `metadata.context_window` and `metadata.input_context_window` SHOULD be equal. The system SHOULD expose `metadata.max_output_tokens` for known GPT-5 Codex models when that output-budget value is known; that value MUST NOT be used to inflate `metadata.context_window`. + +#### Scenario: /v1/models exposes the 272k backend input budget explicitly + +- **WHEN** the upstream model catalog contains a known GPT-5 Codex model with `context_window=272000` +- **THEN** `GET /v1/models` returns that model with `metadata.input_context_window=272000` +- **AND** `metadata.context_window=272000` + +#### Scenario: Explicit reported-context overrides do not hide the backend input budget + +- **WHEN** an operator override sets a model's reported `metadata.context_window` to `515000` +- **AND** the upstream model catalog contains that model with `context_window=272000` and no `max_context_window` +- **THEN** `GET /v1/models` returns that model with `metadata.context_window=515000` +- **AND** `metadata.input_context_window=515000` +- **AND** `context_length`, `contextLength`, and `capabilities.context_length` of `515000` + +#### Scenario: An override never advertises more input than the backend ceiling + +- **WHEN** an operator override sets a model's reported context window to `1000000` +- **AND** the upstream model catalog contains that model with `context_window=272000` and `max_context_window=872000` +- **THEN** `GET /v1/models` returns that model with `metadata.context_window=872000` +- **AND** `metadata.input_context_window=872000` +- **AND** `context_length`, `contextLength`, and `capabilities.context_length` of `872000` + +#### Scenario: A synthesized ceiling equal to the backend budget does not clamp an override + +- **WHEN** an operator override sets a source-catalog model's reported context window to `32768` +- **AND** that model declares `context_window=8192` and no explicit `max_context_window`, so the catalog synthesizes `max_context_window=8192` +- **THEN** `GET /v1/models` returns that model with `metadata.context_window=32768` +- **AND** `metadata.input_context_window=32768` +- **AND** `context_length`, `contextLength`, and `capabilities.context_length` of `32768` + +#### Scenario: /v1/models exposes max output budget for known GPT-5 Codex models + +- **WHEN** `GET /v1/models` returns `gpt-5.5`, `gpt-5.4`, `gpt-5.4-mini`, or `gpt-5.3-codex` +- **THEN** the entry's metadata includes `max_output_tokens=128000` + +### Requirement: Codex-native model catalog keeps backend catalog fields + +When serving `GET /backend-api/codex/models`, the system MUST keep Codex-native model catalog semantics unchanged: the top-level `context_window` field remains the backend compact/input budget unless an explicit operator override applies, and upstream raw fields such as `max_context_window` remain available when upstream provides them. The `/v1/models` compatibility metadata MUST NOT mutate the native Codex endpoint. + +When an explicit operator context-window override applies to a model, the native entry SHALL report the single resolved value — the override clamped to the upstream-declared `max_context_window` when upstream declares one above the backend `context_window`; a `max_context_window` equal to the backend `context_window` (the synthesized parseability default) MUST NOT clamp — on `context_window`, and SHALL rewrite `max_context_window` to that same resolved value when upstream provides the field. The endpoint's OpenAI-compatible `data` alias SHALL report the same resolved value on its `context_length`, `contextLength`, `capabilities.context_length`, `metadata.context_window`, and `metadata.input_context_window` fields, so the native and alias views of one model never advertise different budgets. + +#### Scenario: Native Codex route preserves compact budget + +- **WHEN** the upstream model catalog contains `gpt-5.5` with `context_window=272000` +- **THEN** `GET /backend-api/codex/models` returns `gpt-5.5.context_window=272000` +- **AND** it does not replace that field with `400000` + +#### Scenario: Codex model catalog also exposes OpenAI data alias + +- **WHEN** a client requests `GET /backend-api/codex/models` +- **THEN** the response keeps the Codex-native `models` list +- **AND** the response includes `object: "list"` and an OpenAI-compatible `data` list +- **AND** `data` contains model entries whose Codex visibility is `list` +- **AND** `data` excludes entries whose Codex visibility is `hide` + +#### Scenario: Native Codex catalog reports one resolved budget for a clamped override + +- **WHEN** an operator override sets a model's reported context window to `1000000` +- **AND** the upstream model catalog contains that model with `context_window=272000` and `max_context_window=872000` +- **THEN** `GET /backend-api/codex/models` returns that model with `context_window=872000` +- **AND** `max_context_window=872000` + +#### Scenario: Codex data alias reports the resolved input budget for an override + +- **WHEN** an operator override sets a model's reported context window to `515000` +- **AND** the upstream model catalog contains that model with `context_window=272000` and no explicit `max_context_window` +- **THEN** the `GET /backend-api/codex/models` `data` alias entry for that model reports `context_length`, `contextLength`, and `capabilities.context_length` of `515000` +- **AND** `metadata.context_window=515000` and `metadata.input_context_window=515000` +- **AND** the native `models` entry reports `context_window=515000` diff --git a/openspec/changes/apply-context-window-override-to-v1-input-budget/tasks.md b/openspec/changes/apply-context-window-override-to-v1-input-budget/tasks.md new file mode 100644 index 0000000000..0420712bbc --- /dev/null +++ b/openspec/changes/apply-context-window-override-to-v1-input-budget/tasks.md @@ -0,0 +1,13 @@ +## 1. Report the override on the input-budget fields + +- [x] 1.1 Resolve the `/v1/models` input context window from `model_context_window_overrides` when the model has an entry, falling back to the upstream `context_window` otherwise +- [x] 1.2 Clamp an override to the upstream-declared `max_context_window` when upstream declares one above the backend `context_window`, so the reported input budget never exceeds the backend ceiling; never treat the synthesized `max_context_window == context_window` parseability default (bootstrap and source-catalog models) as a ceiling +- [x] 1.3 Resolve the override once per model into a single clamped value shared by `metadata.context_window`, the input-budget fields, and the Codex-native `context_window`/`max_context_window` rewrite, so no field pair can disagree + +## 2. Tests + +- [x] 2.1 With an override configured, `/v1/models` reports it on `metadata.input_context_window`, `capabilities.context_length`, `contextLength`, and `context_length` (was the un-overridden upstream window) +- [x] 2.2 An override above the upstream `max_context_window` is reported clamped to that ceiling on every field, including `metadata.context_window` and the Codex-native `context_window`/`max_context_window` +- [x] 2.3 Without an override the reported input budget stays the upstream `context_window` +- [x] 2.4 The `/backend-api/codex/models` OpenAI-compatible `data` alias reports the override on its `context_length`-family fields (pin: it shares the `/v1/models` list-item shape) +- [x] 2.5 Route-level source-model regression: a raise override on a source-catalog model (synthesized `max_context_window == context_window`) applies unclamped on `/v1/models` diff --git a/openspec/changes/fix-weekly-primary-placeholder-race/.openspec.yaml b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/.openspec.yaml similarity index 100% rename from openspec/changes/fix-weekly-primary-placeholder-race/.openspec.yaml rename to openspec/changes/archive/2026-08-10-recover-repeated-clean-close/.openspec.yaml diff --git a/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/design.md b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/design.md new file mode 100644 index 0000000000..3922d38c33 --- /dev/null +++ b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/design.md @@ -0,0 +1,116 @@ +## Context + +The HTTP Responses bridge multiplexes downstream requests over a reusable +upstream WebSocket. Recovery can be initiated either by the upstream reader or +by the downstream HTTP stream watchdog, so socket replacement, reader +ownership, pending-request settlement, and retry-circuit accounting cross +several asynchronous lifecycle boundaries. See `proposal.md` for motivation +and `specs/responses-api-compat/spec.md` for the normative contract. + +Hard-affinity retry circuits are durable across replicas. Their evidence must +therefore describe a client-affecting request lifecycle, not merely a socket +lifecycle event, because idle socket retirement is normal bridge maintenance. + +## Goals / Non-Goals + +**Goals:** + +- Transfer reader ownership atomically when a downstream watchdog replaces the + upstream socket. +- Bound pre-visible recovery so it completes before the downstream client + deadline without permitting duplicate visible work. +- Count only request-affecting, pre-response bridge failures toward the durable + hard-key circuit. +- Preserve circuit state across replicas while bounding process-local and + durable stale state. + +**Non-Goals:** + +- Replay work after any response event has become visible. +- Replay delivery-ambiguous liveness failures or continuity-sensitive payloads. +- Suppress a cooldown after two genuine consecutive eventless request + failures. +- Change the Codex client's WebSocket-to-HTTP fallback policy. + +## Decisions + +### Treat the reader and socket as one generation + +When recovery originates outside the reader, the bridge cancels and awaits the +old reader before locally closing its socket, keeps the shared session live +during replacement, and starts exactly one reader for the new socket. The old +reader's finalizer is generation-guarded so it cannot retire pending work that +has moved to the replacement. + +Allowing old and new readers to overlap was rejected because a local close can +wake the old reader after the pending deque has already been transferred. A +simple `closed` flag was also rejected because it cannot distinguish the +superseded socket generation from the shared session lifetime. + +### Keep pre-visible replay bounded and ahead of the client deadline + +The bridge permits one additional clean-close replay only after the existing +first replay, only before any response event, and with bounded jitter. Silent +pre-response recovery starts after no more than six default ten-second +keepalive intervals, leaving headroom before a 120-second client deadline. + +An unbounded reconnect loop was rejected because it can duplicate requests, +hide deterministic input rejection, and outlive the downstream caller. + +### Derive circuit evidence from an owned request lifecycle + +Retirement advances the circuit only when the retiring session still owns a +pending request and that lifecycle has observed zero response events. The +eligibility snapshot is taken while lifecycle ownership is known; an idle +session with no pending request remains visible in diagnostics but is neutral +to the circuit. A request that emitted any event is excluded because the +pre-response circuit cannot safely characterize a midstream failure. + +Counting every socket retirement was rejected because routine idle churn +creates phantom first strikes. Counting only error labels was rejected because +the same transport label can describe idle maintenance, pre-response failure, +or midstream loss. + +### Persist hard-key circuits and merge conservatively + +Circuit rows are scoped by hard-affinity kind, key, and API-key scope. Conflict +updates cannot shorten an existing cooldown, retry decisions refresh durable +state, success clears state, and stale local/durable entries expire. Durable +lookup failures degrade to local state with diagnostics rather than failing the +request. + +Process-local-only state was rejected because another replica could continue +replay during an open cooldown. Treating persistence failure as terminal was +rejected because the circuit is protective metadata, not request continuity +state. + +### Judge stuck gates from upstream activity + +The watchdog uses elapsed upstream inactivity plus the absence of a response +identifier or `response.created` latency. A prior continuity anchor receives a +bounded second threshold, not an indefinite exemption. Admission flags alone +were rejected because they can remain ambiguous while the upstream socket is +silent. + +## Risks / Trade-offs + +- [A replacement is also silent] -> The extra replay remains hard-capped and + the request reaches terminal or circuit handling. +- [Reader cancellation races with pruning] -> Session handoff state keeps the + shared lifecycle live until replacement ownership is established. +- [Concurrent replicas record failures] -> Durable merge semantics preserve + the longest applicable cooldown. +- [A genuine failure occurs after an idle close] -> The idle close contributes + no strike, so the genuine failure is correctly treated as the first one. +- [Database ancestry was stamped before a merge edge existed] -> A separate + forward-only repair reconnects the request-usage rollup history without + rewriting deployed migrations. + +## Migration Plan + +Apply the forward-only database revisions, deploy the revision-labelled image, +and verify bridge create/reuse, timeout, and retry-circuit diagnostics. Health +verification must confirm the expected image revision and current schema. +Rollback is an image replacement; the prior version can ignore the additional +runtime behavior while the durable circuit table and repair revision remain +forward-compatible. diff --git a/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/proposal.md b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/proposal.md new file mode 100644 index 0000000000..45ecce0254 --- /dev/null +++ b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/proposal.md @@ -0,0 +1,57 @@ +## Why + +The HTTP Responses bridge currently opens its retry circuit after a clean +upstream WebSocket close even when the replacement socket also closes before +producing any response event. A downstream idle-recovery task can also replace +the upstream socket without restarting its reader. Closing the old socket then +wakes that stale reader, which misclassifies the proxy-initiated close as an +upstream failure and retires work already moved to the replacement socket. +Together these behaviors make a transient handoff issue visible as a reconnect +loop and require the Codex client to be restarted. + +Post-deploy evidence exposed a related accounting gap: retiring an idle bridge +with no pending request still records a retry-circuit failure. The next real +pre-response timeout can therefore open the repeated-failure cooldown after +only one client-affecting failure. + +## What Changes + +- Permit one additional pre-visible replay when the replacement upstream + WebSocket closes cleanly before any response event. +- Add bounded, configurable jitter before that additional replay to avoid + synchronized reconnects. +- Emit a dedicated diagnostic event for the additional clean-close replay. +- Keep the allowance hard-capped at one and preserve all existing no-replay + behavior after downstream-visible output or continuity-sensitive state. +- When recovery is initiated outside the upstream reader, cancel and await the + old reader before closing its socket, then start exactly one reader for the + replacement socket. +- Keep the shared session live while the replacement socket opens so concurrent + idle pruning cannot evict and fail its pending response during the handoff. +- Start silent pre-response recovery with enough headroom to reconnect before + the downstream client's request timeout boundary. +- Do not let a proxy-initiated close of a superseded socket retire pending work + on the replacement socket or increment the retry circuit. +- Detect a stuck pre-response gate from the absence of upstream activity and + response creation, rather than admission flags alone. Give requests with a + prior continuity anchor a bounded two-threshold grace period, and emit + diagnostic state when the watchdog skips a candidate. +- Count retirement failures only when the bridge still owns a pending request + that has not emitted a response event; idle no-pending closes remain visible + in lifecycle diagnostics but do not consume retry-circuit strikes. + +## Impact + +- Repeated clean handoffs can recover transparently without an immediate + terminal circuit-open response. +- The retry remains bounded and does not create an unbounded replay loop. +- Reader ownership follows the active socket across idle recovery, preventing + locally generated close frames from being counted as upstream instability. +- Idle upstream connection churn no longer turns one later request timeout into + an immediate sixty-second hard-key cooldown. +- Adds the `http_bridge_retry_circuits` durable table and migration so retry + cooldown state survives cross-replica clean-close and incomplete-stream + failures. +- Adds a forward-only request-usage rollup repair migration for deployments + already stamped at the previous merge head, so changing migration ancestry + cannot leave startup schema-drift checks failing. diff --git a/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..61b83d8285 --- /dev/null +++ b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/specs/responses-api-compat/spec.md @@ -0,0 +1,254 @@ +## MODIFIED Requirements + +### Requirement: Clean upstream close before any response event fails fast + +When the HTTP Responses bridge observes an upstream WebSocket close with +`close_code = 1000` before any `response.*` event has been surfaced for the +pending request, the proxy MUST preserve its existing pre-visible replay +guards. If the request has already used exactly one eligible pre-visible +replay and the replacement upstream WebSocket also closes cleanly before any +response event, the proxy MAY perform exactly one additional replay. The +additional replay MUST be hard-capped at one per request, and the configured +maximum MUST NOT raise that cap. + +The proxy MUST NOT replay after downstream-visible output, after a terminal +response event, or when continuity-sensitive request state makes replay unsafe. +Before the additional replay, the proxy MAY sleep for bounded configured +jitter. The proxy MUST emit a dedicated low-cardinality diagnostic event for +the additional replay. + +When a downstream HTTP stream task initiates pre-response recovery while the +upstream reader is blocked on the superseded socket, the proxy MUST cancel and +await that reader before locally closing the socket. It MUST then start exactly +one reader for the replacement socket. A close caused by replacing the socket +MUST NOT be recorded as an upstream clean-close failure, MUST NOT increment the +retry circuit, and MUST NOT retire pending work moved to the replacement. The +cancelled reader's socket-generation finalizer MUST NOT leave the shared session +marked closed while the replacement socket is being selected or opened, so idle +pruning MUST NOT evict the handoff in progress. + +The default pre-response idle-recovery window MUST leave bounded headroom +before the downstream client's request timeout. With the default ten-second +keepalive interval, the proxy MUST initiate eligible recovery after no more +than six silent intervals so replacement connection and first output can occur +before a 120-second client deadline. + +The stuck pre-response watchdog MUST judge staleness using elapsed time since +the last upstream activity and the absence of a response identifier or +`response.created` latency, not admission flags alone. A request with a prior +continuity anchor MUST receive at most two retire-thresholds of grace before +being considered stale. When the watchdog skips a candidate, it MUST emit a +low-cardinality diagnostic containing the session-closed state, candidate +count, and pending-state verdicts. + +#### Scenario: clean close before response.created is not retried + +- **WHEN** the initial upstream HTTP responses bridge closes with `close_code = 1000` before any `response.*` event for the pending request +- **THEN** the proxy returns HTTP 502 with `error.code = "upstream_rejected_input"` +- **AND** does not transparently replay the pre-created request + +#### Scenario: clean close before response output receives one bounded additional replay + +- **GIVEN** an HTTP bridge request has no surfaced `response.*` events +- **AND** its first pre-visible replay has already been used +- **WHEN** the replacement upstream WebSocket closes with code `1000` +- **THEN** the proxy performs one additional pre-visible replay +- **AND** the request replay count increases by one +- **AND** the proxy emits a `retry_precreated_clean_close` diagnostic event + +#### Scenario: repeated clean closes do not create an unbounded replay loop + +- **GIVEN** the additional clean-close replay has already been used +- **WHEN** another upstream WebSocket closes cleanly before response output +- **THEN** the proxy does not replay the request again +- **AND** the existing terminal or circuit handling is used + +#### Scenario: visible output still prevents clean-close replay + +- **GIVEN** the pending request has surfaced any response event downstream +- **WHEN** the upstream WebSocket closes with code `1000` +- **THEN** the proxy does not replay the request + +#### Scenario: clean-close retry jitter is bounded + +- **GIVEN** clean-close retry jitter is configured +- **WHEN** the additional clean-close replay is scheduled +- **THEN** the delay is no greater than the configured jitter maximum +- **AND** the hard replay cap remains one regardless of the configured value + +#### Scenario: downstream idle recovery transfers reader ownership + +- **GIVEN** the upstream reader is blocked on the current bridge socket +- **AND** the downstream HTTP stream task initiates eligible pre-response recovery +- **WHEN** the bridge replaces the upstream socket +- **THEN** the old reader is cancelled and awaited before its socket is closed +- **AND** the shared session remains live while the replacement socket opens +- **AND** idle pruning retains the registered session while the handoff is in progress +- **AND** exactly one reader owns the replacement socket +- **AND** the local close does not open or increment the retry circuit +- **AND** pending work remains attached to the replacement session + +#### Scenario: silent pre-response recovery precedes the client timeout + +- **GIVEN** the upstream has produced no response event +- **AND** the default ten-second keepalive interval is active +- **WHEN** six silent intervals elapse +- **THEN** the proxy initiates eligible pre-response recovery +- **AND** at least sixty seconds remain before a 120-second client request timeout + +#### Scenario: anchored stuck-gate grace is bounded + +- **GIVEN** a pending HTTP bridge request has a prior continuity anchor +- **AND** no response identifier or `response.created` latency has been recorded +- **WHEN** less than two retire thresholds have elapsed since the gate began waiting +- **THEN** the watchdog does not classify the request as stale +- **WHEN** two retire thresholds elapse without upstream activity +- **THEN** the watchdog may classify the request as stale + +#### Scenario: upstream activity resolves admission-flag ambiguity + +- **GIVEN** a pending request has not acquired the response-created gate +- **AND** upstream activity has not produced a response identifier or `response.created` +- **WHEN** the staleness threshold elapses +- **THEN** the watchdog classifies the request as stale +- **AND** emits pending-state verdict inputs when it skips a watchdog pass + +### Requirement: Durable retry-circuit state protects repeated hard-affinity failures + +For a hard-affinity bridge key, the proxy MUST scope retry-circuit state by +affinity kind, affinity key, and API-key scope (using a stable anonymous scope +when no API key is present). The proxy MUST record only the documented +pre-response failure classes (`stream_incomplete`, `clean_close`, and +`stream_idle_timeout`). + +A bridge retirement MUST record one of those failures only when the retiring +session still owns at least one pending request and no response event has been +observed for that request lifecycle. Retiring an idle upstream bridge with no +pending request MUST NOT advance the circuit or cause a later request to be +treated as a repeated failure. A pending request that has already emitted a +response event MUST remain excluded from this pre-response circuit. + +The default circuit MUST open after two consecutive recorded failures. Once +open, it MUST suppress pre-created replay until the persisted cooldown expires, +using exponential backoff from sixty seconds up to ten minutes. Clean-close +failures MUST cap their cooldown at thirty seconds. The proxy MUST persist +failure count, cooldown deadline, last failure detail, and update time in the +`http_bridge_retry_circuits` table and MUST merge conflict updates so concurrent +replicas cannot shorten an existing cooldown. + +The clean-close retry jitter maximum MUST be read from the +`http_responses_session_bridge_clean_close_retry_jitter_max_seconds` runtime +setting and MUST be bounded to the inclusive range 0–30 seconds. + +The proxy MUST evict process-local circuit entries and their loaded/persisted +markers after one hour without use, independently of durable-row cleanup, so +one-shot hard-affinity keys cannot grow the worker's memory without bound. + +Before every hard-affinity retry decision, the proxy MUST refresh the durable +row so a cooldown opened by another replica is observed even when this process +has already loaded the key. A durable lookup or persistence failure MUST NOT +crash the request; the proxy MUST continue using available local state and +record the failure for observability. Rows older than one hour MUST be treated +as expired and removed. A successful terminal response MUST clear the local +and durable circuit state. + +#### Scenario: idle bridge retirement does not consume a circuit strike + +- **GIVEN** a hard-affinity HTTP bridge has no pending requests +- **WHEN** its upstream WebSocket closes and the idle bridge is retired +- **THEN** the retry-circuit failure count for that key remains unchanged +- **AND** a later request is not placed in cooldown because of the idle close + +#### Scenario: eventless pending retirement consumes exactly one strike + +- **GIVEN** a hard-affinity HTTP bridge owns a pending request with no observed response event +- **WHEN** the bridge retires because the upstream fails before acknowledging the request +- **THEN** the retry circuit records exactly one failure for that request lifecycle + +#### Scenario: midstream retirement does not consume a pre-response strike + +- **GIVEN** a hard-affinity HTTP bridge owns a pending request with an observed response event +- **WHEN** the bridge retires before completion +- **THEN** the pre-response retry-circuit failure count remains unchanged + +#### Scenario: the second hard-key failure opens a durable circuit + +- **GIVEN** a hard-affinity key has one recorded pre-response failure +- **WHEN** a second eligible failure is recorded +- **THEN** the proxy opens the retry circuit +- **AND** persists at least two consecutive failures and a cooldown deadline +- **AND** subsequent pre-created replay is suppressed until that deadline + +#### Scenario: retry decisions observe a cooldown opened by another replica + +- **GIVEN** this replica previously looked up a hard-affinity key with no row +- **AND** another replica persists an open cooldown for that same key and API-key scope +- **WHEN** this replica evaluates the next pre-created retry +- **THEN** it refreshes durable state before deciding +- **AND** suppresses the retry for the persisted cooldown + +#### Scenario: circuit state remains isolated by key and API-key scope + +- **GIVEN** one hard-affinity key has an open circuit +- **WHEN** a different affinity key or API-key scope evaluates a retry +- **THEN** that request is not suppressed by the first key's circuit + +#### Scenario: durable circuit lookup failure does not fail the request + +- **GIVEN** durable retry-circuit lookup or persistence is unavailable +- **WHEN** the proxy evaluates or records a retry-circuit event +- **THEN** the request continues using any available local circuit state +- **AND** the failure is logged and exposed through retry-circuit observability + +### Requirement: Upstream websocket drops penalize affected accounts + +When an upstream websocket closes while one or more streamed response requests +are pending and have not reached a terminal event, the proxy MUST record a +transient upstream error for the account before signaling failure for those +pending requests, except when the close carries a classified process-wide +network failure or upstream WebSocket liveness timeout, is a clean close +(`close_code = 1000`) before any `response.*` event, or carries the classified +per-socket `upstream_keepalive_timeout` transport error. Clean pre-response +closes, keepalive timeouts, process-wide network failures, and liveness +timeouts MUST remain account-neutral and use their classified error and bounded +retry or retry-circuit handling. For other closes, the proxy MUST surface +`stream_incomplete` to affected pending requests except when a direct Responses +WebSocket request has already successfully emitted a finite integer +`sequence_number`. For that sequenced direct-WebSocket case, the proxy MUST +record the request outcome as `stream_incomplete` without emitting a synthetic +terminal frame under the active response id, then MUST close the downstream +WebSocket with code 1011. + +#### Scenario: websocket closes before pending responses complete + +- **GIVEN** a streamed response request is pending on an upstream websocket +- **AND** the direct downstream response has not emitted a numeric sequence, or the request uses another transport +- **WHEN** the websocket closes before a terminal response event is observed +- **AND** the close does not carry a classified process-wide network failure or upstream WebSocket liveness timeout +- **THEN** the pending request fails with `stream_incomplete` +- **AND** the account receives a transient upstream failure signal for routing + +#### Scenario: sequenced direct websocket closes before completion + +- **GIVEN** a direct Responses WebSocket request has successfully emitted a finite integer `sequence_number` +- **WHEN** the upstream websocket closes before a terminal response event is observed +- **AND** the close does not carry a classified process-wide network failure or upstream WebSocket liveness timeout +- **THEN** the request is recorded as failed with `stream_incomplete` +- **AND** no synthetic terminal frame is emitted under the active response id +- **AND** the downstream WebSocket closes with code 1011 +- **AND** the account receives a transient upstream failure signal for routing + +#### Scenario: websocket liveness timeout remains account neutral + +- **GIVEN** a streamed response request is pending on an upstream websocket +- **WHEN** its transport reports `upstream_websocket_liveness_timeout` +- **THEN** the pending request fails with that classified error code +- **AND** the account receives no failure-health signal +- **AND** the request is not transparently replayed + +#### Scenario: clean pre-response close does not penalize the account + +- **GIVEN** a hard-affinity HTTP bridge request is pending with no surfaced response event +- **WHEN** the upstream websocket closes cleanly before response output +- **THEN** the proxy records the clean-close retry-circuit outcome +- **AND** the selected account is not penalized diff --git a/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/tasks.md b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/tasks.md new file mode 100644 index 0000000000..7e1be1873a --- /dev/null +++ b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/tasks.md @@ -0,0 +1,22 @@ +- [x] Add bounded clean-close replay settings with safe defaults. +- [x] Allow one additional clean-close replay only before visible output. +- [x] Add jitter and dedicated retry diagnostics. +- [x] Add regression coverage for the second replay and retry cap. +- [x] Restart the upstream reader when pre-response recovery is initiated by the downstream stream task. +- [x] Add regression coverage for old-reader cancellation and replacement-reader ownership. +- [x] Keep the shared session live across the cancelled reader's socket-generation finalizer. +- [x] Add regression coverage for concurrent pruning during reader handoff. +- [x] Move the default pre-response recovery threshold ahead of the client timeout boundary. +- [x] Bound anchored stuck-gate grace and evaluate staleness from upstream activity/response creation. +- [x] Emit stuck-watchdog skip diagnostics with pending-state verdict inputs. +- [x] Add a forward-only repair for databases stamped before request-usage rollups were connected to the merge head. +- [x] Validate the OpenSpec change and run the focused and full test suites. +- [x] Build and deploy the validated image, then verify production health and logs. + +## Post-deploy regression: idle retirement accounting + +- [x] Require an owned eventless pending request before retirement advances the retry circuit. +- [x] Add lifecycle coverage proving idle no-pending retirement is neutral and eventless pending retirement records exactly one strike. +- [x] Add routed coverage proving an idle close plus one real timeout does not open the repeated-failure cooldown. +- [x] Run focused bridge suites, lint/type/architecture checks, and strict OpenSpec validation. +- [x] Build and deploy the revised image, then verify health and retry-circuit diagnostics. diff --git a/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/.openspec.yaml b/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/.openspec.yaml new file mode 100644 index 0000000000..d7bc0110d8 --- /dev/null +++ b/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-10 diff --git a/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/context.md b/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/context.md new file mode 100644 index 0000000000..d8db27b957 --- /dev/null +++ b/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/context.md @@ -0,0 +1,46 @@ +## Purpose and Scope + +This change closes the gap between Codex's conversation-restart semantics and codex-lb's conservative legacy affinity. It applies only to a Codex goal continuation that resends enough state to start fresh on another account. The normative contract is in the delta for `sticky-session-operations`. + +## Rationale and Constraints + +Raw `codex_session` rows remain hard by default because their provenance is ambiguous during rolling upgrades: the key may be an old process-session identifier or an explicit account-scoped turn state. The goal-continuation marker provides intent, while the strict fresh-replay classifier proves that moving the request does not depend on stored upstream state. Both are required. Classification uses the canonical upstream body so accepted compatibility controls and transport-only envelope fields cannot make equivalent requests disagree. Because the incoming header cannot prove which source wrote an old raw row, restart abandonment is persisted with `session_header` scope; an explicit turn-state lookup of equal text still receives the retained owner. The scoped marker leaves the historical timestamp tombstone empty so an older replica that ignores scope continues to treat the retained owner as hard. + +Unavailable means a persisted account status of `PAUSED`, `RATE_LIMITED`, or `QUOTA_EXCEEDED`. A queue cap, retry exclusion, budget threshold, transient runtime-health decision, or healthy owner does not qualify. Retirement is compare-and-set so concurrent owner changes win. + +Sticky rows are global, but an API key may authorize only a subset of accounts. Retirement authority follows account assignment and security authorization before model/service-tier eligibility: a scoped request cannot mark a row owned by another pool, while an in-scope owner does not lose mutation authority merely because it cannot serve the replacement model. Direct WebSocket account changes also discard proxy-generated turn-state from the retired account; only a turn-state header actually supplied by the client is preserved. + +Goal-restart retirement occurs only inside account selection. A live or durable HTTP bridge for the same process session is not additional ownership evidence after the client proves a self-contained resend, so bridge reuse, preferred-owner promotion, and remote forwarding must not consume the request before selection. A successful guarded retirement is also authoritative over any account objects loaded before that transaction; the retired owner remains excluded for the rest of the selection attempt even if that snapshot still reports it active. A selector that loses the retirement compare-and-set to another selector's scoped marker carries the marker's retained owner into the same exclusion path. + +Canonical replacement changes which generation accepts new work; it does not revoke a predecessor request that already reserved admission or release ownership of that predecessor's resources. The request's reservation proof is captured before asynchronous preparation, while every detached generation remains in lifecycle accounting until resource closure completes. `closed` is an admission fence, not a resource-finalization signal: even an idle predecessor can retain its socket and leases while a slow close runs. + +Drain reporting follows unsettled requests rather than admission state, so a closed detached generation remains restart-blocking while its pending or queued work is being finalized. At a one-generation cap, a verified restart waits for the bounded close of its idle predecessor and rechecks capacity before creating the replacement; if close finalization remains incomplete, the ordinary capacity refusal still applies. + +Resource close is single-flight across reader retirement, account invalidation, and shutdown. Direct close cancellation is deferred until common resource finalization completes, and shutdown starts all snapshotted closes before it can propagate cancellation. Durable replica ownership is also generation-fenced: when a new local socket replaces a durable row that still names the same configured replica, its claim advances the owner epoch before serving work, including across model-transition routing isolation. For example, if generation A clean-closes while generation B reconnects on another model, A's delayed epoch-1 release cannot close B's epoch-2 lease. + +Capacity is released by completed resource finalization, not by detachment or expiration of a bounded close wait. Likewise, shutdown removes canonical routing but keeps each generation in detached lifecycle ownership until close succeeds; a failed close remains available to a later shutdown pass. Security-authorized replacement preserves the original typed continuity source when it rechecks a legacy row, so session-header abandonment cannot accidentally be interpreted as a live explicit-turn-state owner. + +## Failure Modes + +- A normal same-session request still returns the existing hard-affinity error while its owner is unavailable. +- A marked request with `previous_response_id`, nonblank `conversation`, an account-scoped file/image, unsafe payload controls, or unresolved tool output remains owner-bound. +- If the owner recovers or another request changes the row before retirement commits, the update does nothing and selection fails closed rather than discarding the newer state. +- If an authenticated request cannot select the persisted owner under its account policy, the request fails closed without mutating that global row. +- If a process-session ID collides with an explicit turn-state value, restart recovery moves only process-session interpretation; the turn-state request remains bound to the retained owner. +- If the unavailable owner is in policy scope but cannot serve the requested model, guarded abandonment remains authorized and model filtering applies only to replacement selection. +- A live HTTP bridge can retain a detached ACTIVE account object after its persisted owner becomes unavailable. A verified restart bypasses and retires that bridge instead of trusting the stale object. +- A pre-retirement selection snapshot can still contain the old owner. Successful retirement, or an authoritative reread after losing the retirement compare-and-set, filters that owner before replacement selection so namespaced affinity cannot be recreated on it. +- An older replica does not understand source scope. The scoped marker therefore leaves the historical timestamp tombstone empty so that replica keeps the retained hard owner instead of globally abandoning it. +- If repeated restarts detach visible or idle generations faster than their resource closes finish, those generations continue to consume the configured bridge-session capacity; shutdown closes them alongside the current canonical generation. +- If detachment marks a generation closed while request settlement is still pending, drain status remains active and restart-blocking until that work finishes. +- If an idle predecessor alone fills the session cap, the verified restart gives it synchronous bounded-close ownership before deciding whether replacement capacity exists. +- If a generation is already admission-closed but has no resource-close owner, account invalidation still schedules teardown instead of trusting the admission flag. +- If a close or shutdown caller is cancelled, or an old generation releases late after a model transition, single-flight lifecycle tracking and the durable owner epoch keep resources and lease effects fenced until finalization completes. + +## Example + +Session `thread-1` has a raw legacy mapping to account A. Account A becomes quota-exceeded, while account B is active. Codex sends the full conversation under `thread-1`, without a previous-response or conversation object, and includes `Continue working toward the active thread goal.` The proxy marks the still-current raw A mapping abandoned for `session_header`, selects B, and records namespaced process-session affinity to B. A later session turn resolves to B; an explicit `x-codex-turn-state: thread-1` request still resolves the retained raw owner A and fails closed while A is unavailable. + +## Operational Notes + +No setting is introduced. The nullable abandonment-scope migration requires no historical backfill because a non-null timestamp with NULL scope continues to mean global abandonment. Source-qualified markers keep that timestamp NULL, so older binaries safely retain hard ownership during rollout or rollback. Dropping the scope column discards only the restart-recovery marker and restores conservative hard ownership. Existing affinity diagnostics and tombstone administration remain applicable. diff --git a/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/design.md b/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/design.md new file mode 100644 index 0000000000..62fae863f5 --- /dev/null +++ b/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/design.md @@ -0,0 +1,70 @@ +## Context + +Current replicas distinguish newly namespaced soft process-session affinity from raw legacy `codex_session` rows. A raw row always wins because it may represent hard turn-state continuity. Codex conversation restart, however, reuses the process-session identifier while deliberately resending the thread without `previous_response_id`; the request body includes the existing goal-continuation internal context marker. If the raw owner is quota-unavailable, selection currently treats the row as hard and returns `hard_affinity_saturated` forever (or until the six-hour stale-owner cleanup), even though this restart payload no longer needs the owner's upstream state. + +## Goals / Non-Goals + +**Goals:** + +- Recover an explicitly marked, self-contained Codex restart immediately when its legacy owner is durably unavailable. +- Reuse the existing strict fresh-replay classifier for account-neutrality and tool-state safety. +- Preserve hard ownership for every ordinary or account-dependent request. +- Make owner retirement safe under concurrent selection and rebinding. + +**Non-Goals:** + +- Reallocate hard rows because of local caps, retry exclusions, transient health, or budget pressure. +- Make arbitrary full-resend requests mobile without the Codex restart marker. +- Change previous-response, conversation, file, bridge, or turn-state ownership semantics. +- Add a setting or new client-visible error code. + +## Decisions + +### Derive a typed restart capability during affinity classification + +The Responses request classifier will expose whether any input item carries the already-recognized `` prefix. `_sticky_key_for_responses_request` will grant an `abandon_unavailable_legacy_owner` capability only when that marker is present and the canonical upstream request body passes `responses_payload_is_account_neutral_fresh_replay`. + +This reuses the proof applied to cross-account replay instead of maintaining a second list of unsafe fields. Canonical request serialization is required because raw model dumps retain accepted compatibility controls and the direct-WebSocket response-create discriminator even though neither is upstream account state. Inferring restart from a missing `previous_response_id` alone was rejected because ordinary first turns and lossy incremental requests have that shape. Introducing a new client header was rejected because the deployed Codex client already supplies a stable payload marker. + +### Limit retirement to legacy process-session ownership and durable statuses + +Selection may consume the capability only for a raw compatibility row reached through typed `session_header` provenance. That provenance describes the current request, not the historical row: raw storage mixed process-session and explicit turn-state values, so equal client-controlled text cannot prove which source wrote it. The repository therefore marks abandonment as applying only to `session_header` interpretation and retains the row's account for explicit `turn_state` lookup. The repository update also requires the current owner to be `PAUSED`, `RATE_LIMITED`, or `QUOTA_EXCEEDED`; local capacity, runtime-health, exclusion, and budget decisions cannot authorize retirement. + +Mutation authority is the authenticated account-assignment and security-policy scope before requested-model and service-tier filtering. Model eligibility still constrains the replacement pool, but cannot make an otherwise authorized raw owner appear out of scope. + +### Mark source-qualified abandonment with compare-and-set, then rerun normal selection + +The repository will atomically set `continuity_abandonment_scope=session_header` while leaving `continuity_abandoned_at` NULL, only when the key, kind, expected account, unmarked state, and unavailable account status still match. Selection clears its cached legacy owner and repeats its normal loop. Source-aware lookup treats that row as abandoned for process-session selection but continues returning the retained account to an explicit turn-state request. The scope-only representation is intentionally asymmetric with historical global tombstones: an older binary ignores the unknown scope but sees no timestamp tombstone, so it continues to fail closed on the retained owner during rollout or rollback. + +Deleting the row outright was rejected because tombstones distinguish deliberate continuity abandonment from an unknown owner. Blindly updating after an earlier status read was rejected because a concurrent rebind or account recovery could otherwise be lost. + +The normal loop may still hold account objects loaded before retirement. Successful retirement therefore records the retired account for the lifetime of the selection call and filters it from later iterations. A compare-and-set loser rereads the marker and receives the retained retired account separately from ownerless affinity, then applies the same exclusion. Re-reading every account after the write was rejected because the request needs only one authoritative exclusion and the broader refresh would add unrelated database work. + +### Force verified HTTP restarts through account selection + +HTTP bridge reuse normally returns before account selection, and durable bridge metadata can promote the prior account to a required preferred owner or forward the request to another replica. For a verified self-contained goal restart, these paths would prevent the guarded raw-owner retirement from running. The bridge path therefore ignores that prior bridge ownership for this request, detaches any matching local bridge, and creates a replacement only after ordinary selection has evaluated the retirement capability. A bridge with visible work is detached without interrupting that work; an idle bridge is closed normally. + +Detachment does not end lifecycle ownership. A predecessor request that reserved its lane before replacement keeps request-scoped submit authority after queue publication clears the mutable reservation field. Detached live generations remain tracked separately from the canonical key, count against capacity until close finishes, participate in drain status, and are included in shutdown cleanup. Treating the canonical map as the complete live-generation registry was rejected because repeated restarts could otherwise hide unbounded sockets, readers, durable leases, and account leases. + +The admission-only `closed` flag does not make pending or queued settlement disappear from drain status. Conversely, an idle predecessor is immediately finalizable: when it alone fills the configured cap, the authorizing restart owns its bounded close synchronously and rechecks actual lifecycle capacity before opening the replacement. Scheduling that close in the background was rejected because cap enforcement could run first and fail the same recovery request even though it had already detached the only idle generation. + +The predecessor may drain its admitted response but cannot publish newly learned aliases under a key now owned by the replacement. Detached ownership is removed only by common close finalization so direct terminal-error paths and bounded background closes share the same capacity lifecycle. Request-final cleanup rechecks detached generations after reservations are released, and account-level invalidation closes both canonical and detached sockets. + +Close finalization shields its resource task from caller cancellation and re-raises cancellation only after releasing the reader, socket, durable lease, and account lease. A newly created local generation also advances the durable owner epoch whenever the existing durable row still names the same replica identity. Replica identity is a routing destination, not a websocket-generation fence; without the epoch advance, a delayed release from the predecessor (or a prior process using the same configured identity) could close the replacement's lease. + +## Risks / Trade-offs + +- [A forged marker requests owner abandonment] → The complete payload must still be account-neutral and self-contained, retirement occurs only while the persisted owner is unavailable, and source-qualified abandonment cannot erase a colliding explicit turn-state owner. +- [A concurrent request changes ownership or restores the account] → One compare-and-set statement verifies both mapping owner and account status at write time; a miss rereads authoritative state and preserves fail-closed behavior. +- [Another restart wins the retirement compare-and-set] → The reread returns the retained retired owner as exclusion evidence so stale account inputs cannot re-pin it. +- [A restart includes unresolved tool output or account-scoped content] → The existing fresh-replay classifier denies the capability, leaving the hard row untouched. +- [Transport wiring drifts] → Carry one typed affinity-policy flag through the shared selection boundary and explicitly forward it from the direct WebSocket call site that expands policy fields. +- [A stale bridge or account snapshot restores the old owner] → Verified HTTP restarts bypass bridge reuse/forwarding, and successful retirement or a winner's scoped marker excludes the old owner from all later iterations of the current selection call. + +## Migration Plan + +Add nullable `sticky_sessions.continuity_abandonment_scope`. Existing rows require no backfill: a non-null abandonment timestamp with NULL scope retains its historical meaning as a global stale-hard tombstone. Goal restart writes `session_header` with a NULL timestamp; ordinary upsert clears both fields, and stale-hard cleanup may later promote a source-qualified marker to global after the normal age threshold. Older binaries and a downgraded schema see the retained account as hard ownership because the timestamp remains NULL. Downgrade therefore loses only the source-qualified recovery marker rather than weakening ownership. + +## Open Questions + +None. diff --git a/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/proposal.md b/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/proposal.md new file mode 100644 index 0000000000..5213a8798c --- /dev/null +++ b/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/proposal.md @@ -0,0 +1,35 @@ +## Why + +Codex conversation restart can resend a self-contained thread under the same process-session identifier after the durable owner exhausts quota. A legacy hard `codex_session` row currently keeps that restarted request pinned to the unavailable owner, so it fails even while another account can serve it. + +## What Changes + +- Recognize the existing Codex goal-continuation context as an explicit restart signal. +- Permit a restart-shaped, self-contained request to retire its unavailable legacy hard owner and select another account. +- Classify accepted compatibility and transport request forms through the same canonical replay-safety body. +- Ensure live/durable HTTP bridge reuse and stale account snapshots cannot bypass or undo guarded owner retirement. +- Keep detached pending work restart-blocking and let an idle predecessor release a cap-constrained replacement slot. +- Keep ordinary incremental, conversation-bound, file-pinned, and unresolved tool-state requests fail-closed on their required owner. +- Make retirement compare-and-set so a concurrent owner change cannot be deleted. +- Scope retirement to process-session interpretation so an equal raw explicit turn state retains its owner. +- Derive mutation authority before model and service-tier replacement eligibility. +- Cover the public Codex Responses route and subsequent continuity on the replacement owner. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `sticky-session-operations`: Define the proof-gated exception that lets an explicit self-contained Codex restart abandon an unavailable legacy hard owner. + +## Impact + +- Account selection and sticky-session persistence in the proxy module. +- One nullable sticky-session column records source-qualified abandonment; existing tombstones remain global without backfill. +- Responses request classification shared by HTTP and WebSocket transports. +- HTTP Responses bridge reuse, forwarding, and replacement-session lifecycle. +- Routed regression coverage for `/backend-api/codex/responses`. +- No configuration or public error-envelope change. diff --git a/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/specs/sticky-session-operations/spec.md b/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/specs/sticky-session-operations/spec.md new file mode 100644 index 0000000000..4673aafa76 --- /dev/null +++ b/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/specs/sticky-session-operations/spec.md @@ -0,0 +1,254 @@ +## MODIFIED Requirements + +### Requirement: Sticky sessions are explicitly typed +The system SHALL persist each sticky-session mapping with an explicit kind so durable Codex backend affinity, durable dashboard sticky-thread routing, and bounded prompt-cache affinity can be managed independently. Budget-pressure reallocation MUST apply only to mappings whose kind/source is soft. A raw or legacy `codex_session` mapping MUST remain owner-bound because it may represent explicit turn-state continuity; budget pressure MUST NOT delete or rebind it. + +An explicit Codex goal-continuation restart MAY abandon a raw legacy `codex_session` owner only when the complete Responses payload is account-neutral and self-contained: it MUST have no nonblank `previous_response_id`, no nonblank `conversation`, no account-scoped input file or image reference, and no unresolved or orphan tool state. Classification MUST use the canonical upstream request form so accepted compatibility controls and transport-envelope fields do not make equivalent requests disagree. The owner MUST be persisted as `PAUSED`, `RATE_LIMITED`, or `QUOTA_EXCEEDED` and MUST belong to the authenticated request's account-assignment and security-policy scope computed before model and service-tier eligibility; local capacity, model eligibility, retry exclusions, runtime health, budget pressure, and an out-of-scope owner MUST NOT determine mutation authority. The retirement write MUST compare the current mapping owner and unavailable account status atomically, MUST preserve a concurrently changed mapping or recovered owner, and on success MUST let normal selection establish affinity to the replacement account. Because a raw key's persisted source is ambiguous, goal-restart abandonment MUST apply only to `session_header` interpretation and MUST retain the stored account as hard ownership for an explicit `turn_state` lookup using the same text. During a rolling deployment or rollback, replicas that do not understand source-qualified abandonment MUST continue treating that retained account as hard ownership. A selector that observes source-qualified abandonment initially or after losing the retirement compare-and-set MUST exclude the retained retired owner until replacement affinity is persisted, even if its account inputs predate retirement. Restart authority MUST remain scoped to the classified request and MUST NOT persist on a reusable bridge for later requests. A live or durable HTTP bridge for the same process session MUST NOT bypass this guarded selection through local reuse, owner forwarding, or preferred-owner promotion. Canonical replacement MUST preserve an already reserved predecessor request's authority to submit on its detached draining generation after queue publication clears the mutable reservation marker. A detached predecessor MAY finish its admitted response but MUST NOT publish new turn-state or previous-response aliases under the replacement generation's canonical key. Every detached generation, including an idle generation already marked closed for admission, MUST remain owned by the bridge lifecycle, MUST count against the configured session cap until resource closure completes, and MUST be closed during service shutdown, account invalidation, or drained reservation cleanup. The admission-only closed state MUST NOT be treated as proof that the socket and leases have a close owner. Resource teardown MUST be single-flight, and all close paths MUST release detached-generation ownership only after resource closure finishes, even when a close caller is cancelled. Shutdown MUST schedule and await every snapshotted generation before propagating cancellation. A new local bridge generation that replaces durable ownership under the same replica identity MUST advance the durable owner epoch before serving requests so a predecessor's late release cannot close the replacement lease, including when model-transition isolation discards the durable lookup as a routing input. + +A later security-authorized bridge replacement that revalidates a raw legacy row MUST preserve the request's typed continuity source. A source-scoped session-header abandonment MUST remain ownerless for that replacement while an explicit turn-state lookup of the same raw value remains owner-bound. A planned capacity eviction MUST NOT stop counting a detached generation merely because its bounded close wait timed out. Shutdown MUST retain any generation whose resource close fails so a later shutdown pass can retry finalization. Drain status MUST count pending or queued work on a detached generation even after it is closed for admission. When a verified restart replaces an idle predecessor that fills the configured session cap, admission MUST give that predecessor synchronous bounded-close ownership and MUST recheck actual lifecycle capacity before opening the replacement. + +#### Scenario: Soft sticky reallocation uses split primary and secondary pressure thresholds +- **WHEN** a request resolves an existing prompt-cache, sticky-thread, or other explicitly soft mapping +- **AND** the pinned account is otherwise eligible to serve traffic +- **AND** the pinned account is strictly above either the configured primary sticky reallocation threshold or the configured secondary sticky reallocation threshold +- **AND** another eligible account remains at or below both configured sticky reallocation thresholds +- **THEN** selection rebinds the sticky-session mapping to the healthier account before sending the request upstream + +#### Scenario: Sticky reallocation preserves a pinned account when every candidate is split-threshold pressured +- **WHEN** a request resolves an existing soft sticky-session mapping +- **AND** the pinned account is otherwise eligible to serve traffic +- **AND** the pinned account is strictly above either configured sticky reallocation threshold +- **AND** every other eligible account is also strictly above at least one configured sticky reallocation threshold +- **THEN** selection retains the existing pinned account to avoid sticky-pin thrashing + +#### Scenario: Fresh selection does not apply sticky secondary pressure threshold +- **WHEN** a request has no sticky-session mapping +- **AND** one eligible account is above the configured secondary sticky reallocation threshold but below the normal primary budget threshold +- **THEN** the account remains eligible for ordinary non-sticky routing according to the selected routing strategy + +#### Scenario: Hard Codex mapping ignores budget-pressure reallocation + +- **GIVEN** a raw `codex_session` mapping points to account A +- **AND** account A is above a sticky budget-pressure threshold +- **AND** account B has more remaining budget +- **WHEN** the request is selected +- **THEN** selection remains constrained to account A +- **AND** the raw mapping is neither deleted nor rebound to account B + +#### Scenario: Unavailable hard Codex owner does not lose its mapping + +- **GIVEN** a raw `codex_session` mapping points to account A +- **AND** account A is temporarily quota-exceeded or otherwise unusable +- **AND** account B is healthy +- **WHEN** an ordinary request or an unsafe restart-shaped request requires the mapping +- **THEN** the request fails closed instead of selecting account B +- **AND** the raw mapping is neither deleted nor rebound + +#### Scenario: Self-contained goal restart abandons unavailable legacy owner + +- **GIVEN** a process-session identifier has a raw legacy `codex_session` mapping to account A +- **AND** account A is paused, rate-limited, or quota-exceeded +- **AND** account B is eligible +- **WHEN** Codex sends the recognized goal-continuation marker with an account-neutral self-contained full resend and no other continuity dependency +- **THEN** the proxy marks the still-current raw mapping to account A abandoned only for process-session interpretation +- **AND** it routes the restarted turn to account B +- **AND** subsequent session or response continuity remains on account B + +#### Scenario: Goal restart cannot erase colliding explicit turn-state ownership + +- **GIVEN** a raw legacy `codex_session` row was written as explicit turn-state ownership for account A +- **AND** a process-session header later uses the same client-controlled text +- **WHEN** a marked self-contained goal restart abandons that text for process-session interpretation +- **THEN** the process-session restart may select account B +- **AND** an explicit turn-state lookup of the same text remains hard-bound to account A + +#### Scenario: Source-qualified retirement fails closed on an older replica + +- **GIVEN** a current replica marks a raw account A mapping abandoned only for `session_header` interpretation +- **WHEN** a replica that does not understand abandonment scope reads the same raw mapping +- **THEN** it continues to resolve account A as hard ownership +- **AND** it cannot re-pin a colliding explicit turn state to another account + +#### Scenario: Model eligibility does not narrow retirement authority + +- **GIVEN** unavailable account A is inside the authenticated account-assignment and security-policy scope +- **AND** account A cannot serve the restart's requested model while account B can +- **WHEN** a marked self-contained goal restart evaluates the raw mapping owned by account A +- **THEN** account A remains authorized for the guarded abandonment mutation +- **AND** model and service-tier eligibility apply only when selecting the replacement + +#### Scenario: Equivalent request forms receive the same restart classification + +- **GIVEN** two marked self-contained goal restarts differ only by accepted compatibility controls or a transport-only response-create envelope +- **WHEN** the proxy classifies their account-neutral replay safety +- **THEN** it evaluates the same canonical upstream request fields for both forms +- **AND** neither form remains pinned merely because its accepted input representation differs + +#### Scenario: Goal restart bypasses a stale live HTTP bridge owner + +- **GIVEN** a live HTTP bridge and raw legacy mapping both identify account A for a process session +- **AND** the bridge's detached account snapshot still reports account A active +- **AND** account A is now persisted as paused, rate-limited, or quota-exceeded +- **WHEN** a marked account-neutral self-contained goal restart arrives for that process session +- **THEN** the proxy does not reuse or forward to account A's bridge +- **AND** guarded selection retires the raw owner before a replacement bridge is created on eligible account B + +#### Scenario: Restart authority does not outlive its request + +- **GIVEN** a marked self-contained goal restart creates a reusable HTTP bridge while legacy owner account A is healthy +- **WHEN** a later ordinary request reuses that bridge and account A has become unavailable +- **THEN** the ordinary request fails closed instead of inheriting the earlier restart's retirement authority +- **AND** the raw mapping to account A is neither tombstoned nor rebound + +#### Scenario: Reserved predecessor submits after canonical replacement + +- **GIVEN** an unanchored request has reserved the canonical session-header bridge before submit +- **AND** a verified goal restart replaces that canonical bridge while the reserved request is preparing its payload +- **WHEN** the reserved request publishes queued activity and clears its mutable reservation marker +- **THEN** the request submits exactly once on its detached predecessor generation +- **AND** canonical replacement does not reject that request as unregistered or replaced + +#### Scenario: Detached restart generations remain capacity bounded + +- **GIVEN** repeated verified restarts replace canonical bridges that still own visible or reserved requests +- **WHEN** the number of canonical, detached-live, and in-flight generations reaches the configured session cap +- **THEN** the service refuses another generation with its bounded local-capacity error +- **AND** detached sockets, readers, durable leases, and account leases are not omitted from capacity accounting + +#### Scenario: Idle detached predecessor remains capacity owned while closing + +- **GIVEN** a verified restart replaces an idle canonical bridge and its resource close is still running +- **WHEN** another restart would exceed the configured session cap +- **THEN** the admission-closed predecessor still counts as a detached generation +- **AND** the service either closes an evictable canonical generation before replacement creation or refuses the new generation + +#### Scenario: Closed detached request settlement blocks restart + +- **GIVEN** canonical replacement marked a detached predecessor closed for admission +- **AND** that predecessor still has pending or queued request settlement +- **WHEN** the service reports HTTP bridge drain status +- **THEN** the bridge remains active and restart-blocking +- **AND** it stops blocking only after the unsettled work reaches zero + +#### Scenario: One-session restart closes its idle predecessor before cap enforcement + +- **GIVEN** the bridge session cap is one and an idle canonical predecessor occupies that generation +- **WHEN** a verified goal restart forces canonical replacement +- **THEN** admission detaches the predecessor and gives it synchronous bounded-close ownership +- **AND** it opens the replacement only after close finalization releases the slot, otherwise it returns the bounded capacity refusal + +#### Scenario: Timed-out LRU close does not manufacture capacity + +- **GIVEN** admission detaches an idle LRU generation and reserves an in-flight replacement slot +- **AND** the bounded close wait returns before that generation's resource finalizer completes +- **WHEN** admission rechecks the configured session cap before opening the replacement socket +- **THEN** the detached generation still consumes capacity +- **AND** the service refuses replacement creation rather than exceeding the cap + +#### Scenario: Shutdown closes detached bridge generations + +- **GIVEN** canonical replacement detached an older generation whose request is still draining +- **WHEN** the service closes all HTTP bridge sessions +- **THEN** it closes both canonical and detached generations +- **AND** no detached socket, reader, durable lease, or account lease escapes shutdown ownership + +#### Scenario: Shutdown cancellation does not orphan later generations + +- **GIVEN** shutdown snapshots multiple canonical or detached bridge generations +- **AND** one generation has a slow resource close +- **WHEN** the shutdown caller is cancelled +- **THEN** every snapshotted generation receives a close owner before cancellation is propagated +- **AND** shutdown awaits all of those closes through resource finalization + +#### Scenario: Failed shutdown close remains retryable + +- **GIVEN** shutdown removes a canonical generation from routing and starts its resource close +- **WHEN** pending settlement or another resource finalizer fails +- **THEN** the generation remains in detached lifecycle ownership +- **AND** a later shutdown pass retries its close instead of losing the socket or leases + +#### Scenario: Detached predecessor cannot publish replacement continuity + +- **GIVEN** canonical replacement detaches an older generation while its admitted response is still draining +- **WHEN** that predecessor receives a new turn-state or previous-response alias +- **THEN** it does not publish the alias under the canonical key now occupied by the replacement +- **AND** the predecessor may still finish delivering its already admitted response + +#### Scenario: Detached generation closes after its final reservation ends + +- **GIVEN** a detached predecessor is retained only by an unsubmitted request reservation +- **WHEN** request finalization releases that reservation without submitting +- **THEN** the service closes the drained predecessor and releases its capacity ownership after resource closure finishes + +#### Scenario: Account invalidation includes detached generations + +- **GIVEN** an account owns both canonical and detached bridge generations +- **WHEN** the account is deactivated, requires reauthentication, or changes proxy binding +- **THEN** the service closes every generation authenticated to that account +- **AND** no detached socket remains routed through the invalid account binding + +#### Scenario: Admission-closed detached generation is still invalidated + +- **GIVEN** a detached generation is marked closed for admission but has no resource-close owner +- **WHEN** its account is invalidated +- **THEN** the service schedules resource teardown for that generation +- **AND** an already owned or successfully finalized close is not scheduled twice + +#### Scenario: Same-replica model replacement advances the durable epoch + +- **GIVEN** a durable bridge row names the current replica and an older model +- **WHEN** model-transition isolation creates a replacement generation and stops using that row for routing +- **THEN** the replacement claim still advances the durable owner epoch +- **AND** the predecessor's late release cannot close the replacement lease + +#### Scenario: Stale selection snapshot cannot repin a retired owner + +- **GIVEN** restart selection loaded account A as active before guarded retirement observes its unavailable persisted status +- **WHEN** guarded retirement tombstones account A's still-current raw legacy mapping +- **THEN** the remainder of that selection excludes account A from the stale snapshot +- **AND** the namespaced process-session mapping is not established on account A + +#### Scenario: Retirement CAS loser excludes the winner's retired owner + +- **GIVEN** two marked restarts read the same raw account A mapping and stale account inputs +- **AND** the first restart marks account A abandoned only for `session_header` interpretation +- **WHEN** the second restart loses its retirement compare-and-set and rereads that marker +- **THEN** the second restart excludes retained account A from its stale inputs +- **AND** it cannot establish replacement affinity on account A + +#### Scenario: Goal marker does not override account-scoped continuity + +- **GIVEN** a marked goal-continuation request carries a nonblank `previous_response_id`, nonblank `conversation`, account-scoped file or image reference, or unresolved tool output +- **WHEN** its hard owner is unavailable +- **THEN** the request fails closed +- **AND** the hard mapping is not abandoned + +#### Scenario: Healthy owner is not abandoned + +- **GIVEN** a marked account-neutral goal-continuation restart has a raw legacy owner that is still active +- **WHEN** the owner is locally capped, excluded, budget-pressured, or transiently unhealthy +- **THEN** the mapping remains owner-bound +- **AND** the restart does not retire it as unavailable + +#### Scenario: Concurrent owner change wins retirement race + +- **GIVEN** restart selection observed a raw legacy mapping to unavailable account A +- **WHEN** another operation rebinds that mapping or restores the owner before the retirement write executes +- **THEN** the compare-and-set retirement does not tombstone the newer state +- **AND** selection preserves fail-closed ownership semantics + +#### Scenario: Scoped API key cannot retire another pool's owner + +- **GIVEN** a raw legacy `codex_session` mapping points to unavailable account A +- **AND** the authenticated API key's effective account-policy scope contains account B but not account A +- **WHEN** the key sends a marked account-neutral goal-continuation restart for that session +- **THEN** the request fails closed before upstream dispatch +- **AND** the raw mapping to account A is neither tombstoned nor rebound + +#### Scenario: Generated WebSocket turn state does not follow a restarted goal + +- **GIVEN** the proxy generated the downstream turn-state header used by an upstream WebSocket on account A +- **AND** a marked self-contained goal restart retires that socket and selects account B +- **WHEN** the proxy opens the replacement WebSocket +- **THEN** it removes account A's generated turn-state token before connect +- **AND** it still forwards the restart's full replay payload to account B diff --git a/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/tasks.md b/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/tasks.md new file mode 100644 index 0000000000..34a34380da --- /dev/null +++ b/openspec/changes/archive/2026-08-10-recover-restarted-conversation-affinity/tasks.md @@ -0,0 +1,45 @@ +## 1. Regression Coverage + +- [x] 1.1 Add a routed `/backend-api/codex/responses` regression proving a marked self-contained restart escapes an unavailable legacy owner and subsequent continuity stays on the replacement. +- [x] 1.2 Add negative coverage for ordinary requests, account-dependent payloads, healthy owners, and compare-and-set races. +- [x] 1.3 Add routed regressions for scoped-owner retirement and synthesized WebSocket turn-state cleanup. +- [x] 1.4 Add regressions for canonical compatibility classification, stale post-retirement selection inputs, and live HTTP bridge bypass. +- [x] 1.5 Add regressions proving a colliding explicit turn state retains its owner and model eligibility does not narrow mutation authority. +- [x] 1.6 Add regressions proving old-reader fail-closed behavior and CAS-loser exclusion of a concurrently retired owner. +- [x] 1.7 Add regressions for reserved predecessor submission, bounded detached generations, and shutdown cleanup. +- [x] 1.8 Add regressions for detached alias fencing, common close finalization, drained reservation cleanup, and account invalidation. +- [x] 1.9 Add regressions for cancellation-safe detached closure and same-replica durable-generation fencing. +- [x] 1.10 Add regressions for idle detached capacity, admission-closed account invalidation, cancellation-safe global shutdown, and same-replica model-transition fencing. +- [x] 1.11 Add regressions for source-aware security rebind, timed-out eviction capacity, and retryable failed shutdown closure. +- [x] 1.12 Add regressions for closed detached drain accounting and cap-constrained idle restart replacement. + +## 2. Implementation + +- [x] 2.1 Expose goal-continuation marker detection and derive the typed restart capability only for account-neutral fresh-replay payloads. +- [x] 2.2 Add atomic unavailable-owner tombstoning guarded by mapping owner and account status. +- [x] 2.3 Thread the capability through HTTP and WebSocket selection and rerun normal selection after successful retirement. +- [x] 2.4 Restrict retirement mutation to the authenticated effective account scope and preserve generated turn-state cleanup across account changes. +- [x] 2.5 Prevent bridge reuse/forwarding, leaked one-shot restart authority, and stale account snapshots from bypassing or undoing guarded restart retirement. +- [x] 2.6 Persist source-qualified session-header abandonment while retaining hard explicit turn-state ownership. +- [x] 2.7 Separate authenticated sticky-mutation authority from model and service-tier replacement eligibility. +- [x] 2.8 Encode source-qualified retirement so legacy readers retain hard ownership, and preserve the retired owner as exclusion evidence on every typed read. +- [x] 2.9 Preserve request-owned admission across replacement and track detached live generations through closure. +- [x] 2.10 Fence detached continuity publication and finalize every detached generation through reservation, account, direct-close, and shutdown paths. +- [x] 2.11 Defer direct-close cancellation through resource finalization and advance same-replica replacement owner epochs. +- [x] 2.12 Keep every detached generation capacity-owned, make resource close single-flight, preserve model-transition epoch provenance, and schedule all shutdown closes before cancellation propagation. +- [x] 2.13 Preserve typed abandonment during security rebind, recheck capacity after bounded eviction closes, and retain failed shutdown generations for retry. +- [x] 2.14 Keep unsettled closed generations restart-blocking and synchronously close an idle forced predecessor when it fills the cap. + +## 3. Validation and Documentation + +- [x] 3.1 Run focused regressions, relevant sticky/session suites, lint/type checks, and strict OpenSpec validation. +- [x] 3.2 Review the diff for fail-closed continuity, async/session ownership, transport parity, and simplicity-gate compliance. +- [x] 3.3 Promote stable context to the main capability docs and verify the change. +- [x] 3.4 Add reversible migration coverage for source-qualified sticky abandonment metadata. +- [x] 3.5 Document and validate rolling-version marker semantics plus concurrent retirement handling. +- [x] 3.6 Document detached drain and one-session replacement capacity semantics. + +## 4. Local Deployment + +- [x] 4.1 Build a revision-labelled local Docker image without exposing deployment secrets. +- [x] 4.2 Replace the running codex-lb container with rollback protection and verify health plus deployed revision. diff --git a/openspec/changes/active-conversations-average/.openspec.yaml b/openspec/changes/archive/2026-08-13-active-conversations-average/.openspec.yaml similarity index 100% rename from openspec/changes/active-conversations-average/.openspec.yaml rename to openspec/changes/archive/2026-08-13-active-conversations-average/.openspec.yaml diff --git a/openspec/changes/active-conversations-average/proposal.md b/openspec/changes/archive/2026-08-13-active-conversations-average/proposal.md similarity index 100% rename from openspec/changes/active-conversations-average/proposal.md rename to openspec/changes/archive/2026-08-13-active-conversations-average/proposal.md diff --git a/openspec/changes/active-conversations-average/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-active-conversations-average/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/active-conversations-average/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-active-conversations-average/specs/frontend-architecture/spec.md diff --git a/openspec/changes/active-conversations-average/tasks.md b/openspec/changes/archive/2026-08-13-active-conversations-average/tasks.md similarity index 100% rename from openspec/changes/active-conversations-average/tasks.md rename to openspec/changes/archive/2026-08-13-active-conversations-average/tasks.md diff --git a/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/context.md b/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/context.md new file mode 100644 index 0000000000..db9e2015db --- /dev/null +++ b/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/context.md @@ -0,0 +1,207 @@ +# Telemetry capability — context + +## Purpose + +Give the project visibility into its install base (version distribution, deployment shapes, +client ecosystem, feature usage) without collecting anything that identifies an operator, +an account, or request content. Consent model is informed opt-out: active by default, +one-time dialog with the exact payload, settings toggle, env kill switch. + +Decision record (2026-08-06, maintainer): default-on with first-run confirmation dialog for +both new and existing users; settings toggle; expanded field set over the minimal version. + +## Collection endpoint + +Self-hosted SHM (kOlapsis/shm) server operated by the maintainer at +`https://telemetry.tokmaxxing.com`. SHM provides Ed25519 instance signing, aggregate dashboards, +and public README badges (`/badge/codex-lb/instances`, `/badge/codex-lb/version`). +The SDK path is `/v1/register`, `/v1/activate`, `/v1/snapshot` (note: NOT `/api/v1/`, +which is SHM's admin namespace). codex-lb implements a small Python client (SHM ships +Go/Node SDKs only). + +## Payload schema v1 (the allowlist) + +Everything below derives from existing data (`request_logs`, settings, module registry). +No new per-request instrumentation. `*_bucket` fields use the documented bucket sets. + +Every outbound request body has an explicit Pydantic model and is covered by the wire-schema +allowlist test. The registration body sent to `/v1/register` is: + +```json +{ + "app_name": "codex-lb", + "app_version": "1.20.2", + "deployment_mode": "docker | k8s | pip | bare", + "environment": "", + "instance_id": "", + "os_arch": "linux/x86_64", + "public_key": "" +} +``` + +`app_name` identifies this project, `app_version` supports upgrade/deprecation decisions, +`deployment_mode` and `os_arch` are coarse deployment signals, `environment` is intentionally +empty, and `instance_id` plus `public_key` establish the random signing identity. Activation +sends only `{"action": "activate"}`. + +The signed `/v1/snapshot` body is an envelope. The consent preview renders this same shape via +the same constructor; its timestamp is the current preview-generation time, while an actual +send regenerates the current transmission time. + +```json +{ + "instance_id": "", + "metrics": { "": "..." }, + "timestamp": "2026-08-06T12:00:00Z" +} +``` + +The `metrics` object is the versioned snapshot schema: + +```json +{ + "schema_version": 1, + "instance_id": "", + "version": "1.20.2", + "python": "3.13", + "os": "linux", + "arch": "x86_64", + "uptime_hours": 168, + + "deploy": { + "method": "docker | k8s | pip | bare", + "db_backend": "sqlite | postgres", + "db_size_bucket": "unknown | ", + "replicas": 3, + "reverse_proxy": true + }, + + "accounts": { + "pool_bucket": "", + "plan_mix": {"plus": "", "pro": "", "team": "", "free": ""}, + "workspace_accounts": true, + "routing_policy": "", + "limit_warmup_enabled": true, + "egress_proxy_used": false + }, + + "usage_7d": { + "requests": 203051, + "success_rate": 0.987, + "tokens_input": 18800000000, + "tokens_output": 94000000, + "tokens_cached_ratio": 0.89, + "cost_usd_bucket": "", + "request_kinds": {"responses": 0.0, "chat": 0.0, "images": 0.0, "unknown": 1.0}, + "transport_mix": {"ws": 0.6, "http_bridge": 0.4}, + "service_tier_mix": {"default": 0.90, "flex": 0.05, "priority": 0.05}, + "clients": {"codex-cli": 0.44, "openai-sdk-python": 0.3, "other": 0.02}, + "clients_other_ratio": 0.02, + "models": [ + { + "name": "gpt-5.4-codex", + "share": 0.62, + "reasoning": {"xhigh": 0.31, "high": 0.48, "medium": 0.21}, + "avg_output_tokens_bucket": "" + } + ], + "latency_ms_p50": 1200, + "ttft_ms_p50": 800, + "ttft_ms_p95": 3400, + "rate_limit_429_ratio": 0.004, + "top_upstream_errors": ["server_overloaded", "usage_limit_reached"] + }, + + "features": { + "api_firewall": true, + "quota_planner": true, + "sticky_sessions": true, + "conversation_archive": false, + "automations": false, + "fleet": false, + "model_sources_count": 2, + "api_keys_bucket": "", + "prometheus": false, + "otel": false, + "dashboard_auth": true, + "reset_credits": true, + "image_api_used": true + } +} +``` + +Field notes: + +- `top_upstream_errors`: enum `upstream_error_code` values only, top 5 by count. Free-text + `error_message` is banned by spec. +- `request_kinds`: current `request_logs` rows do not persist ingress route family. The existing + `request_kind` column is a workload class (`normal`, `warmup`, `compaction`, and similar), + while `source` identifies the upstream. Until an authoritative route-family signal exists, + rows are reported as `unknown`; source and model-name heuristics are deliberately forbidden. +- `clients`: canonical family shares from the normative mapping table in `spec.md`. Raw + `useragent_group` values never leave the instance. +- `models[].name`: official model catalog allowlist match; custom/unknown model names fold + into a single `{"name": "other"}` entry. +- Exact `requests` / token counts are transmitted raw deliberately: they power the global + aggregate counter story and cannot identify an instance. Everything correlated with spend + or org size (accounts, keys, cost, DB size) is bucketed. +- `replicas`: size of the configured HTTP bridge instance ring (multi-replica adoption signal). + +## Bucket sets + +- count buckets (accounts, api keys, plan mix): `0`, `1`, `2-5`, `6-20`, `21-100`, `100+` +- `db_size_bucket`: `unknown`, `<100MB`, `100MB-1GB`, `1-5GB`, `5-10GB`, `10-50GB`, `50GB+` +- `cost_usd_bucket` (7d): `<10`, `10-100`, `100-1k`, `1k-10k`, `10k-50k`, `50k+` +- `avg_output_tokens_bucket`: `<250`, `250-1k`, `1k-4k`, `4k-16k`, `16k+` + +## Consent resolution precedence + +`CODEX_LB_TELEMETRY_ENABLED` env (when set) > persisted decision > default +(`undecided` ⇒ active). The dialog is only shown while persisted state is `undecided` and +no env override exists. + +## Consent API and preview cost + +`GET /api/settings/telemetry` always returns `state`, `source`, `active`, and `preview`. The +default GET includes a preview envelope only for undecided/default consent, when the dialog can +appear; decided and environment-overridden responses return `preview: null` without running the +seven-day aggregate queries. Settings requests the same endpoint with +`include_preview=true` to fetch the current envelope on demand. `PUT /api/settings/telemetry` +persists the decision and returns `preview: null`. + +## Cadence and replica ownership + +The startup and 24-hour ticks run through the shared scheduler leader-election gate. Only the +leader constructs aggregates, transmits the snapshot, and logs the undecided-consent startup +notice. Followers perform none of that work, avoiding duplicate snapshots and duplicate notices. + +## Retention + +Each snapshot summarizes the previous seven days of existing local request logs. codex-lb does +not create a second local telemetry history or queue failed transmissions. The project-operated +collector is `https://telemetry.tokmaxxing.com`; its server-side retention duration is not yet +specified, so operators should assume transmitted snapshots remain stored until a published +retention policy or explicit deletion. + +## Failure modes + +- Endpoint down: bounded timeout (5s), at most one retry per interval, debug-level log, + proxy path untouched. Snapshot is rebuilt fresh next interval (no queue/backlog). +- Aggregation query cost: snapshot queries reuse the same 7-day aggregate shapes as the + dashboard reports module; they run on the leader scheduler once per tick and only on an API + request when the undecided dialog or an explicit settings preview needs them. On Postgres + instances with very large `request_logs` this is the same load class as one dashboard load. +- Clock skew / restart loops: the elected leader transmits the startup snapshot; SHM's + `/v1/activate` is idempotent (active → active refreshes last-seen). Rapid restart loops are + bounded by one snapshot per elected-leader process start; no local rate limiter in v1. + +## Example: privacy review quick check + +An instance with accounts `alice@corp.com` (workspace W1) + 12 others, a custom model source +`corp-internal-gpt`, and traffic from an internal tool `senpi/1.0`: + +- payload has `pool_bucket: "6-20"`, `workspace_accounts: true` +- `corp-internal-gpt` traffic appears as `models[].name == "other"` +- `senpi` traffic appears in `clients` under `other` and inflates `clients_other_ratio` +- the strings `alice`, `corp.com`, `W1`, `corp-internal-gpt`, `senpi` appear nowhere in the + serialized payload (schema snapshot test enforces this) diff --git a/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/proposal.md b/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/proposal.md new file mode 100644 index 0000000000..c68a052eaf --- /dev/null +++ b/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/proposal.md @@ -0,0 +1,77 @@ +# Add anonymous telemetry (informed opt-out) + +## Problem + +codex-lb has grown to ~2.6k stars, ~2.2k unique cloners per 14 days, and an unknown number of +running instances. The project has zero visibility into its install base: version distribution +(how many instances still run pre-1.16 with known bugs), database backend split (SQLite vs +Postgres), transport adoption (WebSocket vs HTTP bridge), deployment shape (docker/helm/pip), +client ecosystem (codex-cli vs SDK integrations), or which optional modules are actually used. +Every roadmap and deprecation decision is currently guesswork. The only existing outbound +signal is the update check in `app/modules/runtime/service.py` (GitHub releases poll), which +proves instances phone GitHub already but gives the project nothing. + +## Solution + +Add a new `telemetry` capability: an anonymous, schema-allowlisted usage snapshot sent to a +project-operated collection endpoint (self-hosted SHM server, `https://telemetry.tokmaxxing.com`) +at startup and every 24 hours. Consent is **informed opt-out**: telemetry is active by default, +every user (new and upgrading) gets a one-time dashboard dialog showing the exact JSON payload +before deciding, a persistent settings toggle, an environment variable kill switch for headless +deployments, and a startup log notice while consent is undecided. + +All payload fields are derived from data codex-lb already stores (`request_logs`, settings, +module registry). No new per-request instrumentation is added. The payload is strictly +allowlisted: raw user-agent strings, custom model names, emails, workspace IDs, IPs, prompts, +API keys, and per-account records are never transmitted. Client statistics go through a +canonical client-family mapping table (raw UA groups like `senpi` must never leave the +instance); model statistics go through the official model catalog allowlist. + +## Why this is correct as a behavior change + +- This is a new operator-visible contract (outbound network traffic + consent flow), which is + exactly the class of change OpenSpec gates. The delta spec makes the privacy allowlist + normative and testable so it cannot regress silently. +- Default-on telemetry in a privacy-sensitive user base is defensible only if the allowlist, + the payload preview, and the kill switches are hard requirements, not implementation + details. Encoding them as MUST requirements with regression tests is the mitigation. +- No existing client or operator behavior changes: proxying, routing, and dashboards are + unaffected; telemetry failure is isolated by requirement. + +## Changes + +### Spec deltas + +- `telemetry` (new capability): payload allowlist, consent state machine, one-time dialog with + exact payload preview, env kill switch, headless notice, client-family mapping table, model + catalog allowlist, random instance identity, transmission cadence + failure isolation, + bucketed sensitive aggregates. + +### Code + +- `app/modules/telemetry/` (new module) — snapshot builder (aggregation queries over + `request_logs` + settings introspection), consent state, scheduler (startup + 24h), sender + (bounded timeout, fire-and-forget). +- `app/core/config/settings.py` — `telemetry_enabled: bool | None = None` (tri-state; env + `CODEX_LB_TELEMETRY_ENABLED` maps to it), `telemetry_endpoint` (default + `https://telemetry.tokmaxxing.com`). +- `app/db/models.py` + Alembic migration — persisted consent decision + `telemetry_instance_id` + (random UUID minted on first run). +- Dashboard (frontend) — one-time consent dialog with payload preview; Settings toggle. +- `app/main.py` — scheduler wiring + undecided-consent startup notice. + +### Tests + +- Unit: payload builder allowlist (schema snapshot test — any new field fails the test until + spec updated), client-family mapping (every observed raw group → family, unknown → `other`), + model allowlist, bucket edges, consent resolution precedence (env > persisted > default). +- Integration: consent API endpoints; disabled ⇒ zero outbound calls (socket-level assert); + telemetry endpoint unreachable ⇒ proxy path unaffected. +- Migration smoke: new columns present with correct defaults. + +## Out of scope + +- The SHM collection server deployment itself (infra task, separate from this repo). +- Public aggregate dashboard / README badges (consumes collected data; follow-up). +- Any new per-request instrumentation or Prometheus metric changes. +- Crash/error report collection (stack traces are content-adjacent; deliberately excluded). diff --git a/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/specs/telemetry/spec.md b/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/specs/telemetry/spec.md new file mode 100644 index 0000000000..43389e1701 --- /dev/null +++ b/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/specs/telemetry/spec.md @@ -0,0 +1,221 @@ +# Add anonymous telemetry + +## ADDED Requirements + +### Requirement: Telemetry payload field allowlist + +The service MUST transmit only fields defined for each outbound body (registration, +activation, and snapshot envelope including its nested metrics) in this capability's +`context.md`, and MUST NOT transmit account emails, workspace identifiers, client IP +addresses, API keys, request or response content, raw user-agent strings, per-account +records, or free-text error messages in any telemetry payload. + +The snapshot metrics schema is versioned (`schema_version`). Adding a field to any transmitted +body requires a spec change to this capability; the outbound wire-schema test suite MUST fail +when registration, activation, the snapshot envelope, or nested metrics contain a field not +present in the documented schema. + +#### Scenario: Every outbound body contains only allowlisted fields + +- **WHEN** the sender serializes registration, activation, and snapshot requests +- **THEN** every top-level and nested field is present in the documented schemas, and an + outbound wire-schema regression test rejects any undeclared field in any body + +#### Scenario: Identifying data never serialized + +- **WHEN** the snapshot is built on an instance with linked accounts, API keys, and request + logs containing raw user agents and error messages +- **THEN** the serialized payload contains no email, workspace ID, IP address, API key + material, raw user-agent string, or free-text error message + +### Requirement: Consent state and default activation + +Telemetry consent MUST be a persisted tri-state (`undecided`, `enabled`, `disabled`) defaulting to `undecided`, and while consent is `undecided` the service SHALL treat telemetry as active. + +Upgrading an existing installation MUST introduce the consent state as `undecided` (existing +users get the same informed default-on treatment as new installs). + +#### Scenario: Fresh install defaults to active + +- **WHEN** codex-lb starts for the first time with no persisted consent and no environment + override +- **THEN** consent is `undecided` and telemetry snapshots are transmitted + +#### Scenario: Upgrade treats existing users as undecided + +- **WHEN** an existing installation migrates to a version with this capability +- **THEN** the migrated consent state is `undecided` and the one-time consent dialog is shown + on next dashboard entry + +### Requirement: One-time consent dialog with exact payload preview + +The dashboard MUST present a one-time consent dialog on first entry while consent is +`undecided`, and the dialog MUST display the exact snapshot envelope the instance would +transmit at that moment. Preview and sender MUST use one shared envelope constructor. The +preview timestamp MUST record preview generation time as a representative current timestamp; +the actual send MUST regenerate that value at transmission time. + +A decision (enable or disable) MUST be persisted and the dialog MUST NOT be shown again after +any decision. The dialog MUST offer disabling with no fewer clicks than enabling. + +The consent API MUST build the preview only while the undecided dialog is eligible or when an +operator explicitly requests it for the settings view. The response MUST retain the `preview` +field and set it to `null` when the preview was not requested and is not dialog-relevant. + +#### Scenario: Undecided operator sees payload preview + +- **WHEN** an operator opens the dashboard while consent is `undecided` +- **THEN** a dialog shows the live snapshot JSON with equally prominent enable and disable + actions + +#### Scenario: Decision is final until changed in settings + +- **WHEN** the operator chooses disable in the dialog +- **THEN** consent persists as `disabled`, no snapshot is transmitted afterward, and the + dialog never reappears + +#### Scenario: Decided consent status is a cheap read + +- **WHEN** the dashboard reads consent after a persisted decision without requesting a preview +- **THEN** the response contains `preview: null` and no snapshot aggregation query runs + +#### Scenario: Settings explicitly requests collected data + +- **WHEN** the settings view requests a preview for any consent state +- **THEN** the response contains a current snapshot envelope built with the same schema as the sender + +### Requirement: Settings toggle and environment kill switch + +The dashboard settings MUST expose a telemetry toggle reflecting the resolved consent state, and the environment variable `CODEX_LB_TELEMETRY_ENABLED` MUST override persisted consent when set (`false` disables all transmission, `true` enables and suppresses the consent dialog). + +#### Scenario: Headless deployment disables via environment + +- **WHEN** the service runs with `CODEX_LB_TELEMETRY_ENABLED=false` +- **THEN** no telemetry network traffic occurs regardless of persisted consent, and the + settings toggle shows telemetry as disabled by environment override + +#### Scenario: Toggle flips persisted consent + +- **WHEN** the operator disables telemetry in settings without an environment override +- **THEN** consent persists as `disabled` and transmission stops without restart + +### Requirement: Startup notice while undecided + +While consent is `undecided`, the elected leader MUST emit a single startup log line stating +that anonymous telemetry is active, where the collected-field documentation lives, and how to +disable it. Non-leader replicas MUST NOT duplicate the notice. + +#### Scenario: Headless operator is informed + +- **WHEN** the service starts with consent `undecided` +- **THEN** exactly one log line names the telemetry documentation location and the + `CODEX_LB_TELEMETRY_ENABLED=false` disable path + +### Requirement: Disabled means zero telemetry traffic + +When resolved consent is `disabled`, the service MUST NOT open any network connection to the telemetry endpoint. + +#### Scenario: No connection attempts when disabled + +- **WHEN** telemetry is disabled and the service runs through startup and a 24-hour scheduler + cycle +- **THEN** no connection attempt to the telemetry endpoint is made + +### Requirement: Client family allowlist mapping + +Telemetry client statistics MUST report only canonical client-family identifiers produced by the documented mapping table, MUST map any unmatched user-agent group to `other`, and MUST NOT transmit raw user-agent group values. + +The canonical mapping table (raw `useragent_group` → family): + +| Raw group(s) | Family | +| --- | --- | +| `codex_exec`, `codex-tui` | `codex-cli` | +| `Codex Desktop` | `codex-desktop` | +| `codex_vscode` | `codex-vscode` | +| `AsyncOpenAI` | `openai-sdk-python` | +| `OpenAI` | `openai-sdk-js` | +| `ai`, `ai-sdk` | `vercel-ai-sdk` | +| `opencode` | `opencode` | +| `Mozilla` | `browser` | +| `curl`, `undici`, `node`, `Python-urllib`, `python-requests`, `aiohttp` | `script` | +| anything else | `other` | + +The payload MUST include `clients_other_ratio` so mapping coverage decay is observable +without ever transmitting the unmatched raw values. + +#### Scenario: Private tool names never leave the instance + +- **WHEN** request logs contain a user-agent group not present in the mapping table +- **THEN** its traffic is attributed to `other` and the raw group string is absent from the + payload + +#### Scenario: Codex CLI variants collapse to one family + +- **WHEN** traffic exists from both `codex_exec` and `codex-tui` +- **THEN** the payload reports a single `codex-cli` family combining both + +### Requirement: Model catalog allowlist with per-model reasoning mix + +Telemetry model statistics MUST include only model names present in the official model catalog allowlist, MUST map unmatched model names to `other`, and MUST report reasoning-effort distribution nested per model entry rather than as an instance-global aggregate. + +#### Scenario: Custom model source names are not transmitted + +- **WHEN** an operator has configured a custom model source with a private model name +- **THEN** that traffic appears under `other` and the private name is absent from the payload + +#### Scenario: Reasoning effort is model-scoped + +- **WHEN** the snapshot reports models +- **THEN** each model entry carries its own reasoning-effort share map and no global + reasoning mix field exists + +### Requirement: Fail-honest request-family attribution + +Request-family telemetry MUST be derived only from an authoritative persisted route-family +signal. Rows without such a signal MUST be attributed to `unknown`; the service MUST NOT infer +Chat, Responses, Images, or Audio families from upstream `source` or model name. + +#### Scenario: Ambiguous persisted rows remain unknown + +- **WHEN** persisted request rows identify only workload kind, upstream source, or model name +- **THEN** their request-family share is reported as `unknown` rather than a named route family + +### Requirement: Random instance identity + +The telemetry instance identifier MUST be a UUID generated randomly on first run, MUST NOT be derived from hardware, network, account, or operating-system identity, and MUST be regenerated if deleted. + +#### Scenario: Identifier carries no fingerprint + +- **WHEN** the instance identifier is created +- **THEN** it is a random UUIDv4 persisted locally, and deleting it yields a fresh unrelated + identifier on next start + +### Requirement: Transmission cadence and failure isolation + +The service SHALL transmit one snapshot at startup and one per 24-hour interval thereafter. +In a multi-replica deployment sharing a database, snapshot construction and transmission MUST +run only under the existing leader-election gate so at most one replica performs each tick. +Telemetry transmission failures MUST NOT affect proxy operation, MUST use a bounded timeout, +MUST NOT retry more than once per interval, and MUST log failures at debug level only. + +#### Scenario: Non-leader replica skips telemetry work + +- **WHEN** a telemetry tick runs in a process that does not hold the scheduler leader lease +- **THEN** that process neither builds a snapshot nor attempts a transmission + +#### Scenario: Collection endpoint outage is invisible + +- **WHEN** the telemetry endpoint is unreachable +- **THEN** proxy requests are unaffected, startup is not delayed beyond the bounded timeout, + and no warning-or-higher log noise is produced + +### Requirement: Bucketed sensitive aggregates + +Account pool size, per-plan account counts, API key count, database size, and cost aggregates MUST be transmitted as documented buckets, never as exact values. + +An unmeasurable database size MUST be reported as `unknown`, not as a plausible size bucket. + +#### Scenario: Pool size is a bucket + +- **WHEN** an instance has 13 linked accounts +- **THEN** the payload reports the `6-20` bucket and no exact account count diff --git a/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/tasks.md b/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/tasks.md new file mode 100644 index 0000000000..6a9d385242 --- /dev/null +++ b/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/tasks.md @@ -0,0 +1,43 @@ +# Tasks + +## Implementation + +- [x] T1: `app/modules/telemetry/` module — snapshot builder aggregating `request_logs` + (7d window, reusing reports-module aggregate shapes), settings/module introspection, + bucket helpers, client-family mapping table, model catalog allowlist filter. +- [x] T2: Consent state — DB columns (`telemetry_consent`, `telemetry_instance_id`) + + Alembic migration on current main head; resolution precedence env > persisted > default. +- [x] T3: Settings — `telemetry_enabled: bool | None` (env `CODEX_LB_TELEMETRY_ENABLED`), + `telemetry_endpoint` default `https://telemetry.tokmaxxing.com`. +- [x] T4: Sender — SHM `/v1/register` + `/v1/activate` + `/v1/snapshot` client (Ed25519 + keypair per instance), 5s timeout, ≤1 retry/interval, debug-only failure logs. +- [x] T5: Scheduler — startup snapshot + 24h interval; undecided-consent startup notice + (single log line with docs link + disable instructions). +- [x] T6: Dashboard consent dialog — one-time while undecided, renders live payload JSON, + equal-prominence enable/disable; Settings toggle wired to consent API. +- [x] T7: Consent API endpoints (get resolved state, set decision). + +## Spec + +- [x] T8: Apply delta `specs/telemetry/spec.md` as new capability; sync payload schema into + `openspec/specs/telemetry/context.md`. + +## Validation + +- [x] T9: Unit — schema snapshot allowlist test (undeclared field ⇒ fail), client mapping + (all observed raw groups + unknown ⇒ `other`), model allowlist, bucket edges, consent + precedence. +- [x] T10: Integration — consent endpoints; disabled ⇒ zero outbound connections + (socket-level); endpoint unreachable ⇒ proxy unaffected. +- [x] T11: Migration smoke — new columns/defaults present (SQLite + Postgres). +- [x] T12: Privacy quick check from context.md reproduced as a test (identifying strings + absent from serialized payload). +- [x] T14: Review remediation — shared preview/sender envelope, typed allowlisted bodies, + preview-on-demand consent API, and wire-schema regressions. +- [x] T15: Review remediation — leader-gated telemetry ticks plus real lifespan wiring and + non-leader regression coverage. +- [x] T16: Review remediation — fail-honest request kinds, derived routing/client allowlists, + honest database-size failure reporting, and typed query expressions. +- [x] T17: Publish anonymous telemetry documentation and register it in the docs navigation. +- [x] T13: `openspec validate add-anonymous-telemetry` → valid; `make lint`; targeted + + broader pytest sweeps. diff --git a/openspec/changes/add-api-key-stream-fair-share/.openspec.yaml b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/.openspec.yaml similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/.openspec.yaml rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/.openspec.yaml diff --git a/openspec/changes/add-api-key-stream-fair-share/design.md b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/design.md similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/design.md rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/design.md diff --git a/openspec/changes/add-api-key-stream-fair-share/proposal.md b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/proposal.md similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/proposal.md rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/proposal.md diff --git a/openspec/changes/add-api-key-stream-fair-share/screenshots/after-per-account-capacity.png b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/screenshots/after-per-account-capacity.png similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/screenshots/after-per-account-capacity.png rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/screenshots/after-per-account-capacity.png diff --git a/openspec/changes/add-api-key-stream-fair-share/screenshots/before-per-account-capacity.png b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/screenshots/before-per-account-capacity.png similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/screenshots/before-per-account-capacity.png rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/screenshots/before-per-account-capacity.png diff --git a/openspec/changes/add-api-key-stream-fair-share/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/specs/frontend-architecture/spec.md diff --git a/openspec/changes/add-api-key-stream-fair-share/specs/proxy-admission-control/spec.md b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/specs/proxy-admission-control/spec.md similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/specs/proxy-admission-control/spec.md rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/specs/proxy-admission-control/spec.md diff --git a/openspec/changes/add-api-key-stream-fair-share/specs/proxy-runtime-observability/spec.md b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/specs/proxy-runtime-observability/spec.md similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/specs/proxy-runtime-observability/spec.md rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/specs/proxy-runtime-observability/spec.md diff --git a/openspec/changes/add-api-key-stream-fair-share/tasks.md b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/tasks.md similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/tasks.md rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/tasks.md diff --git a/openspec/changes/add-capability-aware-routing/.openspec.yaml b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/.openspec.yaml similarity index 100% rename from openspec/changes/add-capability-aware-routing/.openspec.yaml rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/.openspec.yaml diff --git a/openspec/changes/add-capability-aware-routing/design.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/design.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/design.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/design.md diff --git a/openspec/changes/add-capability-aware-routing/proposal.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/proposal.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/proposal.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/proposal.md diff --git a/openspec/changes/add-capability-aware-routing/specs/account-routing/spec.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/account-routing/spec.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/specs/account-routing/spec.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/account-routing/spec.md diff --git a/openspec/changes/add-capability-aware-routing/specs/database-migrations/spec.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/database-migrations/spec.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/specs/database-migrations/spec.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/database-migrations/spec.md diff --git a/openspec/changes/add-capability-aware-routing/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/responses-api-compat/spec.md diff --git a/openspec/changes/add-capability-aware-routing/specs/sticky-session-operations/spec.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/sticky-session-operations/spec.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/specs/sticky-session-operations/spec.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/sticky-session-operations/spec.md diff --git a/openspec/changes/add-capability-aware-routing/tasks.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/tasks.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/tasks.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/tasks.md diff --git a/openspec/changes/add-conversation-dashboard/design.md b/openspec/changes/archive/2026-08-13-add-conversation-dashboard/design.md similarity index 100% rename from openspec/changes/add-conversation-dashboard/design.md rename to openspec/changes/archive/2026-08-13-add-conversation-dashboard/design.md diff --git a/openspec/changes/add-conversation-dashboard/proposal.md b/openspec/changes/archive/2026-08-13-add-conversation-dashboard/proposal.md similarity index 100% rename from openspec/changes/add-conversation-dashboard/proposal.md rename to openspec/changes/archive/2026-08-13-add-conversation-dashboard/proposal.md diff --git a/openspec/changes/add-conversation-dashboard/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-add-conversation-dashboard/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/add-conversation-dashboard/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-add-conversation-dashboard/specs/frontend-architecture/spec.md diff --git a/openspec/changes/add-conversation-dashboard/tasks.md b/openspec/changes/archive/2026-08-13-add-conversation-dashboard/tasks.md similarity index 100% rename from openspec/changes/add-conversation-dashboard/tasks.md rename to openspec/changes/archive/2026-08-13-add-conversation-dashboard/tasks.md diff --git a/openspec/changes/add-conversation-dashboard/verify-report.md b/openspec/changes/archive/2026-08-13-add-conversation-dashboard/verify-report.md similarity index 100% rename from openspec/changes/add-conversation-dashboard/verify-report.md rename to openspec/changes/archive/2026-08-13-add-conversation-dashboard/verify-report.md diff --git a/openspec/changes/add-ko-complete-dashboard-i18n/proposal.md b/openspec/changes/archive/2026-08-13-add-ko-complete-dashboard-i18n/proposal.md similarity index 100% rename from openspec/changes/add-ko-complete-dashboard-i18n/proposal.md rename to openspec/changes/archive/2026-08-13-add-ko-complete-dashboard-i18n/proposal.md diff --git a/openspec/changes/add-ko-complete-dashboard-i18n/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-add-ko-complete-dashboard-i18n/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/add-ko-complete-dashboard-i18n/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-add-ko-complete-dashboard-i18n/specs/frontend-architecture/spec.md diff --git a/openspec/changes/add-ko-complete-dashboard-i18n/tasks.md b/openspec/changes/archive/2026-08-13-add-ko-complete-dashboard-i18n/tasks.md similarity index 100% rename from openspec/changes/add-ko-complete-dashboard-i18n/tasks.md rename to openspec/changes/archive/2026-08-13-add-ko-complete-dashboard-i18n/tasks.md diff --git a/openspec/changes/add-realtime-live-sideband/.openspec.yaml b/openspec/changes/archive/2026-08-13-add-realtime-live-sideband/.openspec.yaml similarity index 100% rename from openspec/changes/add-realtime-live-sideband/.openspec.yaml rename to openspec/changes/archive/2026-08-13-add-realtime-live-sideband/.openspec.yaml diff --git a/openspec/changes/add-realtime-live-sideband/context.md b/openspec/changes/archive/2026-08-13-add-realtime-live-sideband/context.md similarity index 100% rename from openspec/changes/add-realtime-live-sideband/context.md rename to openspec/changes/archive/2026-08-13-add-realtime-live-sideband/context.md diff --git a/openspec/changes/add-realtime-live-sideband/design.md b/openspec/changes/archive/2026-08-13-add-realtime-live-sideband/design.md similarity index 100% rename from openspec/changes/add-realtime-live-sideband/design.md rename to openspec/changes/archive/2026-08-13-add-realtime-live-sideband/design.md diff --git a/openspec/changes/add-realtime-live-sideband/proposal.md b/openspec/changes/archive/2026-08-13-add-realtime-live-sideband/proposal.md similarity index 100% rename from openspec/changes/add-realtime-live-sideband/proposal.md rename to openspec/changes/archive/2026-08-13-add-realtime-live-sideband/proposal.md diff --git a/openspec/changes/add-realtime-live-sideband/specs/realtime-api-compat/spec.md b/openspec/changes/archive/2026-08-13-add-realtime-live-sideband/specs/realtime-api-compat/spec.md similarity index 100% rename from openspec/changes/add-realtime-live-sideband/specs/realtime-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-add-realtime-live-sideband/specs/realtime-api-compat/spec.md diff --git a/openspec/changes/add-realtime-live-sideband/tasks.md b/openspec/changes/archive/2026-08-13-add-realtime-live-sideband/tasks.md similarity index 100% rename from openspec/changes/add-realtime-live-sideband/tasks.md rename to openspec/changes/archive/2026-08-13-add-realtime-live-sideband/tasks.md diff --git a/openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/proposal.md b/openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/proposal.md new file mode 100644 index 0000000000..bbe4130cc0 --- /dev/null +++ b/openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/proposal.md @@ -0,0 +1,19 @@ +## Why + +Reset-credit polling runs in every replica and issues one authenticated upstream `GET /wham/rate-limit-reset-credits` per eligible account per interval. Operators who do not use the reset-credit dashboard surface (or who run many replicas against large account fleets) currently have no way to shed that upstream call volume: the spec mandated that the scheduler always starts, and `rate_limit_reset_credits_refresh_interval_seconds` is constrained to positive values, so "off" is not expressible — stretching the interval still keeps periodic authenticated upstream traffic and the associated log/failure noise. + +## What Changes + +- Add setting `rate_limit_reset_credits_refresh_enabled` (default `true`) that gates background reset-credit polling. +- When disabled, the scheduler's `start()` is a no-op: no background task is created, no upstream fetches occur, and snapshot caches simply stay empty (dashboard reads already handle a missing snapshot as `null`/`0`). +- Default `true` preserves current zero-config behavior; nothing changes for existing deployments. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `rate-limit-reset-credits`: The scheduler starts with the application lifespan only when reset-credit polling is enabled, and the settings surface gains an enable/disable toggle alongside the existing interval control. diff --git a/openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/specs/rate-limit-reset-credits/spec.md b/openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/specs/rate-limit-reset-credits/spec.md new file mode 100644 index 0000000000..d1b7f1267c --- /dev/null +++ b/openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/specs/rate-limit-reset-credits/spec.md @@ -0,0 +1,59 @@ +## MODIFIED Requirements + +### Requirement: Reset credits are polled per account on a fixed cadence + +The system SHALL poll upstream `GET /wham/rate-limit-reset-credits` for each eligible account on a configurable cadence that defaults to 60 seconds, using that account's stored OAuth bearer token and `chatgpt-account-id`. The scheduler SHALL start with the application lifespan when reset-credit polling is enabled. Because snapshots are kept in process-local memory, every running replica SHALL refresh its own snapshot cache instead of relying on leader election, and the scheduler SHALL NOT be leader-gated while snapshots remain process-local. Each replica SHALL apply a randomized startup delay of up to one full interval and randomized per-tick jitter of +/-10% so replica ticks are desynchronized. The aggregate upstream fetch rate scales with the number of running replicas; `rate_limit_reset_credits_refresh_interval_seconds` is the operator control for total upstream load. The poll SHALL skip any account that is paused, requires reauthentication, deactivated, or lacks a usable `chatgpt-account-id`. + +#### Scenario: Default cadence polls every 60 seconds +- **WHEN** the application starts with default settings +- **THEN** each eligible account's credits are fetched from upstream at most once per 60 seconds plus the jitter bound + +#### Scenario: Every replica refreshes its local cache +- **WHEN** the application is deployed with multiple running replicas +- **THEN** each replica refreshes its own in-memory reset-credit snapshots on the configured cadence +- **AND** dashboard reads served by any replica can observe populated reset-credit data after that replica's refresh tick + +#### Scenario: Two replicas do not fetch in lockstep +- **GIVEN** two replicas start with identical configuration +- **WHEN** their refresh loops run +- **THEN** their startup delays are independent uniform draws over the full interval and each tick interval carries independent +/-10% jitter, so the replicas' tick times are not synchronized + +#### Scenario: Ineligible accounts are skipped +- **WHEN** an account is persisted as `paused`, `reauth_required`, or `deactivated` +- **THEN** the scheduler performs no upstream reset-credits fetch for that account +- **AND** the cached snapshot for that account (if any) is left untouched by the skip + +### Requirement: Reset credit polling interval is configurable + +The system SHALL expose setting `rate_limit_reset_credits_refresh_interval_seconds` (default `60`) to control the polling cadence. The system SHALL expose setting `rate_limit_reset_credits_refresh_enabled` (default `true`) to enable or disable background reset-credit polling. Because the refresh loop is the sole driver of automatic reset-credit redemption, disabling background polling SHALL also disable automatic redemption; when polling is disabled while the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is enabled, the system SHALL log a configuration-conflict warning at startup naming both settings. While polling is disabled, the dashboard settings update SHALL reject a request that newly enables `auto_redeem_reset_credits_before_expiry` with a bad-request error naming the polling toggle; an already-persisted opt-in SHALL remain readable and re-savable so unrelated settings edits are not blocked. + +#### Scenario: Operator tunes the polling interval +- **GIVEN** `rate_limit_reset_credits_refresh_interval_seconds` is set to `120` +- **WHEN** the application starts and runs +- **THEN** each eligible account's credits are fetched from upstream at most once per 120 seconds + +#### Scenario: Operator disables background polling +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **WHEN** the application starts +- **THEN** the reset-credit polling scheduler does not create a background polling task +- **AND** no upstream reset-credits fetches occur + +#### Scenario: Disabled polling conflicts with persisted auto-redeem opt-in +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **AND** the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is `true` +- **WHEN** the application starts +- **THEN** the system logs a configuration-conflict warning naming both settings +- **AND** no automatic reset-credit redemption occurs while polling remains disabled + +#### Scenario: Auto-redeem opt-in is rejected while polling is disabled +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **AND** the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is `false` +- **WHEN** a dashboard settings update sets `auto_redeem_reset_credits_before_expiry` to `true` +- **THEN** the update is rejected with a bad-request error naming the polling toggle +- **AND** the persisted setting remains `false` + +#### Scenario: Persisted auto-redeem does not block unrelated settings edits +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **AND** the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is already `true` +- **WHEN** a full settings payload that keeps the opt-in unchanged is submitted +- **THEN** the update succeeds diff --git a/openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/tasks.md b/openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/tasks.md new file mode 100644 index 0000000000..c7fa80f043 --- /dev/null +++ b/openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/tasks.md @@ -0,0 +1,20 @@ +## 1. Settings and scheduler gate + +- [x] 1.1 Add `rate_limit_reset_credits_refresh_enabled: bool = True` to `app/core/config/settings.py` next to the existing interval setting +- [x] 1.2 Add `enabled: bool = True` to `RateLimitResetCreditsRefreshScheduler` and make `start()` a no-op when disabled; wire the setting through `build_rate_limit_reset_credits_scheduler()` +- [x] 1.3 On disabled `start()`, read the persisted dashboard settings and log a configuration-conflict warning when `auto_redeem_reset_credits_before_expiry` is enabled (the refresh loop is the sole auto-redeem driver) + +## 2. Tests + +- [x] 2.1 Unit-test that `start()` creates no task when disabled and creates the loop task when enabled +- [x] 2.2 Unit-test that the factory wires `rate_limit_reset_credits_refresh_enabled` from settings +- [x] 2.3 Unit-test the disabled+auto-redeem conflict warning (warns when persisted opt-in is true, stays silent when false) +- [x] 2.4 Route-level integration tests: PUT rejecting a new auto-redeem opt-in while polling is disabled (`reset_credit_polling_disabled`), and a full PUT with an already-persisted opt-in still succeeding + +## 3.5 Settings API guard + +- [x] 3.5.1 Reject a new `auto_redeem_reset_credits_before_expiry` opt-in in `app/modules/settings/api.py` while polling is disabled; keep already-persisted opt-ins re-savable + +## 3. Spec + +- [x] 3.1 Update the `rate-limit-reset-credits` delta: scheduler starts with the lifespan when polling is enabled; settings expose the toggle with default `true` diff --git a/openspec/changes/add-retention-zero-warning-presets/.openspec.yaml b/openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/.openspec.yaml similarity index 100% rename from openspec/changes/add-retention-zero-warning-presets/.openspec.yaml rename to openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/.openspec.yaml diff --git a/openspec/changes/add-retention-zero-warning-presets/design.md b/openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/design.md similarity index 100% rename from openspec/changes/add-retention-zero-warning-presets/design.md rename to openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/design.md diff --git a/openspec/changes/add-retention-zero-warning-presets/proposal.md b/openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/proposal.md similarity index 100% rename from openspec/changes/add-retention-zero-warning-presets/proposal.md rename to openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/proposal.md diff --git a/openspec/changes/add-retention-zero-warning-presets/specs/data-retention/spec.md b/openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/specs/data-retention/spec.md similarity index 100% rename from openspec/changes/add-retention-zero-warning-presets/specs/data-retention/spec.md rename to openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/specs/data-retention/spec.md diff --git a/openspec/changes/add-retention-zero-warning-presets/tasks.md b/openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/tasks.md similarity index 100% rename from openspec/changes/add-retention-zero-warning-presets/tasks.md rename to openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/tasks.md diff --git a/openspec/changes/add-stale-anchor-metadata/.openspec.yaml b/openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/.openspec.yaml similarity index 100% rename from openspec/changes/add-stale-anchor-metadata/.openspec.yaml rename to openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/.openspec.yaml diff --git a/openspec/changes/add-stale-anchor-metadata/design.md b/openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/design.md similarity index 100% rename from openspec/changes/add-stale-anchor-metadata/design.md rename to openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/design.md diff --git a/openspec/changes/add-stale-anchor-metadata/proposal.md b/openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/proposal.md similarity index 100% rename from openspec/changes/add-stale-anchor-metadata/proposal.md rename to openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/proposal.md diff --git a/openspec/changes/add-stale-anchor-metadata/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/add-stale-anchor-metadata/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/specs/responses-api-compat/spec.md diff --git a/openspec/changes/add-stale-anchor-metadata/tasks.md b/openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/tasks.md similarity index 100% rename from openspec/changes/add-stale-anchor-metadata/tasks.md rename to openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/tasks.md diff --git a/openspec/changes/add-upstream-route-cache/.openspec.yaml b/openspec/changes/archive/2026-08-13-add-upstream-route-cache/.openspec.yaml similarity index 100% rename from openspec/changes/add-upstream-route-cache/.openspec.yaml rename to openspec/changes/archive/2026-08-13-add-upstream-route-cache/.openspec.yaml diff --git a/openspec/changes/add-upstream-route-cache/design.md b/openspec/changes/archive/2026-08-13-add-upstream-route-cache/design.md similarity index 100% rename from openspec/changes/add-upstream-route-cache/design.md rename to openspec/changes/archive/2026-08-13-add-upstream-route-cache/design.md diff --git a/openspec/changes/add-upstream-route-cache/proposal.md b/openspec/changes/archive/2026-08-13-add-upstream-route-cache/proposal.md similarity index 100% rename from openspec/changes/add-upstream-route-cache/proposal.md rename to openspec/changes/archive/2026-08-13-add-upstream-route-cache/proposal.md diff --git a/openspec/changes/add-upstream-route-cache/specs/query-caching/spec.md b/openspec/changes/archive/2026-08-13-add-upstream-route-cache/specs/query-caching/spec.md similarity index 100% rename from openspec/changes/add-upstream-route-cache/specs/query-caching/spec.md rename to openspec/changes/archive/2026-08-13-add-upstream-route-cache/specs/query-caching/spec.md diff --git a/openspec/changes/add-upstream-route-cache/specs/upstream-proxy-routing/spec.md b/openspec/changes/archive/2026-08-13-add-upstream-route-cache/specs/upstream-proxy-routing/spec.md similarity index 100% rename from openspec/changes/add-upstream-route-cache/specs/upstream-proxy-routing/spec.md rename to openspec/changes/archive/2026-08-13-add-upstream-route-cache/specs/upstream-proxy-routing/spec.md diff --git a/openspec/changes/add-upstream-route-cache/tasks.md b/openspec/changes/archive/2026-08-13-add-upstream-route-cache/tasks.md similarity index 100% rename from openspec/changes/add-upstream-route-cache/tasks.md rename to openspec/changes/archive/2026-08-13-add-upstream-route-cache/tasks.md diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/.openspec.yaml b/openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/.openspec.yaml similarity index 100% rename from openspec/changes/allow-developer-interleaved-fresh-resend/.openspec.yaml rename to openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/.openspec.yaml diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/context.md b/openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/context.md similarity index 100% rename from openspec/changes/allow-developer-interleaved-fresh-resend/context.md rename to openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/context.md diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/proposal.md b/openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/proposal.md similarity index 100% rename from openspec/changes/allow-developer-interleaved-fresh-resend/proposal.md rename to openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/proposal.md diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/tasks.md b/openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/tasks.md similarity index 100% rename from openspec/changes/allow-developer-interleaved-fresh-resend/tasks.md rename to openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/tasks.md diff --git a/openspec/changes/attribute-bridge-failure-request-logs/proposal.md b/openspec/changes/archive/2026-08-13-attribute-bridge-failure-request-logs/proposal.md similarity index 100% rename from openspec/changes/attribute-bridge-failure-request-logs/proposal.md rename to openspec/changes/archive/2026-08-13-attribute-bridge-failure-request-logs/proposal.md diff --git a/openspec/changes/attribute-bridge-failure-request-logs/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-attribute-bridge-failure-request-logs/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/attribute-bridge-failure-request-logs/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-attribute-bridge-failure-request-logs/specs/api-keys/spec.md diff --git a/openspec/changes/attribute-bridge-failure-request-logs/tasks.md b/openspec/changes/archive/2026-08-13-attribute-bridge-failure-request-logs/tasks.md similarity index 100% rename from openspec/changes/attribute-bridge-failure-request-logs/tasks.md rename to openspec/changes/archive/2026-08-13-attribute-bridge-failure-request-logs/tasks.md diff --git a/openspec/changes/backoff-codex-review-usage-limits/proposal.md b/openspec/changes/archive/2026-08-13-backoff-codex-review-usage-limits/proposal.md similarity index 100% rename from openspec/changes/backoff-codex-review-usage-limits/proposal.md rename to openspec/changes/archive/2026-08-13-backoff-codex-review-usage-limits/proposal.md diff --git a/openspec/changes/backoff-codex-review-usage-limits/specs/github-automation/spec.md b/openspec/changes/archive/2026-08-13-backoff-codex-review-usage-limits/specs/github-automation/spec.md similarity index 100% rename from openspec/changes/backoff-codex-review-usage-limits/specs/github-automation/spec.md rename to openspec/changes/archive/2026-08-13-backoff-codex-review-usage-limits/specs/github-automation/spec.md diff --git a/openspec/changes/backoff-codex-review-usage-limits/tasks.md b/openspec/changes/archive/2026-08-13-backoff-codex-review-usage-limits/tasks.md similarity index 100% rename from openspec/changes/backoff-codex-review-usage-limits/tasks.md rename to openspec/changes/archive/2026-08-13-backoff-codex-review-usage-limits/tasks.md diff --git a/openspec/changes/bound-multipart-uploads/.openspec.yaml b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/.openspec.yaml similarity index 100% rename from openspec/changes/bound-multipart-uploads/.openspec.yaml rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/.openspec.yaml diff --git a/openspec/changes/bound-multipart-uploads/design.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/design.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/design.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/design.md diff --git a/openspec/changes/bound-multipart-uploads/proposal.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/proposal.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/proposal.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/proposal.md diff --git a/openspec/changes/bound-multipart-uploads/specs/account-import/spec.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/account-import/spec.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/specs/account-import/spec.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/account-import/spec.md diff --git a/openspec/changes/bound-multipart-uploads/specs/audio-transcriptions-compat/spec.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/audio-transcriptions-compat/spec.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/specs/audio-transcriptions-compat/spec.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/audio-transcriptions-compat/spec.md diff --git a/openspec/changes/bound-multipart-uploads/specs/http-ingress-limits/spec.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/http-ingress-limits/spec.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/specs/http-ingress-limits/spec.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/http-ingress-limits/spec.md diff --git a/openspec/changes/bound-multipart-uploads/specs/images-api-compat/spec.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/images-api-compat/spec.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/specs/images-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/images-api-compat/spec.md diff --git a/openspec/changes/bound-multipart-uploads/tasks.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/tasks.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/tasks.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/tasks.md diff --git a/openspec/changes/bound-rate-limit-reset-metadata/.openspec.yaml b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/.openspec.yaml similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/.openspec.yaml rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/.openspec.yaml diff --git a/openspec/changes/bound-rate-limit-reset-metadata/context.md b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/context.md similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/context.md rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/context.md diff --git a/openspec/changes/bound-rate-limit-reset-metadata/design.md b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/design.md similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/design.md rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/design.md diff --git a/openspec/changes/bound-rate-limit-reset-metadata/proposal.md b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/proposal.md similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/proposal.md rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/proposal.md diff --git a/openspec/changes/bound-rate-limit-reset-metadata/specs/account-routing/spec.md b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/specs/account-routing/spec.md similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/specs/account-routing/spec.md rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/specs/account-routing/spec.md diff --git a/openspec/changes/bound-rate-limit-reset-metadata/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/bound-rate-limit-reset-metadata/tasks.md b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/tasks.md similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/tasks.md rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/tasks.md diff --git a/openspec/changes/bound-raw-http-ingress/.openspec.yaml b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/.openspec.yaml similarity index 100% rename from openspec/changes/bound-raw-http-ingress/.openspec.yaml rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/.openspec.yaml diff --git a/openspec/changes/bound-raw-http-ingress/context.md b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/context.md similarity index 100% rename from openspec/changes/bound-raw-http-ingress/context.md rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/context.md diff --git a/openspec/changes/bound-raw-http-ingress/design.md b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/design.md similarity index 100% rename from openspec/changes/bound-raw-http-ingress/design.md rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/design.md diff --git a/openspec/changes/bound-raw-http-ingress/proposal.md b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/proposal.md similarity index 100% rename from openspec/changes/bound-raw-http-ingress/proposal.md rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/proposal.md diff --git a/openspec/changes/bound-raw-http-ingress/specs/http-ingress-limits/spec.md b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/specs/http-ingress-limits/spec.md similarity index 100% rename from openspec/changes/bound-raw-http-ingress/specs/http-ingress-limits/spec.md rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/specs/http-ingress-limits/spec.md diff --git a/openspec/changes/bound-raw-http-ingress/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/bound-raw-http-ingress/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/specs/responses-api-compat/spec.md diff --git a/openspec/changes/bound-raw-http-ingress/tasks.md b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/tasks.md similarity index 100% rename from openspec/changes/bound-raw-http-ingress/tasks.md rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/tasks.md diff --git a/openspec/changes/cache-request-log-count/.openspec.yaml b/openspec/changes/archive/2026-08-13-cache-request-log-count/.openspec.yaml similarity index 100% rename from openspec/changes/cache-request-log-count/.openspec.yaml rename to openspec/changes/archive/2026-08-13-cache-request-log-count/.openspec.yaml diff --git a/openspec/changes/cache-request-log-count/proposal.md b/openspec/changes/archive/2026-08-13-cache-request-log-count/proposal.md similarity index 100% rename from openspec/changes/cache-request-log-count/proposal.md rename to openspec/changes/archive/2026-08-13-cache-request-log-count/proposal.md diff --git a/openspec/changes/cache-request-log-count/specs/query-caching/spec.md b/openspec/changes/archive/2026-08-13-cache-request-log-count/specs/query-caching/spec.md similarity index 100% rename from openspec/changes/cache-request-log-count/specs/query-caching/spec.md rename to openspec/changes/archive/2026-08-13-cache-request-log-count/specs/query-caching/spec.md diff --git a/openspec/changes/cache-request-log-count/tasks.md b/openspec/changes/archive/2026-08-13-cache-request-log-count/tasks.md similarity index 100% rename from openspec/changes/cache-request-log-count/tasks.md rename to openspec/changes/archive/2026-08-13-cache-request-log-count/tasks.md diff --git a/openspec/changes/classify-tool-search-missing-tool-output/proposal.md b/openspec/changes/archive/2026-08-13-classify-tool-search-missing-tool-output/proposal.md similarity index 100% rename from openspec/changes/classify-tool-search-missing-tool-output/proposal.md rename to openspec/changes/archive/2026-08-13-classify-tool-search-missing-tool-output/proposal.md diff --git a/openspec/changes/classify-tool-search-missing-tool-output/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-classify-tool-search-missing-tool-output/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/classify-tool-search-missing-tool-output/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-classify-tool-search-missing-tool-output/specs/responses-api-compat/spec.md diff --git a/openspec/changes/classify-tool-search-missing-tool-output/tasks.md b/openspec/changes/archive/2026-08-13-classify-tool-search-missing-tool-output/tasks.md similarity index 100% rename from openspec/changes/classify-tool-search-missing-tool-output/tasks.md rename to openspec/changes/archive/2026-08-13-classify-tool-search-missing-tool-output/tasks.md diff --git a/openspec/changes/close-sqlite-file-handles/.openspec.yaml b/openspec/changes/archive/2026-08-13-close-sqlite-file-handles/.openspec.yaml similarity index 100% rename from openspec/changes/close-sqlite-file-handles/.openspec.yaml rename to openspec/changes/archive/2026-08-13-close-sqlite-file-handles/.openspec.yaml diff --git a/openspec/changes/close-sqlite-file-handles/design.md b/openspec/changes/archive/2026-08-13-close-sqlite-file-handles/design.md similarity index 100% rename from openspec/changes/close-sqlite-file-handles/design.md rename to openspec/changes/archive/2026-08-13-close-sqlite-file-handles/design.md diff --git a/openspec/changes/close-sqlite-file-handles/proposal.md b/openspec/changes/archive/2026-08-13-close-sqlite-file-handles/proposal.md similarity index 100% rename from openspec/changes/close-sqlite-file-handles/proposal.md rename to openspec/changes/archive/2026-08-13-close-sqlite-file-handles/proposal.md diff --git a/openspec/changes/close-sqlite-file-handles/specs/database-migrations/spec.md b/openspec/changes/archive/2026-08-13-close-sqlite-file-handles/specs/database-migrations/spec.md similarity index 100% rename from openspec/changes/close-sqlite-file-handles/specs/database-migrations/spec.md rename to openspec/changes/archive/2026-08-13-close-sqlite-file-handles/specs/database-migrations/spec.md diff --git a/openspec/changes/close-sqlite-file-handles/tasks.md b/openspec/changes/archive/2026-08-13-close-sqlite-file-handles/tasks.md similarity index 100% rename from openspec/changes/close-sqlite-file-handles/tasks.md rename to openspec/changes/archive/2026-08-13-close-sqlite-file-handles/tasks.md diff --git a/openspec/changes/complete-zh-cn-dashboard-i18n/context.md b/openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/context.md similarity index 100% rename from openspec/changes/complete-zh-cn-dashboard-i18n/context.md rename to openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/context.md diff --git a/openspec/changes/complete-zh-cn-dashboard-i18n/proposal.md b/openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/proposal.md similarity index 100% rename from openspec/changes/complete-zh-cn-dashboard-i18n/proposal.md rename to openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/proposal.md diff --git a/openspec/changes/complete-zh-cn-dashboard-i18n/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/complete-zh-cn-dashboard-i18n/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/specs/frontend-architecture/spec.md diff --git a/openspec/changes/complete-zh-cn-dashboard-i18n/tasks.md b/openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/tasks.md similarity index 100% rename from openspec/changes/complete-zh-cn-dashboard-i18n/tasks.md rename to openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/tasks.md diff --git a/openspec/changes/configure-gateway-api-rules/.openspec.yaml b/openspec/changes/archive/2026-08-13-configure-gateway-api-rules/.openspec.yaml similarity index 100% rename from openspec/changes/configure-gateway-api-rules/.openspec.yaml rename to openspec/changes/archive/2026-08-13-configure-gateway-api-rules/.openspec.yaml diff --git a/openspec/changes/configure-gateway-api-rules/proposal.md b/openspec/changes/archive/2026-08-13-configure-gateway-api-rules/proposal.md similarity index 100% rename from openspec/changes/configure-gateway-api-rules/proposal.md rename to openspec/changes/archive/2026-08-13-configure-gateway-api-rules/proposal.md diff --git a/openspec/changes/configure-gateway-api-rules/specs/deployment-networking/spec.md b/openspec/changes/archive/2026-08-13-configure-gateway-api-rules/specs/deployment-networking/spec.md similarity index 100% rename from openspec/changes/configure-gateway-api-rules/specs/deployment-networking/spec.md rename to openspec/changes/archive/2026-08-13-configure-gateway-api-rules/specs/deployment-networking/spec.md diff --git a/openspec/changes/configure-gateway-api-rules/tasks.md b/openspec/changes/archive/2026-08-13-configure-gateway-api-rules/tasks.md similarity index 100% rename from openspec/changes/configure-gateway-api-rules/tasks.md rename to openspec/changes/archive/2026-08-13-configure-gateway-api-rules/tasks.md diff --git a/openspec/changes/configure-grafana-dashboard-titles/.openspec.yaml b/openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/.openspec.yaml similarity index 100% rename from openspec/changes/configure-grafana-dashboard-titles/.openspec.yaml rename to openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/.openspec.yaml diff --git a/openspec/changes/configure-grafana-dashboard-titles/proposal.md b/openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/proposal.md similarity index 100% rename from openspec/changes/configure-grafana-dashboard-titles/proposal.md rename to openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/proposal.md diff --git a/openspec/changes/configure-grafana-dashboard-titles/specs/deployment-installation/spec.md b/openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/specs/deployment-installation/spec.md similarity index 100% rename from openspec/changes/configure-grafana-dashboard-titles/specs/deployment-installation/spec.md rename to openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/specs/deployment-installation/spec.md diff --git a/openspec/changes/configure-grafana-dashboard-titles/tasks.md b/openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/tasks.md similarity index 100% rename from openspec/changes/configure-grafana-dashboard-titles/tasks.md rename to openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/tasks.md diff --git a/openspec/changes/conversation-list-metrics/design.md b/openspec/changes/archive/2026-08-13-conversation-list-metrics/design.md similarity index 100% rename from openspec/changes/conversation-list-metrics/design.md rename to openspec/changes/archive/2026-08-13-conversation-list-metrics/design.md diff --git a/openspec/changes/conversation-list-metrics/proposal.md b/openspec/changes/archive/2026-08-13-conversation-list-metrics/proposal.md similarity index 100% rename from openspec/changes/conversation-list-metrics/proposal.md rename to openspec/changes/archive/2026-08-13-conversation-list-metrics/proposal.md diff --git a/openspec/changes/conversation-list-metrics/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-conversation-list-metrics/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/conversation-list-metrics/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-conversation-list-metrics/specs/frontend-architecture/spec.md diff --git a/openspec/changes/conversation-list-metrics/tasks.md b/openspec/changes/archive/2026-08-13-conversation-list-metrics/tasks.md similarity index 100% rename from openspec/changes/conversation-list-metrics/tasks.md rename to openspec/changes/archive/2026-08-13-conversation-list-metrics/tasks.md diff --git a/openspec/changes/create-application-gateway/.openspec.yaml b/openspec/changes/archive/2026-08-13-create-application-gateway/.openspec.yaml similarity index 100% rename from openspec/changes/create-application-gateway/.openspec.yaml rename to openspec/changes/archive/2026-08-13-create-application-gateway/.openspec.yaml diff --git a/openspec/changes/create-application-gateway/proposal.md b/openspec/changes/archive/2026-08-13-create-application-gateway/proposal.md similarity index 100% rename from openspec/changes/create-application-gateway/proposal.md rename to openspec/changes/archive/2026-08-13-create-application-gateway/proposal.md diff --git a/openspec/changes/create-application-gateway/specs/deployment-networking/spec.md b/openspec/changes/archive/2026-08-13-create-application-gateway/specs/deployment-networking/spec.md similarity index 100% rename from openspec/changes/create-application-gateway/specs/deployment-networking/spec.md rename to openspec/changes/archive/2026-08-13-create-application-gateway/specs/deployment-networking/spec.md diff --git a/openspec/changes/create-application-gateway/tasks.md b/openspec/changes/archive/2026-08-13-create-application-gateway/tasks.md similarity index 100% rename from openspec/changes/create-application-gateway/tasks.md rename to openspec/changes/archive/2026-08-13-create-application-gateway/tasks.md diff --git a/openspec/changes/customize-external-secret-refs/proposal.md b/openspec/changes/archive/2026-08-13-customize-external-secret-refs/proposal.md similarity index 100% rename from openspec/changes/customize-external-secret-refs/proposal.md rename to openspec/changes/archive/2026-08-13-customize-external-secret-refs/proposal.md diff --git a/openspec/changes/customize-external-secret-refs/specs/deployment-installation/spec.md b/openspec/changes/archive/2026-08-13-customize-external-secret-refs/specs/deployment-installation/spec.md similarity index 100% rename from openspec/changes/customize-external-secret-refs/specs/deployment-installation/spec.md rename to openspec/changes/archive/2026-08-13-customize-external-secret-refs/specs/deployment-installation/spec.md diff --git a/openspec/changes/customize-external-secret-refs/tasks.md b/openspec/changes/archive/2026-08-13-customize-external-secret-refs/tasks.md similarity index 100% rename from openspec/changes/customize-external-secret-refs/tasks.md rename to openspec/changes/archive/2026-08-13-customize-external-secret-refs/tasks.md diff --git a/openspec/changes/date-display-format-setting/.openspec.yaml b/openspec/changes/archive/2026-08-13-date-display-format-setting/.openspec.yaml similarity index 100% rename from openspec/changes/date-display-format-setting/.openspec.yaml rename to openspec/changes/archive/2026-08-13-date-display-format-setting/.openspec.yaml diff --git a/openspec/changes/date-display-format-setting/design.md b/openspec/changes/archive/2026-08-13-date-display-format-setting/design.md similarity index 100% rename from openspec/changes/date-display-format-setting/design.md rename to openspec/changes/archive/2026-08-13-date-display-format-setting/design.md diff --git a/openspec/changes/date-display-format-setting/proposal.md b/openspec/changes/archive/2026-08-13-date-display-format-setting/proposal.md similarity index 100% rename from openspec/changes/date-display-format-setting/proposal.md rename to openspec/changes/archive/2026-08-13-date-display-format-setting/proposal.md diff --git a/openspec/changes/date-display-format-setting/specs/date-display-format/spec.md b/openspec/changes/archive/2026-08-13-date-display-format-setting/specs/date-display-format/spec.md similarity index 100% rename from openspec/changes/date-display-format-setting/specs/date-display-format/spec.md rename to openspec/changes/archive/2026-08-13-date-display-format-setting/specs/date-display-format/spec.md diff --git a/openspec/changes/date-display-format-setting/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-date-display-format-setting/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/date-display-format-setting/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-date-display-format-setting/specs/frontend-architecture/spec.md diff --git a/openspec/changes/date-display-format-setting/tasks.md b/openspec/changes/archive/2026-08-13-date-display-format-setting/tasks.md similarity index 100% rename from openspec/changes/date-display-format-setting/tasks.md rename to openspec/changes/archive/2026-08-13-date-display-format-setting/tasks.md diff --git a/openspec/changes/dedup-and-cap-response-create-dumps/.openspec.yaml b/openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/.openspec.yaml similarity index 100% rename from openspec/changes/dedup-and-cap-response-create-dumps/.openspec.yaml rename to openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/.openspec.yaml diff --git a/openspec/changes/dedup-and-cap-response-create-dumps/design.md b/openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/design.md similarity index 100% rename from openspec/changes/dedup-and-cap-response-create-dumps/design.md rename to openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/design.md diff --git a/openspec/changes/dedup-and-cap-response-create-dumps/proposal.md b/openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/proposal.md similarity index 100% rename from openspec/changes/dedup-and-cap-response-create-dumps/proposal.md rename to openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/proposal.md diff --git a/openspec/changes/dedup-and-cap-response-create-dumps/specs/deployment-installation/spec.md b/openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/specs/deployment-installation/spec.md similarity index 100% rename from openspec/changes/dedup-and-cap-response-create-dumps/specs/deployment-installation/spec.md rename to openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/specs/deployment-installation/spec.md diff --git a/openspec/changes/dedup-and-cap-response-create-dumps/tasks.md b/openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/tasks.md similarity index 100% rename from openspec/changes/dedup-and-cap-response-create-dumps/tasks.md rename to openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/tasks.md diff --git a/openspec/changes/drain-active-websocket-turns/.openspec.yaml b/openspec/changes/archive/2026-08-13-drain-active-websocket-turns/.openspec.yaml similarity index 100% rename from openspec/changes/drain-active-websocket-turns/.openspec.yaml rename to openspec/changes/archive/2026-08-13-drain-active-websocket-turns/.openspec.yaml diff --git a/openspec/changes/drain-active-websocket-turns/design.md b/openspec/changes/archive/2026-08-13-drain-active-websocket-turns/design.md similarity index 100% rename from openspec/changes/drain-active-websocket-turns/design.md rename to openspec/changes/archive/2026-08-13-drain-active-websocket-turns/design.md diff --git a/openspec/changes/drain-active-websocket-turns/proposal.md b/openspec/changes/archive/2026-08-13-drain-active-websocket-turns/proposal.md similarity index 100% rename from openspec/changes/drain-active-websocket-turns/proposal.md rename to openspec/changes/archive/2026-08-13-drain-active-websocket-turns/proposal.md diff --git a/openspec/changes/drain-active-websocket-turns/specs/deployment-installation/spec.md b/openspec/changes/archive/2026-08-13-drain-active-websocket-turns/specs/deployment-installation/spec.md similarity index 100% rename from openspec/changes/drain-active-websocket-turns/specs/deployment-installation/spec.md rename to openspec/changes/archive/2026-08-13-drain-active-websocket-turns/specs/deployment-installation/spec.md diff --git a/openspec/changes/drain-active-websocket-turns/specs/graceful-shutdown/spec.md b/openspec/changes/archive/2026-08-13-drain-active-websocket-turns/specs/graceful-shutdown/spec.md similarity index 100% rename from openspec/changes/drain-active-websocket-turns/specs/graceful-shutdown/spec.md rename to openspec/changes/archive/2026-08-13-drain-active-websocket-turns/specs/graceful-shutdown/spec.md diff --git a/openspec/changes/drain-active-websocket-turns/tasks.md b/openspec/changes/archive/2026-08-13-drain-active-websocket-turns/tasks.md similarity index 100% rename from openspec/changes/drain-active-websocket-turns/tasks.md rename to openspec/changes/archive/2026-08-13-drain-active-websocket-turns/tasks.md diff --git a/openspec/changes/drain-audit-fleet-tasks/.openspec.yaml b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/.openspec.yaml similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/.openspec.yaml rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/.openspec.yaml diff --git a/openspec/changes/drain-audit-fleet-tasks/context.md b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/context.md similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/context.md rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/context.md diff --git a/openspec/changes/drain-audit-fleet-tasks/design.md b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/design.md similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/design.md rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/design.md diff --git a/openspec/changes/drain-audit-fleet-tasks/proposal.md b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/proposal.md similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/proposal.md rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/proposal.md diff --git a/openspec/changes/drain-audit-fleet-tasks/specs/audit-logging/spec.md b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/specs/audit-logging/spec.md similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/specs/audit-logging/spec.md rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/specs/audit-logging/spec.md diff --git a/openspec/changes/drain-audit-fleet-tasks/specs/fleet-summary/spec.md b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/specs/fleet-summary/spec.md similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/specs/fleet-summary/spec.md rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/specs/fleet-summary/spec.md diff --git a/openspec/changes/drain-audit-fleet-tasks/tasks.md b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/tasks.md similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/tasks.md rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/tasks.md diff --git a/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/.openspec.yaml b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/.openspec.yaml new file mode 100644 index 0000000000..878dc3156e --- /dev/null +++ b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-07 diff --git a/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/context.md b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/context.md new file mode 100644 index 0000000000..1e1128e906 --- /dev/null +++ b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/context.md @@ -0,0 +1,41 @@ +## Context + +The bridge has no upstream idempotency/status endpoint, so an ambiguous turn +must be fenced locally. The durable operation row is the local proof. SQLite +and PostgreSQL deployments must therefore retain that row and its session while +another instance takes ownership. + +## Decisions + +- Scope the fingerprint hash with the normalized API-key scope instead of + changing the public request contract or exposing a new database column. +- Treat submitted, acknowledged, and unknown operations as recoverable; only + terminal rows may be removed by normal retention. +- Clear event rows and byte accounting in the same transaction that rebinds a + failed operation, so a retry has a fresh transcript. +- Require a matching fingerprint before using a completed sibling as a new + continuation anchor. A different request remains attached to its requested + parent. +- Use a no-op Alembic merge revision to converge the operation-ledger branch + with additive migrations already present on main. +- Treat the event spool as incomplete until the asynchronous batcher drains it; + SQLite's table default is rebuilt explicitly because SQLite does not support + a direct ALTER COLUMN operation. +- Run transcript retention from the existing leader-gated cleanup loop, + draining bounded repository batches without adding a new scheduler process; + transcript retention remains active when sticky mapping cleanup is disabled. +- Reset partial operation events before server-owned indefinite retries and + persist deferred reasoning blocks before the visible block they precede. + +## Failure modes + +- If durable persistence is unavailable, the bridge remains fail-closed rather + than dispatching an untracked duplicate. +- If a transcript is incomplete, recovery may not replay it; the existing + bounded retry policy remains authoritative. + +## Example + +Two API keys submit identical JSON against the same parent response. Their +normalized scopes produce distinct operation fingerprints, so neither request +can consume the other's completion or event spool. diff --git a/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/proposal.md b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/proposal.md new file mode 100644 index 0000000000..f99b2e901c --- /dev/null +++ b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/proposal.md @@ -0,0 +1,30 @@ +## Why + +Ambiguous HTTP Responses bridge disconnects can leave an upstream +`response.create` in flight. Recovery needs a durable, tenant-scoped operation +identity and transcript that survives process ownership changes without +replaying stale terminal events or deleting recoverable rows during startup. + +## What Changes + +- Keep durable operation fingerprints isolated by API-key scope. +- Preserve sessions with submitted, acknowledged, or unknown operations during + startup takeover and detach their ownership for recovery. +- Reset an operation's event spool before rebinding an explicit failed retry. +- Only advance a continuation anchor when the completed sibling proves the same + logical request fingerprint. +- Keep the operation-ledger migration lineage converged with the current main + Alembic head. + +## Capabilities + +### Modified Capabilities + +- `responses-api-compat`: ambiguous HTTP bridge operations recover safely across + owner changes and retries. + +## Impact + +The proxy durable repository, HTTP bridge request submission, startup takeover, +Alembic graph, and focused recovery tests are affected. Public request and +response shapes remain unchanged. diff --git a/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..92e2b41de3 --- /dev/null +++ b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/specs/responses-api-compat/spec.md @@ -0,0 +1,358 @@ +# responses-api-compat Delta + +## ADDED Requirements + +### Requirement: Scoped operation identity + +The system MUST include the normalized API-key scope in every durable HTTP +bridge operation fingerprint and MUST apply that scope to fingerprint and +completed-operation lookups. + +#### Scenario: Equal requests from different keys remain isolated + +- **WHEN** two API keys submit the same logical request +- **THEN** each key receives an independent durable operation identity + +### Requirement: Recoverable startup takeover + +Startup cleanup MUST retain sessions that own submitted, acknowledged, or +unknown operations and MUST detach ownership before a replacement instance +takes over. + +#### Scenario: Restart preserves an in-flight operation + +- **WHEN** an instance restarts while an operation is nonterminal +- **THEN** cleanup detaches the old owner without deleting the operation spool + +### Requirement: Fresh retry transcript + +When an explicit failed operation is rebound, the system MUST atomically remove +the prior operation events and reset event-byte/spool state before accepting new +events. + +#### Scenario: Failed retry cannot replay stale failure output + +- **WHEN** a failed operation is retried and later completes +- **THEN** replay contains only the new attempt's events + +### Requirement: Proof-gated sibling anchoring + +The system MUST advance a continuation to a completed sibling response only +when the sibling has the same parent and logical request fingerprint in the +same API-key scope. + +#### Scenario: Distinct sibling input keeps its requested parent + +- **WHEN** a request reuses a parent with a different fingerprint +- **THEN** the service does not silently anchor it to another child response + +### Requirement: Single migration head + +The Alembic graph MUST converge the durable operation revisions with the current +release head and MUST expose one canonical head after upgrade. + +#### Scenario: Upgrade resolves one head + +- **WHEN** migrations are upgraded to the release tip +- **THEN** Alembic reports one canonical head + +### Requirement: Conservative spool defaults + +New operation rows MUST start with an incomplete event spool on SQLite and +PostgreSQL. A transcript MUST become replayable only after terminal event drain +and explicit finalization. + +#### Scenario: Nonterminal spool is not replayable + +- **WHEN** an operation has events but no finalized terminal event +- **THEN** recovery does not replay its transcript as complete + +### Requirement: Retain completed recovery transcripts + +Startup ownership cleanup MUST retain sessions with operation transcripts that +remain inside the configured operation retention window, including completed +operations, and MUST let normal spool retention remove the operation rows. + +#### Scenario: Recent completed transcript survives takeover + +- **WHEN** startup cleanup sees a recent completed transcript +- **THEN** it retains the session until normal retention expires it + +### Requirement: Continuous transcript retention + +Operation transcript cleanup MUST run periodically in a leader-gated scheduler +and MUST drain all eligible batches during each pass. Disabling the existing +sticky-session mapping cleanup switch MUST NOT disable operation transcript +retention; that switch MAY skip sticky mapping maintenance while durable +operation retention continues. + +#### Scenario: Retention drains all eligible batches + +- **WHEN** more rows are eligible than one deletion batch +- **THEN** one scheduler pass removes every eligible batch + +#### Scenario: Sticky cleanup toggle does not disable transcript retention + +- **WHEN** sticky-session cleanup is disabled and the durable bridge schema is + available +- **THEN** the leader-gated scheduler still drains expired operation transcript + rows while skipping sticky mapping cleanup + +### Requirement: Fresh indefinite-recovery spool + +Before dispatching a server-owned retry for a nonterminal operation, the system +MUST atomically clear any partial event spool under the durable owner fence. + +#### Scenario: Retry starts with a clean transcript + +- **WHEN** an anchored retry is dispatched after partial persistence +- **THEN** old events and byte counts are cleared before new output is accepted + +### Requirement: Ordered deferred reasoning persistence + +Deferred reasoning events released before a visible event MUST be persisted in +the same order in which they are delivered downstream, before the visible +event is persisted. + +#### Scenario: Deferred events preserve downstream order + +- **WHEN** buffered reasoning is released before visible output +- **THEN** the durable spool stores the reasoning blocks before that output + +### Requirement: Per-operation disconnect classification + +When a shared bridge websocket closes, each pending operation MUST be +classified from that operation's own observed response-event count. Activity +from a sibling request MUST NOT make an eventless operation safely retryable. + +#### Scenario: Sibling output does not acknowledge an eventless request + +- **WHEN** one pending request emitted output and another emitted none +- **THEN** the two operations receive different disconnect classifications + +### Requirement: Abandoned operation retention + +Operation retention MUST expire stale submitted and acknowledged rows in +addition to terminal and ambiguous rows, so a crashed or abandoned operation +cannot retain raw request data indefinitely. + +#### Scenario: Stale abandoned request is purged + +- **WHEN** a submitted operation exceeds retention age +- **THEN** its request data and event spool are removed + +### Requirement: Acknowledged alias persistence failure + +If upstream has acknowledged a response but local continuity-alias persistence +fails, the downstream error MUST NOT transition the durable operation to a +retryable failed state. The operation MUST remain acknowledged/ambiguous so an +identical retry cannot dispatch a duplicate upstream turn. + +#### Scenario: Alias write failure remains fail-closed + +- **WHEN** an acknowledged response cannot publish its continuity alias +- **THEN** the operation remains non-retryable and the client receives a terminal error + +### Requirement: Cross-session nonterminal handoff + +When a scoped operation fingerprint is found under a different durable +session, a nonterminal operation MUST be atomically rebound to the currently +owned session before its event spool is reset or a recovery attempt is sent. +Completed replayable operations MUST remain attached to their original session. +The handoff MUST be refused while the prior session has an unexpired owner +lease, preventing concurrent owners from dispatching the same turn. + +#### Scenario: Active prior owner fences handoff + +- **WHEN** a duplicate request finds a nonterminal operation under another session +- **AND** that session still has an unexpired owner lease +- **THEN** the operation remains with the prior session and no concurrent retry is dispatched + +#### Scenario: Expired prior owner permits handoff + +- **WHEN** the prior session lease is absent or expired +- **THEN** the operation can be atomically rebound before recovery + +### Requirement: Fenced one-shot recovery dispatch + +The durable recovery journal MUST persist a one-shot replay budget for every +recovery-safe request. The budget MUST be consumed atomically when a replay is +claimed for dispatch, and a caller that proves the replay never reached the +upstream send boundary MUST restore that claim under the same session owner +fence. A replacement session MUST retain or transfer a fenced origin owner +until the claim is rolled back or settled; selecting a replacement or failing +preflight MUST NOT permanently consume an unsent replay. + +#### Scenario: Concurrent reconnects consume one replay + +- **WHEN** concurrent reconnects observe the same ambiguous operation +- **THEN** exactly one owner atomically claims the persisted replay budget and + other reconnects fail closed without dispatching a duplicate + +#### Scenario: Pre-dispatch replacement failure restores the budget + +- **WHEN** a replay claim is made but replacement admission or preflight fails + before the exact upstream frame is sent +- **THEN** the claim returns to the available state and the fenced origin + owner is released only after that rollback succeeds + +#### Scenario: Successful replacement settles the origin journal + +- **WHEN** a replacement session dispatches the claimed replay and receives a + terminal response event +- **THEN** settlement uses the retained origin owner fence before releasing it + and the replay budget cannot be claimed again + +### Requirement: Lease-aware operation retention + +Retention MUST NOT delete stale submitted or acknowledged operations while +their session is actively owned with an unexpired lease. The owner/lease +predicate MUST be rechecked in the deletion transaction. + +#### Scenario: Active lease protects stale operation + +- **WHEN** a stale operation belongs to a session with a live lease +- **THEN** retention leaves it intact + +### Requirement: Anchored indefinite recovery gate + +The server-indefinite recovery loop MUST be installed only for an eventless +anchored continuation with a durable parent operation. Fresh first-turn +requests and streams that already emitted downstream response events MUST +terminate normally rather than being resent indefinitely. + +#### Scenario: Fresh request is not held indefinitely + +- **WHEN** a first-turn request loses its upstream connection +- **THEN** the proxy returns its normal error path without an indefinite loop + +### Requirement: Retry reservation terminalization + +If reacquiring API-key usage limits for a recovery attempt fails, the proxy +MUST settle the prior reservation and emit a terminal `response.failed` SSE +event instead of aborting the already-started stream. + +#### Scenario: Quota failure produces terminal SSE + +- **WHEN** a recovery retry cannot reacquire its usage reservation +- **THEN** the client receives `response.failed` and the prior reservation is settled + +#### Scenario: Unexpected admission failure produces terminal SSE + +- **WHEN** recovery admission raises an unexpected infrastructure error before + a replacement stream starts +- **THEN** the client receives `response.failed` and the prior reservation is + settled instead of receiving a truncated stream + +### Requirement: Failure spool/state ordering + +For an explicit deterministic failure, the proxy MUST persist the terminal SSE +block before exposing the durable operation as failed. The event append and +failed-state transition MUST use the same owner fence and transaction when the +durable repository supports it. + +#### Scenario: Concurrent retry cannot reset an unspooled failure + +- **WHEN** a response failure is being settled while an identical reconnect is + admitted +- **THEN** the reconnect observes the terminal operation fence and cannot reset + or mix the previous failure into a new transcript + +### Requirement: Partial disconnect acknowledgement + +When a bridge disconnects after an operation has emitted any response event but +before a terminal event, the durable operation MUST remain acknowledged or +ambiguous. It MUST NOT be classified as retryable failed solely because the +disconnect was non-terminal. + +#### Scenario: Partial output is never resent as a fresh turn + +- **WHEN** the upstream closes after `response.created` but before completion +- **THEN** the operation remains non-retryable + +### Requirement: Retry output stops indefinite recovery + +An indefinite recovery attempt MUST stop retrying once that attempt emits any +downstream response event, even if the attempt later fails with a retryable +transport error. + +#### Scenario: Retry output prevents a second attempt + +- **WHEN** a retry emits a data event and then times out +- **THEN** the server stops the indefinite loop instead of appending another response + +### Requirement: Preserve repeated event occurrences + +The durable event spool MUST preserve repeated identical SSE blocks as distinct +ordered occurrences. Event identity MUST include its operation-local sequence +position rather than content alone. + +#### Scenario: Identical deltas replay twice + +- **WHEN** two consecutive SSE blocks have identical text +- **THEN** both occurrences are present in the replay transcript + +### Requirement: Stop event persistence during shutdown + +Proxy shutdown MUST close the HTTP bridge event batcher and cancel its +background flusher before the process exits. + +#### Scenario: Shutdown cancels the flusher + +- **WHEN** the proxy service begins shutdown after queueing an event +- **THEN** the batcher's background task is cancelled and awaited + +### Requirement: Classify response.incomplete as terminal + +An anchored `response.incomplete` event MUST transition the durable operation to +an explicit terminal state and finalize its transcript so it is not left in an +unknown in-flight state. + +#### Scenario: Incomplete response is replayable as terminal + +- **WHEN** upstream emits `response.incomplete` +- **THEN** the operation is terminalized and its drained transcript is eligible for replay + +### Requirement: Settle reservations before timeout health + +When an eventless timeout retires a keyed bridge, the proxy MUST settle all +pending request reservations before recording the account timeout health signal. +If settlement fails, the health signal MUST NOT claim that cleanup completed. + +#### Scenario: Failed reservation release does not poison health state + +- **WHEN** the timeout cleanup cannot release a pending reservation +- **THEN** the account timeout signal is not recorded before that failure is surfaced + +### Requirement: Replay finalized incomplete operations + +A finalized `incomplete` operation transcript MUST be replayed for an identical +request and MUST NOT be reset or treated as an unknown in-flight operation. + +#### Scenario: Reconnect receives stored incomplete transcript + +- **WHEN** an identical request finds a finalized incomplete operation +- **THEN** the stored terminal transcript is delivered without a new upstream dispatch + +### Requirement: Validate final response.create size + +After adding durable operation metadata, the proxy MUST revalidate the exact +serialized `response.create` frame against the upstream size limit before +sending it. + +#### Scenario: Metadata cannot create an oversized frame + +- **WHEN** operation metadata makes the final frame exceed the configured limit +- **THEN** the request is rejected or slimmed before any upstream send + +### Requirement: Fence same-session active operations + +Server-indefinite recovery MUST NOT reset or redispatch a nonterminal operation +when another pending request in the same durable session still references that +operation. Submitted and acknowledged operations MUST remain fail-closed; +only an inactive `unknown` operation may enter a fresh recovery attempt. + +#### Scenario: Active same-session operation is not duplicated + +- **WHEN** a duplicate request finds a submitted operation still referenced by another pending request +- **THEN** the proxy refuses a second dispatch and preserves the existing spool diff --git a/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/tasks.md b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/tasks.md new file mode 100644 index 0000000000..8c4b47e7b8 --- /dev/null +++ b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/tasks.md @@ -0,0 +1,49 @@ +## 1. Implementation + +- [x] 1.1 Scope operation fingerprints and lookups by API-key namespace. +- [x] 1.2 Preserve recoverable operation sessions during startup takeover. +- [x] 1.3 Reset failed-operation event spools atomically. +- [x] 1.4 Gate sibling continuation anchoring on matching fingerprints. +- [x] 1.5 Merge the operation-ledger migration lineage with latest main. +- [x] 1.6 Keep SQLite event-spool defaults conservative and explicit. +- [x] 1.7 Retain completed transcripts through startup takeover and drain + periodic retention batches. +- [x] 1.8 Reset partial spools before indefinite recovery retries. +- [x] 1.9 Persist deferred reasoning events in downstream order. +- [x] 1.10 Classify shared-websocket disconnects per operation event count. +- [x] 1.11 Expire stale submitted and acknowledged operation rows. +- [x] 1.12 Preserve acknowledged state after alias persistence failure. +- [x] 1.13 Rebind nonterminal cross-session operations before recovery reset. +- [x] 1.14 Protect actively leased operations during retention cleanup. +- [x] 1.15 Keep event-spool settings compatible with legacy test doubles. +- [x] 1.16 Gate indefinite recovery to eventless anchored operations. +- [x] 1.17 Convert recovery reservation failures into terminal SSE events. +- [x] 1.18 Preserve acknowledged state after partial response output and disconnect. +- [x] 1.19 Stop indefinite recovery after a retry attempt emits downstream output. +- [x] 1.20 Include sequence position in event fingerprints so repeated SSE blocks survive replay. +- [x] 1.21 Close the event batcher flusher from the proxy shutdown path. +- [x] 1.22 Refuse cross-session handoff while the prior session lease is active. +- [x] 1.23 Keep eventless local transport failures retryable in indefinite recovery. +- [x] 1.24 Terminalize and persist `response.incomplete` operation outcomes. +- [x] 1.25 Place all response-compatibility requirements in the capability delta path. +- [x] 1.26 Record timeout health only after pending reservation settlement. +- [x] 1.27 Replay finalized incomplete operations without resetting their terminal spool. +- [x] 1.28 Return reservation settlement status to timeout health handling. +- [x] 1.29 Revalidate the final response.create frame after operation metadata injection. +- [x] 1.30 Require an inactive unknown operation before same-session recovery reset. +- [x] 1.31 Keep operation transcript retention active when sticky mapping cleanup is disabled. +- [x] 1.32 Persist and fence the one-shot recovery dispatch budget through + replacement-session handoff, rollback, and terminal settlement. +- [x] 1.33 Restore claimed recovery operations on every pre-admission exit and + atomically spool deterministic terminal failures before exposing `failed`. + +## 2. Validation + +- [x] 2.1 Add or update focused repository and request-submit regressions. +- [x] 2.2 Run focused HTTP bridge tests, Ruff, Ty, diff checks, and strict + OpenSpec validation. + - Evidence: focused HTTP bridge/API tests, Ruff, Ty, migration checks, and + strict OpenSpec validation passed after the recovery-budget handoff fix. +- [x] 2.3 Verify disabled sticky cleanup still runs durable transcript retention. +- [x] 2.4 Add regressions for pre-admission claim restoration and terminal + failure spool/state ordering. diff --git a/openspec/changes/enforced-service-tier-model-fallback/.openspec.yaml b/openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/.openspec.yaml similarity index 100% rename from openspec/changes/enforced-service-tier-model-fallback/.openspec.yaml rename to openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/.openspec.yaml diff --git a/openspec/changes/enforced-service-tier-model-fallback/design.md b/openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/design.md similarity index 100% rename from openspec/changes/enforced-service-tier-model-fallback/design.md rename to openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/design.md diff --git a/openspec/changes/enforced-service-tier-model-fallback/proposal.md b/openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/proposal.md similarity index 100% rename from openspec/changes/enforced-service-tier-model-fallback/proposal.md rename to openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/proposal.md diff --git a/openspec/changes/enforced-service-tier-model-fallback/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/specs/model-catalog-compat/spec.md similarity index 100% rename from openspec/changes/enforced-service-tier-model-fallback/specs/model-catalog-compat/spec.md rename to openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/specs/model-catalog-compat/spec.md diff --git a/openspec/changes/enforced-service-tier-model-fallback/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/enforced-service-tier-model-fallback/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/specs/responses-api-compat/spec.md diff --git a/openspec/changes/enforced-service-tier-model-fallback/tasks.md b/openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/tasks.md similarity index 100% rename from openspec/changes/enforced-service-tier-model-fallback/tasks.md rename to openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/tasks.md diff --git a/openspec/changes/expose-fleet-usage-refresh-timestamp/.openspec.yaml b/openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/.openspec.yaml similarity index 100% rename from openspec/changes/expose-fleet-usage-refresh-timestamp/.openspec.yaml rename to openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/.openspec.yaml diff --git a/openspec/changes/expose-fleet-usage-refresh-timestamp/design.md b/openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/design.md similarity index 100% rename from openspec/changes/expose-fleet-usage-refresh-timestamp/design.md rename to openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/design.md diff --git a/openspec/changes/expose-fleet-usage-refresh-timestamp/proposal.md b/openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/proposal.md similarity index 100% rename from openspec/changes/expose-fleet-usage-refresh-timestamp/proposal.md rename to openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/proposal.md diff --git a/openspec/changes/expose-fleet-usage-refresh-timestamp/specs/fleet-summary/spec.md b/openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/specs/fleet-summary/spec.md similarity index 100% rename from openspec/changes/expose-fleet-usage-refresh-timestamp/specs/fleet-summary/spec.md rename to openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/specs/fleet-summary/spec.md diff --git a/openspec/changes/expose-fleet-usage-refresh-timestamp/tasks.md b/openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/tasks.md similarity index 100% rename from openspec/changes/expose-fleet-usage-refresh-timestamp/tasks.md rename to openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/tasks.md diff --git a/openspec/changes/extend-websocket-stream-budget/proposal.md b/openspec/changes/archive/2026-08-13-extend-websocket-stream-budget/proposal.md similarity index 100% rename from openspec/changes/extend-websocket-stream-budget/proposal.md rename to openspec/changes/archive/2026-08-13-extend-websocket-stream-budget/proposal.md diff --git a/openspec/changes/extend-websocket-stream-budget/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-extend-websocket-stream-budget/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/extend-websocket-stream-budget/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-extend-websocket-stream-budget/specs/responses-api-compat/spec.md diff --git a/openspec/changes/extend-websocket-stream-budget/tasks.md b/openspec/changes/archive/2026-08-13-extend-websocket-stream-budget/tasks.md similarity index 100% rename from openspec/changes/extend-websocket-stream-budget/tasks.md rename to openspec/changes/archive/2026-08-13-extend-websocket-stream-budget/tasks.md diff --git a/openspec/changes/fix-auth-guardian-detached-candidates/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/.openspec.yaml similarity index 100% rename from openspec/changes/fix-auth-guardian-detached-candidates/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/.openspec.yaml diff --git a/openspec/changes/fix-auth-guardian-detached-candidates/context.md b/openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/context.md similarity index 100% rename from openspec/changes/fix-auth-guardian-detached-candidates/context.md rename to openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/context.md diff --git a/openspec/changes/fix-auth-guardian-detached-candidates/design.md b/openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/design.md similarity index 100% rename from openspec/changes/fix-auth-guardian-detached-candidates/design.md rename to openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/design.md diff --git a/openspec/changes/fix-auth-guardian-detached-candidates/proposal.md b/openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/proposal.md similarity index 100% rename from openspec/changes/fix-auth-guardian-detached-candidates/proposal.md rename to openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/proposal.md diff --git a/openspec/changes/fix-auth-guardian-detached-candidates/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/fix-auth-guardian-detached-candidates/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/fix-auth-guardian-detached-candidates/tasks.md b/openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/tasks.md similarity index 100% rename from openspec/changes/fix-auth-guardian-detached-candidates/tasks.md rename to openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/tasks.md diff --git a/openspec/changes/fix-codex-catalog-required-fields/context.md b/openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/context.md similarity index 100% rename from openspec/changes/fix-codex-catalog-required-fields/context.md rename to openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/context.md diff --git a/openspec/changes/fix-codex-catalog-required-fields/proposal.md b/openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/proposal.md similarity index 100% rename from openspec/changes/fix-codex-catalog-required-fields/proposal.md rename to openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/proposal.md diff --git a/openspec/changes/fix-codex-catalog-required-fields/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/specs/model-catalog-compat/spec.md similarity index 100% rename from openspec/changes/fix-codex-catalog-required-fields/specs/model-catalog-compat/spec.md rename to openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/specs/model-catalog-compat/spec.md diff --git a/openspec/changes/fix-codex-catalog-required-fields/tasks.md b/openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/tasks.md similarity index 100% rename from openspec/changes/fix-codex-catalog-required-fields/tasks.md rename to openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/tasks.md diff --git a/openspec/changes/fix-dashboard-error-rate-cancelled/proposal.md b/openspec/changes/archive/2026-08-13-fix-dashboard-error-rate-cancelled/proposal.md similarity index 100% rename from openspec/changes/fix-dashboard-error-rate-cancelled/proposal.md rename to openspec/changes/archive/2026-08-13-fix-dashboard-error-rate-cancelled/proposal.md diff --git a/openspec/changes/fix-dashboard-error-rate-cancelled/specs/usage-error-metrics/spec.md b/openspec/changes/archive/2026-08-13-fix-dashboard-error-rate-cancelled/specs/usage-error-metrics/spec.md similarity index 100% rename from openspec/changes/fix-dashboard-error-rate-cancelled/specs/usage-error-metrics/spec.md rename to openspec/changes/archive/2026-08-13-fix-dashboard-error-rate-cancelled/specs/usage-error-metrics/spec.md diff --git a/openspec/changes/fix-dashboard-error-rate-cancelled/tasks.md b/openspec/changes/archive/2026-08-13-fix-dashboard-error-rate-cancelled/tasks.md similarity index 100% rename from openspec/changes/fix-dashboard-error-rate-cancelled/tasks.md rename to openspec/changes/archive/2026-08-13-fix-dashboard-error-rate-cancelled/tasks.md diff --git a/openspec/changes/fix-gpt-5-6-pricing/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/.openspec.yaml similarity index 100% rename from openspec/changes/fix-gpt-5-6-pricing/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/.openspec.yaml diff --git a/openspec/changes/fix-gpt-5-6-pricing/design.md b/openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/design.md similarity index 100% rename from openspec/changes/fix-gpt-5-6-pricing/design.md rename to openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/design.md diff --git a/openspec/changes/fix-gpt-5-6-pricing/proposal.md b/openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/proposal.md similarity index 100% rename from openspec/changes/fix-gpt-5-6-pricing/proposal.md rename to openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/proposal.md diff --git a/openspec/changes/fix-gpt-5-6-pricing/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/fix-gpt-5-6-pricing/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/specs/api-keys/spec.md diff --git a/openspec/changes/fix-gpt-5-6-pricing/tasks.md b/openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/tasks.md similarity index 100% rename from openspec/changes/fix-gpt-5-6-pricing/tasks.md rename to openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/tasks.md diff --git a/openspec/changes/fix-postgres-pool-budget/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/.openspec.yaml similarity index 100% rename from openspec/changes/fix-postgres-pool-budget/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/.openspec.yaml diff --git a/openspec/changes/fix-postgres-pool-budget/design.md b/openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/design.md similarity index 100% rename from openspec/changes/fix-postgres-pool-budget/design.md rename to openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/design.md diff --git a/openspec/changes/fix-postgres-pool-budget/proposal.md b/openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/proposal.md similarity index 100% rename from openspec/changes/fix-postgres-pool-budget/proposal.md rename to openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/proposal.md diff --git a/openspec/changes/fix-postgres-pool-budget/specs/database-backends/spec.md b/openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/specs/database-backends/spec.md similarity index 100% rename from openspec/changes/fix-postgres-pool-budget/specs/database-backends/spec.md rename to openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/specs/database-backends/spec.md diff --git a/openspec/changes/fix-postgres-pool-budget/specs/deployment-installation/spec.md b/openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/specs/deployment-installation/spec.md similarity index 100% rename from openspec/changes/fix-postgres-pool-budget/specs/deployment-installation/spec.md rename to openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/specs/deployment-installation/spec.md diff --git a/openspec/changes/fix-postgres-pool-budget/tasks.md b/openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/tasks.md similarity index 100% rename from openspec/changes/fix-postgres-pool-budget/tasks.md rename to openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/tasks.md diff --git a/openspec/changes/fix-promql-5xx-error-rate/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/.openspec.yaml similarity index 100% rename from openspec/changes/fix-promql-5xx-error-rate/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/.openspec.yaml diff --git a/openspec/changes/fix-promql-5xx-error-rate/design.md b/openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/design.md similarity index 100% rename from openspec/changes/fix-promql-5xx-error-rate/design.md rename to openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/design.md diff --git a/openspec/changes/fix-promql-5xx-error-rate/proposal.md b/openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/proposal.md similarity index 100% rename from openspec/changes/fix-promql-5xx-error-rate/proposal.md rename to openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/proposal.md diff --git a/openspec/changes/fix-promql-5xx-error-rate/specs/proxy-runtime-observability/spec.md b/openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/specs/proxy-runtime-observability/spec.md similarity index 100% rename from openspec/changes/fix-promql-5xx-error-rate/specs/proxy-runtime-observability/spec.md rename to openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/specs/proxy-runtime-observability/spec.md diff --git a/openspec/changes/fix-promql-5xx-error-rate/tasks.md b/openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/tasks.md similarity index 100% rename from openspec/changes/fix-promql-5xx-error-rate/tasks.md rename to openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/tasks.md diff --git a/openspec/changes/fix-replayed-namespaced-function-call/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/.openspec.yaml similarity index 100% rename from openspec/changes/fix-replayed-namespaced-function-call/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/.openspec.yaml diff --git a/openspec/changes/fix-replayed-namespaced-function-call/context.md b/openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/context.md similarity index 100% rename from openspec/changes/fix-replayed-namespaced-function-call/context.md rename to openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/context.md diff --git a/openspec/changes/fix-replayed-namespaced-function-call/design.md b/openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/design.md similarity index 100% rename from openspec/changes/fix-replayed-namespaced-function-call/design.md rename to openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/design.md diff --git a/openspec/changes/fix-replayed-namespaced-function-call/proposal.md b/openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/proposal.md similarity index 100% rename from openspec/changes/fix-replayed-namespaced-function-call/proposal.md rename to openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/proposal.md diff --git a/openspec/changes/fix-replayed-namespaced-function-call/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/fix-replayed-namespaced-function-call/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/specs/responses-api-compat/spec.md diff --git a/openspec/changes/fix-replayed-namespaced-function-call/tasks.md b/openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/tasks.md similarity index 100% rename from openspec/changes/fix-replayed-namespaced-function-call/tasks.md rename to openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/tasks.md diff --git a/openspec/changes/fix-reports-local-day-averages/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/.openspec.yaml similarity index 100% rename from openspec/changes/fix-reports-local-day-averages/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/.openspec.yaml diff --git a/openspec/changes/fix-reports-local-day-averages/design.md b/openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/design.md similarity index 100% rename from openspec/changes/fix-reports-local-day-averages/design.md rename to openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/design.md diff --git a/openspec/changes/fix-reports-local-day-averages/proposal.md b/openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/proposal.md similarity index 100% rename from openspec/changes/fix-reports-local-day-averages/proposal.md rename to openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/proposal.md diff --git a/openspec/changes/fix-reports-local-day-averages/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/fix-reports-local-day-averages/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/specs/frontend-architecture/spec.md diff --git a/openspec/changes/fix-reports-local-day-averages/tasks.md b/openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/tasks.md similarity index 100% rename from openspec/changes/fix-reports-local-day-averages/tasks.md rename to openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/tasks.md diff --git a/openspec/changes/fix-spark-quota-routing/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-spark-quota-routing/.openspec.yaml similarity index 100% rename from openspec/changes/fix-spark-quota-routing/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-spark-quota-routing/.openspec.yaml diff --git a/openspec/changes/fix-spark-quota-routing/design.md b/openspec/changes/archive/2026-08-13-fix-spark-quota-routing/design.md similarity index 100% rename from openspec/changes/fix-spark-quota-routing/design.md rename to openspec/changes/archive/2026-08-13-fix-spark-quota-routing/design.md diff --git a/openspec/changes/fix-spark-quota-routing/proposal.md b/openspec/changes/archive/2026-08-13-fix-spark-quota-routing/proposal.md similarity index 100% rename from openspec/changes/fix-spark-quota-routing/proposal.md rename to openspec/changes/archive/2026-08-13-fix-spark-quota-routing/proposal.md diff --git a/openspec/changes/fix-spark-quota-routing/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-13-fix-spark-quota-routing/specs/model-catalog-compat/spec.md similarity index 100% rename from openspec/changes/fix-spark-quota-routing/specs/model-catalog-compat/spec.md rename to openspec/changes/archive/2026-08-13-fix-spark-quota-routing/specs/model-catalog-compat/spec.md diff --git a/openspec/changes/fix-spark-quota-routing/tasks.md b/openspec/changes/archive/2026-08-13-fix-spark-quota-routing/tasks.md similarity index 100% rename from openspec/changes/fix-spark-quota-routing/tasks.md rename to openspec/changes/archive/2026-08-13-fix-spark-quota-routing/tasks.md diff --git a/openspec/changes/fix-useragent-migration-unicode-whitespace/design.md b/openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/design.md similarity index 100% rename from openspec/changes/fix-useragent-migration-unicode-whitespace/design.md rename to openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/design.md diff --git a/openspec/changes/fix-useragent-migration-unicode-whitespace/proposal.md b/openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/proposal.md similarity index 100% rename from openspec/changes/fix-useragent-migration-unicode-whitespace/proposal.md rename to openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/proposal.md diff --git a/openspec/changes/fix-useragent-migration-unicode-whitespace/specs/proxy-runtime-observability/spec.md b/openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/specs/proxy-runtime-observability/spec.md similarity index 100% rename from openspec/changes/fix-useragent-migration-unicode-whitespace/specs/proxy-runtime-observability/spec.md rename to openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/specs/proxy-runtime-observability/spec.md diff --git a/openspec/changes/fix-useragent-migration-unicode-whitespace/tasks.md b/openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/tasks.md similarity index 100% rename from openspec/changes/fix-useragent-migration-unicode-whitespace/tasks.md rename to openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/tasks.md diff --git a/openspec/changes/fix-warm-now-reset-utc-gate/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/.openspec.yaml similarity index 100% rename from openspec/changes/fix-warm-now-reset-utc-gate/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/.openspec.yaml diff --git a/openspec/changes/fix-warm-now-reset-utc-gate/proposal.md b/openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/proposal.md similarity index 100% rename from openspec/changes/fix-warm-now-reset-utc-gate/proposal.md rename to openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/proposal.md diff --git a/openspec/changes/fix-warm-now-reset-utc-gate/specs/quota-phase-planner/spec.md b/openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/specs/quota-phase-planner/spec.md similarity index 100% rename from openspec/changes/fix-warm-now-reset-utc-gate/specs/quota-phase-planner/spec.md rename to openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/specs/quota-phase-planner/spec.md diff --git a/openspec/changes/fix-warm-now-reset-utc-gate/tasks.md b/openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/tasks.md similarity index 100% rename from openspec/changes/fix-warm-now-reset-utc-gate/tasks.md rename to openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/tasks.md diff --git a/openspec/changes/preserve-dashboard-overview-on-log-failure/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/.openspec.yaml similarity index 100% rename from openspec/changes/preserve-dashboard-overview-on-log-failure/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/.openspec.yaml diff --git a/openspec/changes/fix-weekly-primary-placeholder-race/context.md b/openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/context.md similarity index 100% rename from openspec/changes/fix-weekly-primary-placeholder-race/context.md rename to openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/context.md diff --git a/openspec/changes/fix-weekly-primary-placeholder-race/design.md b/openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/design.md similarity index 100% rename from openspec/changes/fix-weekly-primary-placeholder-race/design.md rename to openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/design.md diff --git a/openspec/changes/fix-weekly-primary-placeholder-race/proposal.md b/openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/proposal.md similarity index 100% rename from openspec/changes/fix-weekly-primary-placeholder-race/proposal.md rename to openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/proposal.md diff --git a/openspec/changes/fix-weekly-primary-placeholder-race/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/fix-weekly-primary-placeholder-race/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/fix-weekly-primary-placeholder-race/tasks.md b/openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/tasks.md similarity index 100% rename from openspec/changes/fix-weekly-primary-placeholder-race/tasks.md rename to openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/tasks.md diff --git a/openspec/changes/forward-codex-alpha-search/.openspec.yaml b/openspec/changes/archive/2026-08-13-forward-codex-alpha-search/.openspec.yaml similarity index 100% rename from openspec/changes/forward-codex-alpha-search/.openspec.yaml rename to openspec/changes/archive/2026-08-13-forward-codex-alpha-search/.openspec.yaml diff --git a/openspec/changes/forward-codex-alpha-search/proposal.md b/openspec/changes/archive/2026-08-13-forward-codex-alpha-search/proposal.md similarity index 100% rename from openspec/changes/forward-codex-alpha-search/proposal.md rename to openspec/changes/archive/2026-08-13-forward-codex-alpha-search/proposal.md diff --git a/openspec/changes/forward-codex-alpha-search/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-forward-codex-alpha-search/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/forward-codex-alpha-search/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-forward-codex-alpha-search/specs/responses-api-compat/spec.md diff --git a/openspec/changes/forward-codex-alpha-search/tasks.md b/openspec/changes/archive/2026-08-13-forward-codex-alpha-search/tasks.md similarity index 100% rename from openspec/changes/forward-codex-alpha-search/tasks.md rename to openspec/changes/archive/2026-08-13-forward-codex-alpha-search/tasks.md diff --git a/openspec/changes/header-brand-navigate-to-dashboard/.openspec.yaml b/openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/.openspec.yaml similarity index 100% rename from openspec/changes/header-brand-navigate-to-dashboard/.openspec.yaml rename to openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/.openspec.yaml diff --git a/openspec/changes/header-brand-navigate-to-dashboard/proposal.md b/openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/proposal.md similarity index 100% rename from openspec/changes/header-brand-navigate-to-dashboard/proposal.md rename to openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/proposal.md diff --git a/openspec/changes/header-brand-navigate-to-dashboard/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/header-brand-navigate-to-dashboard/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/specs/frontend-architecture/spec.md diff --git a/openspec/changes/header-brand-navigate-to-dashboard/tasks.md b/openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/tasks.md similarity index 100% rename from openspec/changes/header-brand-navigate-to-dashboard/tasks.md rename to openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/tasks.md diff --git a/openspec/changes/keep-request-shape-rejections-account-neutral/proposal.md b/openspec/changes/archive/2026-08-13-keep-request-shape-rejections-account-neutral/proposal.md similarity index 100% rename from openspec/changes/keep-request-shape-rejections-account-neutral/proposal.md rename to openspec/changes/archive/2026-08-13-keep-request-shape-rejections-account-neutral/proposal.md diff --git a/openspec/changes/keep-request-shape-rejections-account-neutral/specs/account-routing/spec.md b/openspec/changes/archive/2026-08-13-keep-request-shape-rejections-account-neutral/specs/account-routing/spec.md similarity index 100% rename from openspec/changes/keep-request-shape-rejections-account-neutral/specs/account-routing/spec.md rename to openspec/changes/archive/2026-08-13-keep-request-shape-rejections-account-neutral/specs/account-routing/spec.md diff --git a/openspec/changes/keep-request-shape-rejections-account-neutral/tasks.md b/openspec/changes/archive/2026-08-13-keep-request-shape-rejections-account-neutral/tasks.md similarity index 100% rename from openspec/changes/keep-request-shape-rejections-account-neutral/tasks.md rename to openspec/changes/archive/2026-08-13-keep-request-shape-rejections-account-neutral/tasks.md diff --git a/openspec/changes/persist-usage-snapshot-transactionally/.openspec.yaml b/openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/.openspec.yaml similarity index 100% rename from openspec/changes/persist-usage-snapshot-transactionally/.openspec.yaml rename to openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/.openspec.yaml diff --git a/openspec/changes/persist-usage-snapshot-transactionally/design.md b/openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/design.md similarity index 100% rename from openspec/changes/persist-usage-snapshot-transactionally/design.md rename to openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/design.md diff --git a/openspec/changes/persist-usage-snapshot-transactionally/proposal.md b/openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/proposal.md similarity index 100% rename from openspec/changes/persist-usage-snapshot-transactionally/proposal.md rename to openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/proposal.md diff --git a/openspec/changes/persist-usage-snapshot-transactionally/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/persist-usage-snapshot-transactionally/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/persist-usage-snapshot-transactionally/tasks.md b/openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/tasks.md similarity index 100% rename from openspec/changes/persist-usage-snapshot-transactionally/tasks.md rename to openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/tasks.md diff --git a/openspec/changes/pin-asyncpg-session-timezone-utc/context.md b/openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/context.md similarity index 100% rename from openspec/changes/pin-asyncpg-session-timezone-utc/context.md rename to openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/context.md diff --git a/openspec/changes/pin-asyncpg-session-timezone-utc/proposal.md b/openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/proposal.md similarity index 100% rename from openspec/changes/pin-asyncpg-session-timezone-utc/proposal.md rename to openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/proposal.md diff --git a/openspec/changes/pin-asyncpg-session-timezone-utc/specs/database-backends/spec.md b/openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/specs/database-backends/spec.md similarity index 100% rename from openspec/changes/pin-asyncpg-session-timezone-utc/specs/database-backends/spec.md rename to openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/specs/database-backends/spec.md diff --git a/openspec/changes/pin-asyncpg-session-timezone-utc/tasks.md b/openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/tasks.md similarity index 100% rename from openspec/changes/pin-asyncpg-session-timezone-utc/tasks.md rename to openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/tasks.md diff --git a/openspec/changes/recover-repeated-clean-close/.openspec.yaml b/openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/.openspec.yaml similarity index 100% rename from openspec/changes/recover-repeated-clean-close/.openspec.yaml rename to openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/.openspec.yaml diff --git a/openspec/changes/preserve-dashboard-overview-on-log-failure/design.md b/openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/design.md similarity index 100% rename from openspec/changes/preserve-dashboard-overview-on-log-failure/design.md rename to openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/design.md diff --git a/openspec/changes/preserve-dashboard-overview-on-log-failure/proposal.md b/openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/proposal.md similarity index 100% rename from openspec/changes/preserve-dashboard-overview-on-log-failure/proposal.md rename to openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/proposal.md diff --git a/openspec/changes/preserve-dashboard-overview-on-log-failure/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/preserve-dashboard-overview-on-log-failure/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/specs/frontend-architecture/spec.md diff --git a/openspec/changes/preserve-dashboard-overview-on-log-failure/tasks.md b/openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/tasks.md similarity index 100% rename from openspec/changes/preserve-dashboard-overview-on-log-failure/tasks.md rename to openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/tasks.md diff --git a/openspec/changes/preserve-historical-compact-side-effects/proposal.md b/openspec/changes/archive/2026-08-13-preserve-historical-compact-side-effects/proposal.md similarity index 100% rename from openspec/changes/preserve-historical-compact-side-effects/proposal.md rename to openspec/changes/archive/2026-08-13-preserve-historical-compact-side-effects/proposal.md diff --git a/openspec/changes/preserve-historical-compact-side-effects/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-preserve-historical-compact-side-effects/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/preserve-historical-compact-side-effects/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-preserve-historical-compact-side-effects/specs/responses-api-compat/spec.md diff --git a/openspec/changes/preserve-historical-compact-side-effects/tasks.md b/openspec/changes/archive/2026-08-13-preserve-historical-compact-side-effects/tasks.md similarity index 100% rename from openspec/changes/preserve-historical-compact-side-effects/tasks.md rename to openspec/changes/archive/2026-08-13-preserve-historical-compact-side-effects/tasks.md diff --git a/openspec/changes/preserve-http-bridge-terminal-delivery/.openspec.yaml b/openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/.openspec.yaml similarity index 100% rename from openspec/changes/preserve-http-bridge-terminal-delivery/.openspec.yaml rename to openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/.openspec.yaml diff --git a/openspec/changes/preserve-http-bridge-terminal-delivery/proposal.md b/openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/proposal.md similarity index 100% rename from openspec/changes/preserve-http-bridge-terminal-delivery/proposal.md rename to openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/proposal.md diff --git a/openspec/changes/preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md diff --git a/openspec/changes/preserve-http-bridge-terminal-delivery/tasks.md b/openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/tasks.md similarity index 100% rename from openspec/changes/preserve-http-bridge-terminal-delivery/tasks.md rename to openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/tasks.md diff --git a/openspec/changes/prevent-http-bridge-model-transition-loop/.openspec.yaml b/openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/.openspec.yaml similarity index 100% rename from openspec/changes/prevent-http-bridge-model-transition-loop/.openspec.yaml rename to openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/.openspec.yaml diff --git a/openspec/changes/prevent-http-bridge-model-transition-loop/proposal.md b/openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/proposal.md similarity index 100% rename from openspec/changes/prevent-http-bridge-model-transition-loop/proposal.md rename to openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/proposal.md diff --git a/openspec/changes/prevent-http-bridge-model-transition-loop/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/prevent-http-bridge-model-transition-loop/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/specs/responses-api-compat/spec.md diff --git a/openspec/changes/prevent-http-bridge-model-transition-loop/tasks.md b/openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/tasks.md similarity index 100% rename from openspec/changes/prevent-http-bridge-model-transition-loop/tasks.md rename to openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/tasks.md diff --git a/openspec/changes/propagate-forwarded-compact-settlement-failure/.openspec.yaml b/openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/.openspec.yaml similarity index 100% rename from openspec/changes/propagate-forwarded-compact-settlement-failure/.openspec.yaml rename to openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/.openspec.yaml diff --git a/openspec/changes/propagate-forwarded-compact-settlement-failure/design.md b/openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/design.md similarity index 100% rename from openspec/changes/propagate-forwarded-compact-settlement-failure/design.md rename to openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/design.md diff --git a/openspec/changes/propagate-forwarded-compact-settlement-failure/proposal.md b/openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/proposal.md similarity index 100% rename from openspec/changes/propagate-forwarded-compact-settlement-failure/proposal.md rename to openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/proposal.md diff --git a/openspec/changes/propagate-forwarded-compact-settlement-failure/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/propagate-forwarded-compact-settlement-failure/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/propagate-forwarded-compact-settlement-failure/tasks.md b/openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/tasks.md similarity index 100% rename from openspec/changes/propagate-forwarded-compact-settlement-failure/tasks.md rename to openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/tasks.md diff --git a/openspec/changes/purge-stale-bridge-sessions-on-startup/.openspec.yaml b/openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/.openspec.yaml similarity index 100% rename from openspec/changes/purge-stale-bridge-sessions-on-startup/.openspec.yaml rename to openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/.openspec.yaml diff --git a/openspec/changes/purge-stale-bridge-sessions-on-startup/design.md b/openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/design.md similarity index 100% rename from openspec/changes/purge-stale-bridge-sessions-on-startup/design.md rename to openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/design.md diff --git a/openspec/changes/purge-stale-bridge-sessions-on-startup/proposal.md b/openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/proposal.md similarity index 100% rename from openspec/changes/purge-stale-bridge-sessions-on-startup/proposal.md rename to openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/proposal.md diff --git a/openspec/changes/purge-stale-bridge-sessions-on-startup/specs/sticky-session-operations/spec.md b/openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/specs/sticky-session-operations/spec.md similarity index 100% rename from openspec/changes/purge-stale-bridge-sessions-on-startup/specs/sticky-session-operations/spec.md rename to openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/specs/sticky-session-operations/spec.md diff --git a/openspec/changes/purge-stale-bridge-sessions-on-startup/tasks.md b/openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/tasks.md similarity index 100% rename from openspec/changes/purge-stale-bridge-sessions-on-startup/tasks.md rename to openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/tasks.md diff --git a/openspec/changes/quarantine-silent-bridge-sessions/.openspec.yaml b/openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/.openspec.yaml similarity index 100% rename from openspec/changes/quarantine-silent-bridge-sessions/.openspec.yaml rename to openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/.openspec.yaml diff --git a/openspec/changes/quarantine-silent-bridge-sessions/proposal.md b/openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/proposal.md similarity index 100% rename from openspec/changes/quarantine-silent-bridge-sessions/proposal.md rename to openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/proposal.md diff --git a/openspec/changes/quarantine-silent-bridge-sessions/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/quarantine-silent-bridge-sessions/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/specs/responses-api-compat/spec.md diff --git a/openspec/changes/quarantine-silent-bridge-sessions/tasks.md b/openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/tasks.md similarity index 100% rename from openspec/changes/quarantine-silent-bridge-sessions/tasks.md rename to openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/tasks.md diff --git a/openspec/changes/record-early-downstream-cancellations/.openspec.yaml b/openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/.openspec.yaml similarity index 100% rename from openspec/changes/record-early-downstream-cancellations/.openspec.yaml rename to openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/.openspec.yaml diff --git a/openspec/changes/record-early-downstream-cancellations/proposal.md b/openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/proposal.md similarity index 100% rename from openspec/changes/record-early-downstream-cancellations/proposal.md rename to openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/proposal.md diff --git a/openspec/changes/record-early-downstream-cancellations/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/record-early-downstream-cancellations/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/specs/responses-api-compat/spec.md diff --git a/openspec/changes/record-early-downstream-cancellations/tasks.md b/openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/tasks.md similarity index 100% rename from openspec/changes/record-early-downstream-cancellations/tasks.md rename to openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/tasks.md diff --git a/openspec/changes/recover-safe-http-bridge-continuations/.openspec.yaml b/openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/.openspec.yaml similarity index 100% rename from openspec/changes/recover-safe-http-bridge-continuations/.openspec.yaml rename to openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/.openspec.yaml diff --git a/openspec/changes/recover-safe-http-bridge-continuations/proposal.md b/openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/proposal.md similarity index 100% rename from openspec/changes/recover-safe-http-bridge-continuations/proposal.md rename to openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/proposal.md diff --git a/openspec/changes/recover-safe-http-bridge-continuations/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/recover-safe-http-bridge-continuations/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/specs/responses-api-compat/spec.md diff --git a/openspec/changes/recover-safe-http-bridge-continuations/tasks.md b/openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/tasks.md similarity index 100% rename from openspec/changes/recover-safe-http-bridge-continuations/tasks.md rename to openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/tasks.md diff --git a/openspec/changes/refresh-selected-account-usage/.openspec.yaml b/openspec/changes/archive/2026-08-13-refresh-selected-account-usage/.openspec.yaml similarity index 100% rename from openspec/changes/refresh-selected-account-usage/.openspec.yaml rename to openspec/changes/archive/2026-08-13-refresh-selected-account-usage/.openspec.yaml diff --git a/openspec/changes/refresh-selected-account-usage/design.md b/openspec/changes/archive/2026-08-13-refresh-selected-account-usage/design.md similarity index 100% rename from openspec/changes/refresh-selected-account-usage/design.md rename to openspec/changes/archive/2026-08-13-refresh-selected-account-usage/design.md diff --git a/openspec/changes/refresh-selected-account-usage/proposal.md b/openspec/changes/archive/2026-08-13-refresh-selected-account-usage/proposal.md similarity index 100% rename from openspec/changes/refresh-selected-account-usage/proposal.md rename to openspec/changes/archive/2026-08-13-refresh-selected-account-usage/proposal.md diff --git a/openspec/changes/refresh-selected-account-usage/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-refresh-selected-account-usage/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/refresh-selected-account-usage/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-refresh-selected-account-usage/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/refresh-selected-account-usage/tasks.md b/openspec/changes/archive/2026-08-13-refresh-selected-account-usage/tasks.md similarity index 100% rename from openspec/changes/refresh-selected-account-usage/tasks.md rename to openspec/changes/archive/2026-08-13-refresh-selected-account-usage/tasks.md diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/.openspec.yaml b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/.openspec.yaml similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/.openspec.yaml rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/.openspec.yaml diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/design.md b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/design.md similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/design.md rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/design.md diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/proposal.md b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/proposal.md similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/proposal.md rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/proposal.md diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/specs/admin-auth/spec.md b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/specs/admin-auth/spec.md similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/specs/admin-auth/spec.md rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/specs/admin-auth/spec.md diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/specs/api-keys/spec.md diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/specs/deployment-installation/spec.md b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/specs/deployment-installation/spec.md similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/specs/deployment-installation/spec.md rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/specs/deployment-installation/spec.md diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/tasks.md b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/tasks.md similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/tasks.md rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/tasks.md diff --git a/openspec/changes/reject-duplicate-api-key-limit-rules/proposal.md b/openspec/changes/archive/2026-08-13-reject-duplicate-api-key-limit-rules/proposal.md similarity index 100% rename from openspec/changes/reject-duplicate-api-key-limit-rules/proposal.md rename to openspec/changes/archive/2026-08-13-reject-duplicate-api-key-limit-rules/proposal.md diff --git a/openspec/changes/reject-duplicate-api-key-limit-rules/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-reject-duplicate-api-key-limit-rules/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/reject-duplicate-api-key-limit-rules/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-reject-duplicate-api-key-limit-rules/specs/api-keys/spec.md diff --git a/openspec/changes/reject-duplicate-api-key-limit-rules/tasks.md b/openspec/changes/archive/2026-08-13-reject-duplicate-api-key-limit-rules/tasks.md similarity index 100% rename from openspec/changes/reject-duplicate-api-key-limit-rules/tasks.md rename to openspec/changes/archive/2026-08-13-reject-duplicate-api-key-limit-rules/tasks.md diff --git a/openspec/changes/reject-empty-migration-db-url/.openspec.yaml b/openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/.openspec.yaml similarity index 100% rename from openspec/changes/reject-empty-migration-db-url/.openspec.yaml rename to openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/.openspec.yaml diff --git a/openspec/changes/reject-empty-migration-db-url/context.md b/openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/context.md similarity index 100% rename from openspec/changes/reject-empty-migration-db-url/context.md rename to openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/context.md diff --git a/openspec/changes/reject-empty-migration-db-url/design.md b/openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/design.md similarity index 100% rename from openspec/changes/reject-empty-migration-db-url/design.md rename to openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/design.md diff --git a/openspec/changes/reject-empty-migration-db-url/proposal.md b/openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/proposal.md similarity index 100% rename from openspec/changes/reject-empty-migration-db-url/proposal.md rename to openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/proposal.md diff --git a/openspec/changes/reject-empty-migration-db-url/specs/database-migrations/spec.md b/openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/specs/database-migrations/spec.md similarity index 100% rename from openspec/changes/reject-empty-migration-db-url/specs/database-migrations/spec.md rename to openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/specs/database-migrations/spec.md diff --git a/openspec/changes/reject-empty-migration-db-url/tasks.md b/openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/tasks.md similarity index 100% rename from openspec/changes/reject-empty-migration-db-url/tasks.md rename to openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/tasks.md diff --git a/openspec/changes/reject-inverted-report-date-ranges/.openspec.yaml b/openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/.openspec.yaml similarity index 100% rename from openspec/changes/reject-inverted-report-date-ranges/.openspec.yaml rename to openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/.openspec.yaml diff --git a/openspec/changes/reject-inverted-report-date-ranges/design.md b/openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/design.md similarity index 100% rename from openspec/changes/reject-inverted-report-date-ranges/design.md rename to openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/design.md diff --git a/openspec/changes/reject-inverted-report-date-ranges/proposal.md b/openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/proposal.md similarity index 100% rename from openspec/changes/reject-inverted-report-date-ranges/proposal.md rename to openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/proposal.md diff --git a/openspec/changes/reject-inverted-report-date-ranges/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/reject-inverted-report-date-ranges/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/specs/frontend-architecture/spec.md diff --git a/openspec/changes/reject-inverted-report-date-ranges/tasks.md b/openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/tasks.md similarity index 100% rename from openspec/changes/reject-inverted-report-date-ranges/tasks.md rename to openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/tasks.md diff --git a/openspec/changes/reject-out-of-range-server-port/.openspec.yaml b/openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/.openspec.yaml similarity index 100% rename from openspec/changes/reject-out-of-range-server-port/.openspec.yaml rename to openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/.openspec.yaml diff --git a/openspec/changes/reject-out-of-range-server-port/context.md b/openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/context.md similarity index 100% rename from openspec/changes/reject-out-of-range-server-port/context.md rename to openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/context.md diff --git a/openspec/changes/reject-out-of-range-server-port/design.md b/openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/design.md similarity index 100% rename from openspec/changes/reject-out-of-range-server-port/design.md rename to openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/design.md diff --git a/openspec/changes/reject-out-of-range-server-port/proposal.md b/openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/proposal.md similarity index 100% rename from openspec/changes/reject-out-of-range-server-port/proposal.md rename to openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/proposal.md diff --git a/openspec/changes/reject-out-of-range-server-port/specs/runtime-portability/spec.md b/openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/specs/runtime-portability/spec.md similarity index 100% rename from openspec/changes/reject-out-of-range-server-port/specs/runtime-portability/spec.md rename to openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/specs/runtime-portability/spec.md diff --git a/openspec/changes/reject-out-of-range-server-port/tasks.md b/openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/tasks.md similarity index 100% rename from openspec/changes/reject-out-of-range-server-port/tasks.md rename to openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/tasks.md diff --git a/openspec/changes/release-idle-bridge-stream-leases/proposal.md b/openspec/changes/archive/2026-08-13-release-idle-bridge-stream-leases/proposal.md similarity index 100% rename from openspec/changes/release-idle-bridge-stream-leases/proposal.md rename to openspec/changes/archive/2026-08-13-release-idle-bridge-stream-leases/proposal.md diff --git a/openspec/changes/release-idle-bridge-stream-leases/specs/proxy-admission-control/spec.md b/openspec/changes/archive/2026-08-13-release-idle-bridge-stream-leases/specs/proxy-admission-control/spec.md similarity index 100% rename from openspec/changes/release-idle-bridge-stream-leases/specs/proxy-admission-control/spec.md rename to openspec/changes/archive/2026-08-13-release-idle-bridge-stream-leases/specs/proxy-admission-control/spec.md diff --git a/openspec/changes/release-idle-bridge-stream-leases/tasks.md b/openspec/changes/archive/2026-08-13-release-idle-bridge-stream-leases/tasks.md similarity index 100% rename from openspec/changes/release-idle-bridge-stream-leases/tasks.md rename to openspec/changes/archive/2026-08-13-release-idle-bridge-stream-leases/tasks.md diff --git a/openspec/changes/release-models-list-reservation/proposal.md b/openspec/changes/archive/2026-08-13-release-models-list-reservation/proposal.md similarity index 100% rename from openspec/changes/release-models-list-reservation/proposal.md rename to openspec/changes/archive/2026-08-13-release-models-list-reservation/proposal.md diff --git a/openspec/changes/release-models-list-reservation/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-13-release-models-list-reservation/specs/model-catalog-compat/spec.md similarity index 100% rename from openspec/changes/release-models-list-reservation/specs/model-catalog-compat/spec.md rename to openspec/changes/archive/2026-08-13-release-models-list-reservation/specs/model-catalog-compat/spec.md diff --git a/openspec/changes/release-models-list-reservation/tasks.md b/openspec/changes/archive/2026-08-13-release-models-list-reservation/tasks.md similarity index 100% rename from openspec/changes/release-models-list-reservation/tasks.md rename to openspec/changes/archive/2026-08-13-release-models-list-reservation/tasks.md diff --git a/openspec/changes/release-quota-reservations-on-header-failure/.openspec.yaml b/openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/.openspec.yaml similarity index 100% rename from openspec/changes/release-quota-reservations-on-header-failure/.openspec.yaml rename to openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/.openspec.yaml diff --git a/openspec/changes/release-quota-reservations-on-header-failure/context.md b/openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/context.md similarity index 100% rename from openspec/changes/release-quota-reservations-on-header-failure/context.md rename to openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/context.md diff --git a/openspec/changes/release-quota-reservations-on-header-failure/design.md b/openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/design.md similarity index 100% rename from openspec/changes/release-quota-reservations-on-header-failure/design.md rename to openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/design.md diff --git a/openspec/changes/release-quota-reservations-on-header-failure/proposal.md b/openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/proposal.md similarity index 100% rename from openspec/changes/release-quota-reservations-on-header-failure/proposal.md rename to openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/proposal.md diff --git a/openspec/changes/release-quota-reservations-on-header-failure/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/release-quota-reservations-on-header-failure/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/specs/api-keys/spec.md diff --git a/openspec/changes/release-quota-reservations-on-header-failure/tasks.md b/openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/tasks.md similarity index 100% rename from openspec/changes/release-quota-reservations-on-header-failure/tasks.md rename to openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/tasks.md diff --git a/openspec/changes/report-pool-usage-exhaustion/.openspec.yaml b/openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/.openspec.yaml similarity index 100% rename from openspec/changes/report-pool-usage-exhaustion/.openspec.yaml rename to openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/.openspec.yaml diff --git a/openspec/changes/report-pool-usage-exhaustion/proposal.md b/openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/proposal.md similarity index 100% rename from openspec/changes/report-pool-usage-exhaustion/proposal.md rename to openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/proposal.md diff --git a/openspec/changes/report-pool-usage-exhaustion/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/report-pool-usage-exhaustion/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/specs/responses-api-compat/spec.md diff --git a/openspec/changes/report-pool-usage-exhaustion/tasks.md b/openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/tasks.md similarity index 100% rename from openspec/changes/report-pool-usage-exhaustion/tasks.md rename to openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/tasks.md diff --git a/openspec/changes/require-beta-soak-before-stable/proposal.md b/openspec/changes/archive/2026-08-13-require-beta-soak-before-stable/proposal.md similarity index 100% rename from openspec/changes/require-beta-soak-before-stable/proposal.md rename to openspec/changes/archive/2026-08-13-require-beta-soak-before-stable/proposal.md diff --git a/openspec/changes/require-beta-soak-before-stable/specs/release-management/spec.md b/openspec/changes/archive/2026-08-13-require-beta-soak-before-stable/specs/release-management/spec.md similarity index 100% rename from openspec/changes/require-beta-soak-before-stable/specs/release-management/spec.md rename to openspec/changes/archive/2026-08-13-require-beta-soak-before-stable/specs/release-management/spec.md diff --git a/openspec/changes/require-beta-soak-before-stable/tasks.md b/openspec/changes/archive/2026-08-13-require-beta-soak-before-stable/tasks.md similarity index 100% rename from openspec/changes/require-beta-soak-before-stable/tasks.md rename to openspec/changes/archive/2026-08-13-require-beta-soak-before-stable/tasks.md diff --git a/openspec/changes/restore-proxy-architecture-ratchets/.openspec.yaml b/openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/.openspec.yaml similarity index 100% rename from openspec/changes/restore-proxy-architecture-ratchets/.openspec.yaml rename to openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/.openspec.yaml diff --git a/openspec/changes/restore-proxy-architecture-ratchets/context.md b/openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/context.md similarity index 100% rename from openspec/changes/restore-proxy-architecture-ratchets/context.md rename to openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/context.md diff --git a/openspec/changes/restore-proxy-architecture-ratchets/design.md b/openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/design.md similarity index 100% rename from openspec/changes/restore-proxy-architecture-ratchets/design.md rename to openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/design.md diff --git a/openspec/changes/restore-proxy-architecture-ratchets/proposal.md b/openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/proposal.md similarity index 100% rename from openspec/changes/restore-proxy-architecture-ratchets/proposal.md rename to openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/proposal.md diff --git a/openspec/changes/restore-proxy-architecture-ratchets/specs/proxy-architecture/spec.md b/openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/specs/proxy-architecture/spec.md similarity index 100% rename from openspec/changes/restore-proxy-architecture-ratchets/specs/proxy-architecture/spec.md rename to openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/specs/proxy-architecture/spec.md diff --git a/openspec/changes/restore-proxy-architecture-ratchets/tasks.md b/openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/tasks.md similarity index 100% rename from openspec/changes/restore-proxy-architecture-ratchets/tasks.md rename to openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/tasks.md diff --git a/openspec/changes/retry-account-proxy-connect-failures/design.md b/openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/design.md similarity index 100% rename from openspec/changes/retry-account-proxy-connect-failures/design.md rename to openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/design.md diff --git a/openspec/changes/retry-account-proxy-connect-failures/proposal.md b/openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/proposal.md similarity index 100% rename from openspec/changes/retry-account-proxy-connect-failures/proposal.md rename to openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/proposal.md diff --git a/openspec/changes/retry-account-proxy-connect-failures/specs/upstream-proxy-routing/spec.md b/openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/specs/upstream-proxy-routing/spec.md similarity index 100% rename from openspec/changes/retry-account-proxy-connect-failures/specs/upstream-proxy-routing/spec.md rename to openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/specs/upstream-proxy-routing/spec.md diff --git a/openspec/changes/retry-account-proxy-connect-failures/tasks.md b/openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/tasks.md similarity index 100% rename from openspec/changes/retry-account-proxy-connect-failures/tasks.md rename to openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/tasks.md diff --git a/openspec/changes/retry-model-capacity-errors/proposal.md b/openspec/changes/archive/2026-08-13-retry-model-capacity-errors/proposal.md similarity index 100% rename from openspec/changes/retry-model-capacity-errors/proposal.md rename to openspec/changes/archive/2026-08-13-retry-model-capacity-errors/proposal.md diff --git a/openspec/changes/retry-model-capacity-errors/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-retry-model-capacity-errors/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/retry-model-capacity-errors/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-retry-model-capacity-errors/specs/responses-api-compat/spec.md diff --git a/openspec/changes/retry-model-capacity-errors/tasks.md b/openspec/changes/archive/2026-08-13-retry-model-capacity-errors/tasks.md similarity index 100% rename from openspec/changes/retry-model-capacity-errors/tasks.md rename to openspec/changes/archive/2026-08-13-retry-model-capacity-errors/tasks.md diff --git a/openspec/changes/retry-server-is-overloaded/.openspec.yaml b/openspec/changes/archive/2026-08-13-retry-server-is-overloaded/.openspec.yaml similarity index 100% rename from openspec/changes/retry-server-is-overloaded/.openspec.yaml rename to openspec/changes/archive/2026-08-13-retry-server-is-overloaded/.openspec.yaml diff --git a/openspec/changes/retry-server-is-overloaded/design.md b/openspec/changes/archive/2026-08-13-retry-server-is-overloaded/design.md similarity index 100% rename from openspec/changes/retry-server-is-overloaded/design.md rename to openspec/changes/archive/2026-08-13-retry-server-is-overloaded/design.md diff --git a/openspec/changes/retry-server-is-overloaded/proposal.md b/openspec/changes/archive/2026-08-13-retry-server-is-overloaded/proposal.md similarity index 100% rename from openspec/changes/retry-server-is-overloaded/proposal.md rename to openspec/changes/archive/2026-08-13-retry-server-is-overloaded/proposal.md diff --git a/openspec/changes/retry-server-is-overloaded/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-retry-server-is-overloaded/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/retry-server-is-overloaded/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-retry-server-is-overloaded/specs/responses-api-compat/spec.md diff --git a/openspec/changes/retry-server-is-overloaded/tasks.md b/openspec/changes/archive/2026-08-13-retry-server-is-overloaded/tasks.md similarity index 100% rename from openspec/changes/retry-server-is-overloaded/tasks.md rename to openspec/changes/archive/2026-08-13-retry-server-is-overloaded/tasks.md diff --git a/openspec/changes/retry-stale-account-model-rejection/design.md b/openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/design.md similarity index 100% rename from openspec/changes/retry-stale-account-model-rejection/design.md rename to openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/design.md diff --git a/openspec/changes/retry-stale-account-model-rejection/proposal.md b/openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/proposal.md similarity index 100% rename from openspec/changes/retry-stale-account-model-rejection/proposal.md rename to openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/proposal.md diff --git a/openspec/changes/retry-stale-account-model-rejection/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/retry-stale-account-model-rejection/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/specs/responses-api-compat/spec.md diff --git a/openspec/changes/retry-stale-account-model-rejection/tasks.md b/openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/tasks.md similarity index 100% rename from openspec/changes/retry-stale-account-model-rejection/tasks.md rename to openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/tasks.md diff --git a/openspec/changes/self-host-mono-font/.openspec.yaml b/openspec/changes/archive/2026-08-13-self-host-mono-font/.openspec.yaml similarity index 100% rename from openspec/changes/self-host-mono-font/.openspec.yaml rename to openspec/changes/archive/2026-08-13-self-host-mono-font/.openspec.yaml diff --git a/openspec/changes/self-host-mono-font/proposal.md b/openspec/changes/archive/2026-08-13-self-host-mono-font/proposal.md similarity index 100% rename from openspec/changes/self-host-mono-font/proposal.md rename to openspec/changes/archive/2026-08-13-self-host-mono-font/proposal.md diff --git a/openspec/changes/self-host-mono-font/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-self-host-mono-font/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/self-host-mono-font/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-self-host-mono-font/specs/frontend-architecture/spec.md diff --git a/openspec/changes/self-host-mono-font/tasks.md b/openspec/changes/archive/2026-08-13-self-host-mono-font/tasks.md similarity index 100% rename from openspec/changes/self-host-mono-font/tasks.md rename to openspec/changes/archive/2026-08-13-self-host-mono-font/tasks.md diff --git a/openspec/changes/separate-dashboard-credit-metrics/proposal.md b/openspec/changes/archive/2026-08-13-separate-dashboard-credit-metrics/proposal.md similarity index 100% rename from openspec/changes/separate-dashboard-credit-metrics/proposal.md rename to openspec/changes/archive/2026-08-13-separate-dashboard-credit-metrics/proposal.md diff --git a/openspec/changes/separate-dashboard-credit-metrics/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-separate-dashboard-credit-metrics/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/separate-dashboard-credit-metrics/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-separate-dashboard-credit-metrics/specs/frontend-architecture/spec.md diff --git a/openspec/changes/separate-dashboard-credit-metrics/tasks.md b/openspec/changes/archive/2026-08-13-separate-dashboard-credit-metrics/tasks.md similarity index 100% rename from openspec/changes/separate-dashboard-credit-metrics/tasks.md rename to openspec/changes/archive/2026-08-13-separate-dashboard-credit-metrics/tasks.md diff --git a/openspec/changes/separate-service-and-usage-health-status/.openspec.yaml b/openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/.openspec.yaml similarity index 100% rename from openspec/changes/separate-service-and-usage-health-status/.openspec.yaml rename to openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/.openspec.yaml diff --git a/openspec/changes/separate-service-and-usage-health-status/design.md b/openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/design.md similarity index 100% rename from openspec/changes/separate-service-and-usage-health-status/design.md rename to openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/design.md diff --git a/openspec/changes/separate-service-and-usage-health-status/proposal.md b/openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/proposal.md similarity index 100% rename from openspec/changes/separate-service-and-usage-health-status/proposal.md rename to openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/proposal.md diff --git a/openspec/changes/separate-service-and-usage-health-status/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/separate-service-and-usage-health-status/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/specs/frontend-architecture/spec.md diff --git a/openspec/changes/separate-service-and-usage-health-status/tasks.md b/openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/tasks.md similarity index 100% rename from openspec/changes/separate-service-and-usage-health-status/tasks.md rename to openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/tasks.md diff --git a/openspec/changes/sequence-public-response-failures/.openspec.yaml b/openspec/changes/archive/2026-08-13-sequence-public-response-failures/.openspec.yaml similarity index 100% rename from openspec/changes/sequence-public-response-failures/.openspec.yaml rename to openspec/changes/archive/2026-08-13-sequence-public-response-failures/.openspec.yaml diff --git a/openspec/changes/sequence-public-response-failures/proposal.md b/openspec/changes/archive/2026-08-13-sequence-public-response-failures/proposal.md similarity index 100% rename from openspec/changes/sequence-public-response-failures/proposal.md rename to openspec/changes/archive/2026-08-13-sequence-public-response-failures/proposal.md diff --git a/openspec/changes/sequence-public-response-failures/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-sequence-public-response-failures/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/sequence-public-response-failures/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-sequence-public-response-failures/specs/responses-api-compat/spec.md diff --git a/openspec/changes/sequence-public-response-failures/tasks.md b/openspec/changes/archive/2026-08-13-sequence-public-response-failures/tasks.md similarity index 100% rename from openspec/changes/sequence-public-response-failures/tasks.md rename to openspec/changes/archive/2026-08-13-sequence-public-response-failures/tasks.md diff --git a/openspec/changes/sequence-websocket-health-after-settlement/.openspec.yaml b/openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/.openspec.yaml similarity index 100% rename from openspec/changes/sequence-websocket-health-after-settlement/.openspec.yaml rename to openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/.openspec.yaml diff --git a/openspec/changes/sequence-websocket-health-after-settlement/design.md b/openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/design.md similarity index 100% rename from openspec/changes/sequence-websocket-health-after-settlement/design.md rename to openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/design.md diff --git a/openspec/changes/sequence-websocket-health-after-settlement/proposal.md b/openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/proposal.md similarity index 100% rename from openspec/changes/sequence-websocket-health-after-settlement/proposal.md rename to openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/proposal.md diff --git a/openspec/changes/sequence-websocket-health-after-settlement/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/sequence-websocket-health-after-settlement/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/specs/api-keys/spec.md diff --git a/openspec/changes/sequence-websocket-health-after-settlement/tasks.md b/openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/tasks.md similarity index 100% rename from openspec/changes/sequence-websocket-health-after-settlement/tasks.md rename to openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/tasks.md diff --git a/openspec/changes/serialize-rate-limit-usage-reads/.openspec.yaml b/openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/.openspec.yaml similarity index 100% rename from openspec/changes/serialize-rate-limit-usage-reads/.openspec.yaml rename to openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/.openspec.yaml diff --git a/openspec/changes/serialize-rate-limit-usage-reads/context.md b/openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/context.md similarity index 100% rename from openspec/changes/serialize-rate-limit-usage-reads/context.md rename to openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/context.md diff --git a/openspec/changes/serialize-rate-limit-usage-reads/design.md b/openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/design.md similarity index 100% rename from openspec/changes/serialize-rate-limit-usage-reads/design.md rename to openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/design.md diff --git a/openspec/changes/serialize-rate-limit-usage-reads/proposal.md b/openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/proposal.md similarity index 100% rename from openspec/changes/serialize-rate-limit-usage-reads/proposal.md rename to openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/proposal.md diff --git a/openspec/changes/serialize-rate-limit-usage-reads/specs/query-caching/spec.md b/openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/specs/query-caching/spec.md similarity index 100% rename from openspec/changes/serialize-rate-limit-usage-reads/specs/query-caching/spec.md rename to openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/specs/query-caching/spec.md diff --git a/openspec/changes/serialize-rate-limit-usage-reads/tasks.md b/openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/tasks.md similarity index 100% rename from openspec/changes/serialize-rate-limit-usage-reads/tasks.md rename to openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/tasks.md diff --git a/openspec/changes/settings-reference-page/context.md b/openspec/changes/archive/2026-08-13-settings-reference-page/context.md similarity index 100% rename from openspec/changes/settings-reference-page/context.md rename to openspec/changes/archive/2026-08-13-settings-reference-page/context.md diff --git a/openspec/changes/settings-reference-page/proposal.md b/openspec/changes/archive/2026-08-13-settings-reference-page/proposal.md similarity index 100% rename from openspec/changes/settings-reference-page/proposal.md rename to openspec/changes/archive/2026-08-13-settings-reference-page/proposal.md diff --git a/openspec/changes/settings-reference-page/specs/user-documentation/spec.md b/openspec/changes/archive/2026-08-13-settings-reference-page/specs/user-documentation/spec.md similarity index 100% rename from openspec/changes/settings-reference-page/specs/user-documentation/spec.md rename to openspec/changes/archive/2026-08-13-settings-reference-page/specs/user-documentation/spec.md diff --git a/openspec/changes/settings-reference-page/tasks.md b/openspec/changes/archive/2026-08-13-settings-reference-page/tasks.md similarity index 100% rename from openspec/changes/settings-reference-page/tasks.md rename to openspec/changes/archive/2026-08-13-settings-reference-page/tasks.md diff --git a/openspec/changes/settle-aborted-terminal-bookkeeping/.openspec.yaml b/openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/.openspec.yaml similarity index 100% rename from openspec/changes/settle-aborted-terminal-bookkeeping/.openspec.yaml rename to openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/.openspec.yaml diff --git a/openspec/changes/settle-aborted-terminal-bookkeeping/proposal.md b/openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/proposal.md similarity index 100% rename from openspec/changes/settle-aborted-terminal-bookkeeping/proposal.md rename to openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/proposal.md diff --git a/openspec/changes/settle-aborted-terminal-bookkeeping/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/settle-aborted-terminal-bookkeeping/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/specs/api-keys/spec.md diff --git a/openspec/changes/settle-aborted-terminal-bookkeeping/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/settle-aborted-terminal-bookkeeping/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/specs/responses-api-compat/spec.md diff --git a/openspec/changes/settle-aborted-terminal-bookkeeping/tasks.md b/openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/tasks.md similarity index 100% rename from openspec/changes/settle-aborted-terminal-bookkeeping/tasks.md rename to openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/tasks.md diff --git a/openspec/changes/source-upstream-timing-metrics/design.md b/openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/design.md similarity index 100% rename from openspec/changes/source-upstream-timing-metrics/design.md rename to openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/design.md diff --git a/openspec/changes/source-upstream-timing-metrics/proposal.md b/openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/proposal.md similarity index 100% rename from openspec/changes/source-upstream-timing-metrics/proposal.md rename to openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/proposal.md diff --git a/openspec/changes/source-upstream-timing-metrics/specs/proxy-runtime-observability/spec.md b/openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/specs/proxy-runtime-observability/spec.md similarity index 100% rename from openspec/changes/source-upstream-timing-metrics/specs/proxy-runtime-observability/spec.md rename to openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/specs/proxy-runtime-observability/spec.md diff --git a/openspec/changes/source-upstream-timing-metrics/tasks.md b/openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/tasks.md similarity index 100% rename from openspec/changes/source-upstream-timing-metrics/tasks.md rename to openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/tasks.md diff --git a/openspec/changes/spill-unanchored-forks-on-account-cap/.openspec.yaml b/openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/.openspec.yaml similarity index 100% rename from openspec/changes/spill-unanchored-forks-on-account-cap/.openspec.yaml rename to openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/.openspec.yaml diff --git a/openspec/changes/spill-unanchored-forks-on-account-cap/proposal.md b/openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/proposal.md similarity index 100% rename from openspec/changes/spill-unanchored-forks-on-account-cap/proposal.md rename to openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/proposal.md diff --git a/openspec/changes/spill-unanchored-forks-on-account-cap/specs/proxy-admission-control/spec.md b/openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/specs/proxy-admission-control/spec.md similarity index 100% rename from openspec/changes/spill-unanchored-forks-on-account-cap/specs/proxy-admission-control/spec.md rename to openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/specs/proxy-admission-control/spec.md diff --git a/openspec/changes/spill-unanchored-forks-on-account-cap/tasks.md b/openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/tasks.md similarity index 100% rename from openspec/changes/spill-unanchored-forks-on-account-cap/tasks.md rename to openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/tasks.md diff --git a/openspec/changes/split-dashboard-routes/.openspec.yaml b/openspec/changes/archive/2026-08-13-split-dashboard-routes/.openspec.yaml similarity index 100% rename from openspec/changes/split-dashboard-routes/.openspec.yaml rename to openspec/changes/archive/2026-08-13-split-dashboard-routes/.openspec.yaml diff --git a/openspec/changes/split-dashboard-routes/proposal.md b/openspec/changes/archive/2026-08-13-split-dashboard-routes/proposal.md similarity index 100% rename from openspec/changes/split-dashboard-routes/proposal.md rename to openspec/changes/archive/2026-08-13-split-dashboard-routes/proposal.md diff --git a/openspec/changes/split-dashboard-routes/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-split-dashboard-routes/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/split-dashboard-routes/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-split-dashboard-routes/specs/frontend-architecture/spec.md diff --git a/openspec/changes/split-dashboard-routes/tasks.md b/openspec/changes/archive/2026-08-13-split-dashboard-routes/tasks.md similarity index 100% rename from openspec/changes/split-dashboard-routes/tasks.md rename to openspec/changes/archive/2026-08-13-split-dashboard-routes/tasks.md diff --git a/openspec/changes/thread-goal-openapi-operation-ids/.openspec.yaml b/openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/.openspec.yaml similarity index 100% rename from openspec/changes/thread-goal-openapi-operation-ids/.openspec.yaml rename to openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/.openspec.yaml diff --git a/openspec/changes/thread-goal-openapi-operation-ids/design.md b/openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/design.md similarity index 100% rename from openspec/changes/thread-goal-openapi-operation-ids/design.md rename to openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/design.md diff --git a/openspec/changes/thread-goal-openapi-operation-ids/proposal.md b/openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/proposal.md similarity index 100% rename from openspec/changes/thread-goal-openapi-operation-ids/proposal.md rename to openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/proposal.md diff --git a/openspec/changes/thread-goal-openapi-operation-ids/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/thread-goal-openapi-operation-ids/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/specs/responses-api-compat/spec.md diff --git a/openspec/changes/thread-goal-openapi-operation-ids/tasks.md b/openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/tasks.md similarity index 100% rename from openspec/changes/thread-goal-openapi-operation-ids/tasks.md rename to openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/tasks.md diff --git a/openspec/changes/warm-free-monthly-limit-reset/proposal.md b/openspec/changes/archive/2026-08-13-warm-free-monthly-limit-reset/proposal.md similarity index 100% rename from openspec/changes/warm-free-monthly-limit-reset/proposal.md rename to openspec/changes/archive/2026-08-13-warm-free-monthly-limit-reset/proposal.md diff --git a/openspec/changes/warm-free-monthly-limit-reset/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-warm-free-monthly-limit-reset/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/warm-free-monthly-limit-reset/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-warm-free-monthly-limit-reset/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/warm-free-monthly-limit-reset/tasks.md b/openspec/changes/archive/2026-08-13-warm-free-monthly-limit-reset/tasks.md similarity index 100% rename from openspec/changes/warm-free-monthly-limit-reset/tasks.md rename to openspec/changes/archive/2026-08-13-warm-free-monthly-limit-reset/tasks.md diff --git a/openspec/changes/windows-sqlite-url-encoding/.openspec.yaml b/openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/.openspec.yaml similarity index 100% rename from openspec/changes/windows-sqlite-url-encoding/.openspec.yaml rename to openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/.openspec.yaml diff --git a/openspec/changes/windows-sqlite-url-encoding/proposal.md b/openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/proposal.md similarity index 100% rename from openspec/changes/windows-sqlite-url-encoding/proposal.md rename to openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/proposal.md diff --git a/openspec/changes/windows-sqlite-url-encoding/specs/database-backends/spec.md b/openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/specs/database-backends/spec.md similarity index 100% rename from openspec/changes/windows-sqlite-url-encoding/specs/database-backends/spec.md rename to openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/specs/database-backends/spec.md diff --git a/openspec/changes/windows-sqlite-url-encoding/specs/database-migrations/spec.md b/openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/specs/database-migrations/spec.md similarity index 100% rename from openspec/changes/windows-sqlite-url-encoding/specs/database-migrations/spec.md rename to openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/specs/database-migrations/spec.md diff --git a/openspec/changes/windows-sqlite-url-encoding/tasks.md b/openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/tasks.md similarity index 100% rename from openspec/changes/windows-sqlite-url-encoding/tasks.md rename to openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/tasks.md diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/.openspec.yaml b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/.openspec.yaml new file mode 100644 index 0000000000..4af864176c --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-14 diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/design.md b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/design.md new file mode 100644 index 0000000000..8edd2e5838 --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/design.md @@ -0,0 +1,35 @@ +## Context + +The Responses request models, upstream transports, request logs, and model registry already carry service tiers as normalized strings. They therefore preserve `ultrafast` without a transport change and can route it using live per-account catalog metadata. The remaining hard-coded allowlists are the API-key CRUD contract and dashboard controls. + +OpenAI documents `ultrafast` as an access-controlled processing tier currently available for `gpt-5.6-sol`. Entitlement must therefore come from each account's live upstream catalog instead of a static plan or bootstrap assumption. + +## Goals / Non-Goals + +**Goals:** + +- Make `ultrafast` a supported canonical API-key service tier. +- Expose the tier through the existing dashboard API-key controls. +- Preserve existing entitlement-aware account routing and response-tier logging. +- Add focused regression coverage and user-facing compatibility notes. + +**Non-Goals:** + +- Invent an `ultrafast` model-name alias. +- Advertise Ultrafast from bootstrap metadata or grant it to a plan statically. +- Add a setting, dependency, or database migration. +- Guess a distinct Ultrafast token price that OpenAI has not published. + +## Decisions + +1. Add `ultrafast` only to the existing backend and frontend API-key tier allowlists. The request models and transports already pass it through, so adding another normalization layer would duplicate working behavior. +2. Keep `ultrafast` canonical. Unlike the legacy `fast` alias, it is an upstream wire value and must not normalize to `priority`. +3. Reuse live model-catalog routing. An explicit or enforced Ultrafast request can select only accounts whose catalog advertises that tier; the existing enforced-tier fallback still removes it for models that do not advertise it. +4. Do not add Ultrafast to the bundled model catalog. Static metadata cannot prove access to an access-controlled preview and would expose a tier that an imported account may not hold. +5. Keep pricing unchanged. No distinct public Ultrafast token price is available in the official OpenAI documentation, so this change does not introduce a speculative multiplier. + +## Risks / Trade-offs + +- [An entitled account's catalog does not advertise `ultrafast`] → The existing explicit-tier routing error remains visible instead of silently selecting an ineligible account. +- [OpenAI later publishes distinct Ultrafast pricing] → Add the published rates in a focused pricing change before claiming separate cost accuracy. +- [Dashboard-visible option requires review evidence] → Include before and after screenshots in the PR body as required by the simplicity gates. diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/proposal.md b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/proposal.md new file mode 100644 index 0000000000..cc3ec42afe --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/proposal.md @@ -0,0 +1,27 @@ +## Why + +OpenAI introduced an access-controlled Ultrafast processing tier for `gpt-5.6-sol`. codex-lb already preserves unknown request tier strings, but its API-key policy and dashboard reject `ultrafast`, leaving the feature incomplete and untested. + +## What Changes + +- Accept and persist `ultrafast` as an API-key-enforced service tier. +- Expose Ultrafast in the API key create and edit controls. +- Preserve and forward the canonical `ultrafast` value through Responses-compatible routes. +- Use live upstream model-catalog entitlement data to route Ultrafast requests only to advertising accounts. +- Document the upstream availability constraint and add focused regression coverage. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `api-keys`: allow dashboard API keys to enforce the canonical `ultrafast` tier. +- `responses-api-compat`: define pass-through behavior for explicit and enforced Ultrafast requests. +- `model-catalog-compat`: define entitlement-aware account routing for the access-controlled tier. + +## Impact + +The change affects API-key request validation and normalization, dashboard API-key forms and translations, Responses compatibility documentation, model-catalog routing tests, and focused backend/frontend tests. It adds no dependency, setting, database migration, or bootstrap entitlement metadata. diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/api-keys/spec.md new file mode 100644 index 0000000000..591b8408d6 --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/api-keys/spec.md @@ -0,0 +1,17 @@ +## ADDED Requirements + +### Requirement: API keys can enforce the Ultrafast service tier + +The dashboard API key CRUD surface MUST accept and persist `ultrafast` as a canonical enforced service tier. The service MUST return the same canonical value and MUST NOT normalize it to `priority`. + +#### Scenario: Create an API key with Ultrafast enforcement + +- **WHEN** a dashboard client creates an API key with `enforcedServiceTier: "ultrafast"` +- **THEN** the request is accepted +- **AND** the persisted and returned enforced service tier is `ultrafast` + +#### Scenario: Enforce Ultrafast on an advertising model + +- **GIVEN** an account model advertises the `ultrafast` service tier +- **WHEN** a request uses an API key whose enforced service tier is `ultrafast` +- **THEN** the upstream request carries `service_tier: "ultrafast"` diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/model-catalog-compat/spec.md new file mode 100644 index 0000000000..b69a2ced33 --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/model-catalog-compat/spec.md @@ -0,0 +1,17 @@ +## ADDED Requirements + +### Requirement: Ultrafast routing follows live account entitlement + +The system MUST treat `ultrafast` as an access-controlled service tier and MUST derive account eligibility from live or retained per-account upstream catalog metadata. The bundled bootstrap catalog MUST NOT invent Ultrafast entitlement. + +#### Scenario: Only an advertising account is eligible + +- **GIVEN** two accounts advertise `gpt-5.6-sol` +- **AND** only one account advertises the `ultrafast` service tier +- **WHEN** a request explicitly asks for `service_tier: "ultrafast"` +- **THEN** account selection considers only the advertising account + +#### Scenario: Bootstrap metadata does not grant preview access + +- **WHEN** no live or retained account catalog advertises `ultrafast` +- **THEN** bootstrap model metadata does not expose or grant that tier diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..c2bace9663 --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/responses-api-compat/spec.md @@ -0,0 +1,15 @@ +## ADDED Requirements + +### Requirement: Responses routes preserve the Ultrafast service tier + +Responses-compatible routes MUST accept the canonical `ultrafast` service tier and MUST forward it unchanged. When upstream reports the actual response tier, request logging MUST preserve `ultrafast` using the existing requested, actual, and billable tier contract. + +#### Scenario: Explicit Ultrafast request is forwarded + +- **WHEN** a client sends a Responses request with `service_tier: "ultrafast"` +- **THEN** the forwarded upstream payload contains `service_tier: "ultrafast"` + +#### Scenario: Upstream confirms Ultrafast processing + +- **WHEN** upstream completes a request with `response.service_tier: "ultrafast"` +- **THEN** the actual and billable request-log tiers are `ultrafast` diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/tasks.md b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/tasks.md new file mode 100644 index 0000000000..bf283c6da4 --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/tasks.md @@ -0,0 +1,15 @@ +## 1. Backend support + +- [x] 1.1 Accept and persist canonical `ultrafast` API-key enforcement values +- [x] 1.2 Add focused request, API-key, catalog-routing, and logging regression coverage + +## 2. Dashboard and documentation + +- [x] 2.1 Add Ultrafast to dashboard schemas, create/edit controls, and translations +- [x] 2.2 Add frontend schema and interaction coverage for the new option +- [x] 2.3 Document official availability, entitlement behavior, and a concrete request example + +## 3. Verification + +- [x] 3.1 Validate OpenSpec artifacts and run focused backend/frontend checks +- [x] 3.2 Run the repository local CI gate and capture dashboard before/after evidence diff --git a/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/.openspec.yaml b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/.openspec.yaml new file mode 100644 index 0000000000..41c30bab88 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-19 diff --git a/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/context.md b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/context.md new file mode 100644 index 0000000000..08de111df6 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/context.md @@ -0,0 +1,54 @@ +# Previous-response replay owner fencing + +## Purpose + +This change distinguishes continuation-anchor recovery from payload +portability. A retry may safely remove a stale anchor yet still be forbidden +from changing accounts because retained request items remain account-scoped. + +## Example + +Account A first receives: + +```json +{ + "previous_response_id": "resp_owner", + "input": [ + { + "type": "reasoning", + "id": "rs_owner", + "encrypted_content": "owner-bound-ciphertext" + } + ] +} +``` + +If a pre-visible failure triggers stale-anchor recovery, the proxy may remove +`previous_response_id` only as part of a verified replay. Because the retained +encrypted reasoning is not account-neutral, the replacement remains bound to +account A. Account B must never receive it. + +An ordinary fresh request containing only portable user input can pass the +canonical predicate and may use normal account selection. + +A selected account is not recorded as owner when transport evidence proves the +request failed before dispatch. The body may then make its first real dispatch +on another eligible account. Ambiguous failures remain pinned. + +HTTP bridge operation IDs are proxy-owned but still identify an in-flight +operation. A bridge retry carrying an existing operation ID remains on its +current account unless the operation is explicitly rebound before selection. + +## Operational Notes + +- Owner-unavailable failures are internal retry decisions; they do not add a + setting or require operator action. +- Existing file ownership remains an independent strict pin. +- Verified fresh-body installation clears the old dispatch owner atomically + with replacing the request body. +- A bound request may perform one forced authentication refresh on the same + owner; it does not become eligible for cross-account auth failover. +- API-key reservation settlement still completes before deferred account-health + writes. +- The change covers HTTP streaming, HTTP bridge, and direct WebSocket paths so + operators do not observe transport-dependent account ownership. diff --git a/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/design.md b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/design.md new file mode 100644 index 0000000000..2b0ab9f16c --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/design.md @@ -0,0 +1,110 @@ +## Context + +Responses requests can carry both a server-side continuation anchor and +client-retained material. Removing a stale `previous_response_id` does not make +the remaining body portable: encrypted reasoning, account-scoped probe items, +and other retained state can still belong to the account that first received +the request. + +The selector already supports strict required-account routing for known +previous-response and file owners. The missing state is payload dispatch +provenance: after a pre-visible retry excludes an account, later selection can +no longer tell that the retained body was already dispatched there. + +## Goals / Non-Goals + +**Goals:** + +- Bind nonportable payloads to their first dispatch account. +- Enforce the binding consistently in HTTP streaming, HTTP bridge, and direct + WebSocket retry paths. +- Allow cross-account replay only after exact-wire verification proves the + resulting request is an account-neutral fresh replay. +- Preserve existing settlement, health-write, and file-owner invariants. + +**Non-Goals:** + +- Changing stale previous-response error classification from PR #1818. +- Preserving or reshaping unrelated bare/raw upstream error fields. +- Changing public API envelopes, retry counts, quota accounting, or settings. +- Making encrypted reasoning or account-scoped probe items portable. + +## Decisions + +### Use the canonical portability predicate + +Every candidate body is evaluated with +`responses_payload_is_account_neutral_fresh_replay`. Ad hoc checks for files or +`previous_response_id` are insufficient because account scope can live in +retained input items. + +Alternative: extend each transport's file checks. Rejected because it +duplicates an incomplete allowlist and already failed to catch encrypted +reasoning. + +### Bind on first nonportable dispatch + +A request-local dispatch-owner ID is authorized before the first nonportable +payload is sent and persisted after the first upstream event or normal stream +completion. Ambiguous/post-dispatch failures also preserve that owner, while a +positively confirmed pre-dispatch transport failure does not create one. Every +later selection treats a persisted owner like any other strict continuity +requirement. + +Alternative: infer ownership from the current preferred account. Rejected +because retry branches intentionally clear or replace preference state. + +### Clear ownership only after verified neutral replay + +Verified stale-anchor recovery may replace the wire body with a reconstructed +fresh request. The dispatch binding is cleared only when that exact replacement +passes the canonical account-neutral predicate. Body replacement and +owner-fence clearing occur in one transition so a retry cannot observe mixed +state. A verified nonneutral replacement may be installed for a same-owner +retry, but that transition preserves the existing dispatch-owner fence. + +Alternative: clear ownership whenever the anchor is removed. Rejected because +the reproduced defect retained owner-bound ciphertext after anchor removal. + +### Treat proxy-owned operation metadata as account-bound + +HTTP bridge sends may add `codex_lb_operation_id` after request preparation. +Until a dedicated rebind path replaces that operation identity, selection +treats the request as nonportable and requires the current account. + +Alternative: remove the proxy-owned field before portability checks. Rejected +because normalization would authorize a different account while preserving the +same operation identity on the final wire request. + +### Fail closed across transport-specific recovery + +Trusted Access migration/degradation, owner exclusion, bridge reconnect, and +WebSocket account switching may not bypass payload ownership. If the owner +cannot satisfy the retry, the proxy returns the stable owner-unavailable error +without dispatching the retained body elsewhere. + +A generic authentication failure is split into two decisions: one forced token +refresh may replay a bound body on the same owner, while owner exclusion or +cross-account migration still requires an atomically installed neutral body. +Permanent authentication failure remains terminal for a bound body. + +## Risks / Trade-offs + +- **Fewer automatic retries for account-bound bodies** → This is intentional; + confidentiality and continuation correctness outrank cross-account fallback. +- **False nonportability** → The canonical predicate is an explicit allowlist, + so unknown retained item types fail closed. +- **Transport drift** → Shared helpers plus focused HTTP, bridge, and WebSocket + regressions keep the invariant aligned. +- **Settlement regression** → The change does not move reservation settlement + or deferred health writes; existing settlement tests remain mandatory. + +## Migration Plan + +No data or configuration migration is required. Deploy the proxy code normally. +Rollback is a code rollback; no persisted format changes. + +## Open Questions + +None. Current-main runtime probes reproduce the cross-account dispatch and the +existing selector already provides the strict owner-routing primitive. diff --git a/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/proposal.md b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/proposal.md new file mode 100644 index 0000000000..8ad755213e --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/proposal.md @@ -0,0 +1,46 @@ +## Why + +A pre-visible Responses retry can remove a `previous_response_id` anchor while +retaining account-scoped request material, exclude the original account, and +dispatch the retained payload to another account. Encrypted reasoning was +reproduced crossing accounts through the HTTP stream path; the same missing +dispatch provenance affects HTTP bridge and direct WebSocket retries. + +PR #1818 fixed parameterless stale-response classification but intentionally +did not add payload-owner fencing. The remaining defect violates account +ownership even when session/file continuity and API-key settlement work as +designed. + +## What Changes + +- Classify exact-wire replay candidates with the canonical + account-neutral-fresh-replay predicate. +- Bind every nonportable Responses payload to its first dispatch account. +- Merge payload ownership with previous-response and file ownership during + every HTTP stream, HTTP bridge, and direct WebSocket selection. +- Fail closed rather than excluding the owner or moving retained + account-scoped material during Trusted Access migration/degradation. +- Clear payload ownership only after verified anchor removal produces a + canonical account-neutral fresh replay. +- Keep proxy-owned operation metadata on its current account unless a + dedicated operation-rebind path replaces that identity before selection. +- Preserve existing file pinning, API-key settlement ordering, error + classification, and raw error-envelope behavior. + +## Capabilities + +### Modified Capabilities + +- `responses-api-compat`: require account-bound retry payloads to remain on + their dispatch owner across all Responses transports. + +## Impact + +- **Affected code:** Responses replay safety, HTTP streaming retries, HTTP + bridge reconnects, and direct WebSocket account switching. +- **Affected tests:** proxy streaming utilities and WebSocket Responses + integration tests. +- **API/schema changes:** none. +- **Configuration changes:** none. +- **Security impact:** prevents account-scoped request material from crossing + account boundaries during internal retries. diff --git a/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..e09e31f394 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/specs/responses-api-compat/spec.md @@ -0,0 +1,92 @@ +## ADDED Requirements + +### Requirement: Account-bound retries remain on their dispatch owner + +The proxy MUST bind a Responses request body that is not a canonical +account-neutral fresh replay to the account that first receives that exact +body. Every later selection for that request MUST treat the dispatch owner as a +strict required account across HTTP streaming, HTTP bridge, and direct +WebSocket transports. + +The proxy MUST NOT exclude the dispatch owner and send the retained body to a +different account during stale-anchor recovery, retryable account failure, +Trusted Access migration or degradation, bridge reconnect, or WebSocket account +switching. If the required owner is unavailable, the proxy MUST fail closed +without dispatching the retained body to another account. + +The proxy MAY perform one forced authentication refresh and replay a retained +account-bound body on the same dispatch owner. It MUST NOT use that refresh to +exclude the owner or migrate the body to another account, and a permanent +authentication failure MUST remain terminal for the bound body. + +The proxy MAY clear the dispatch-owner binding only after verified recovery +replaces the exact wire body and the replacement passes the canonical +account-neutral-fresh-replay predicate. Removing `previous_response_id` alone +MUST NOT make retained account-scoped input portable. + +Proxy-owned operation metadata that will be added at the send boundary MUST +remain bound to the current account unless an explicit operation-rebind path +replaces that identity before account selection. Installing a verified fresh +body and clearing its dispatch-owner binding MUST occur as one state +transition. + +#### Scenario: Encrypted reasoning remains on its first dispatch account + +- **GIVEN** account A first receives a Responses request containing encrypted + reasoning or another account-scoped retained item +- **WHEN** a pre-visible retry excludes account A or requests a differently + authorized account +- **THEN** the proxy does not dispatch the retained body to account B +- **AND** the retry fails closed when account A is unavailable + +#### Scenario: Verified account-neutral fresh replay may change accounts + +- **GIVEN** verified recovery removes a stale continuation anchor +- **AND** the exact replacement body contains only canonical account-neutral + fresh input +- **WHEN** normal retry selection chooses account B +- **THEN** the proxy may dispatch the replacement body to account B + +#### Scenario: Confirmed pre-dispatch failure does not create an owner + +- **GIVEN** account A is selected for a nonportable Responses body +- **WHEN** transport evidence confirms the request failed before any upstream + bytes were dispatched +- **THEN** the proxy does not record account A as the dispatch owner +- **AND** normal retry selection may dispatch the body first on account B + +#### Scenario: HTTP bridge preserves payload ownership + +- **GIVEN** an HTTP bridge request has already dispatched a nonportable body to + account A +- **WHEN** pre-created recovery or reconnect selection excludes account A +- **THEN** the bridge does not submit that body on account B + +#### Scenario: Direct WebSocket preserves payload ownership + +- **GIVEN** a direct WebSocket request has already dispatched a nonportable body + to account A +- **WHEN** retry handling prepares an account switch +- **THEN** the proxy rejects the switch unless the exact replacement body is a + canonical account-neutral fresh replay + +#### Scenario: Bound authentication refresh stays on the owner + +- **GIVEN** a nonportable body is bound to account A +- **WHEN** account A reports a refreshable authentication failure before + visible output +- **THEN** the proxy may refresh and replay once on account A +- **AND** it does not dispatch the retained body to account B + +#### Scenario: HTTP bridge operation identity remains on its owner + +- **GIVEN** an HTTP bridge retry retains a proxy-owned operation identity +- **AND** no explicit operation rebind has replaced that identity +- **WHEN** retry selection evaluates another account +- **THEN** the bridge requires the current operation owner + +#### Scenario: Existing settlement ordering is unchanged + +- **GIVEN** an API-key reservation requires settlement during the failed retry +- **WHEN** account health is updated +- **THEN** required settlement still completes before deferred health writes diff --git a/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/tasks.md b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/tasks.md new file mode 100644 index 0000000000..53d6b3264a --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/tasks.md @@ -0,0 +1,29 @@ +## 1. Regression coverage + +- [x] 1.1 Add a deterministic HTTP streaming regression proving account-bound + encrypted reasoning never dispatches to a Trusted Access replacement. +- [x] 1.2 Add direct WebSocket regressions for unanchored and verified-fresh + account-bound request bodies. +- [x] 1.3 Add HTTP bridge coverage for owner exclusion and account-neutral + replacement controls. +- [x] 1.4 Add a confirmed pre-dispatch regression proving owner registration + waits for actual upstream dispatch. + +## 2. Owner-fencing implementation + +- [x] 2.1 Route every replay candidate through the canonical account-neutral + fresh-replay predicate. +- [x] 2.2 Bind nonportable HTTP stream payloads to their first dispatch owner + and require that owner during later selections. +- [x] 2.3 Enforce the same binding in HTTP bridge and direct WebSocket account + switching without changing settlement ordering. + +## 3. Verification and publication + +- [x] 3.1 Capture genuine focused RED, implement the minimal owner fence, and + run focused HTTP/bridge/WebSocket tests GREEN. +- [x] 3.2 Run diagnostics, Ruff, typecheck, architecture gates, full affected + tests, and strict affected OpenSpec validation. +- [x] 3.3 Execute an isolated real-surface account-switch scenario proving no + cross-account dispatch and an account-neutral control. +- [x] 3.4 Complete independent review and sync the verified change for archive. diff --git a/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/.openspec.yaml b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/.openspec.yaml new file mode 100644 index 0000000000..41c30bab88 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-19 diff --git a/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/context.md b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/context.md new file mode 100644 index 0000000000..2e24173f3c --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/context.md @@ -0,0 +1,54 @@ +# Context: TTFT dashboard SQL datasource binding + +## Purpose + +Make the sidecar-provisioned TTFT dashboard immediately bindable to an +operator's PostgreSQL datasource without baking a cluster-specific UID into +the chart. + +## Decisions + +- `DS_SQL` remains a runtime dashboard variable because datasource UIDs differ + across Grafana installations. +- The variable is visible and single-select so operators can inspect and + change the active PostgreSQL datasource without editing dashboard JSON. +- Every panel uses Grafana's typed datasource object with PostgreSQL plugin + type and `${DS_SQL}` UID. One dashboard variable remains the single source + of truth for all four panels. +- The dashboard stays in `dashboards/*.json`; the existing Helm template, + sidecar labels, folder annotation, and optional title override remain + unchanged. + +## Constraints + +- Do not add a Helm value for a datasource UID: datasource selection is a + Grafana runtime concern and a new chart setting would duplicate the + dashboard variable. +- Do not provision PostgreSQL credentials or a Grafana datasource from this + chart. +- Preserve each panel's SQL, layout, IDs, titles, and visualization type. + +## Failure Modes + +- A scalar `"datasource": "${DS_SQL}"` is ambiguous to modern Grafana and can + be resolved as a literal missing UID. +- A hidden, multi-value, or include-all variable can make panel execution + non-deterministic or leave operators unable to repair a stale selection. +- A variable that accepts non-PostgreSQL plugins can select a datasource that + cannot execute the dashboard's SQL. + +## Example + +After the Grafana sidecar imports `ttft-breakdown.json`, an operator opens the +dashboard and selects datasource UID `codex-lb-postgres` from the `DS_SQL` +dropdown. All four panels resolve to: + +```json +{ + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}" +} +``` + +Grafana substitutes `codex-lb-postgres` at runtime and executes every TTFT +query against that datasource. diff --git a/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/design.md b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/design.md new file mode 100644 index 0000000000..2c151dca89 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/design.md @@ -0,0 +1,98 @@ +## Context + +The Helm chart packages `ttft-breakdown.json` unchanged in a +sidecar-discoverable ConfigMap. The dashboard's four SQL panels currently use +the legacy scalar `"${DS_SQL}"` datasource form while `templating.list` is +empty. Grafana 12.4.4 therefore has no runtime value to interpolate and can +resolve the placeholder as a missing UID. + +Datasource UIDs and credentials are installation-specific. The chart must +remain portable and must not take ownership of provisioning an operator's +Grafana PostgreSQL datasource. + +## Goals / Non-Goals + +**Goals:** + +- Make the selected PostgreSQL datasource explicit, visible, and deterministic + at dashboard runtime. +- Route all four panels through the same selected UID using Grafana 12.4.4's + typed datasource-reference schema. +- Preserve current SQL, layout, sidecar packaging, title overrides, and chart + values. + +**Non-Goals:** + +- Provisioning a Grafana datasource, PostgreSQL credentials, or database + permissions. +- Adding a Helm value for a cluster-specific datasource UID. +- Changing TTFT queries, visualizations, panel layout, navigation, or + application runtime behavior. + +## Decisions + +### Use a classic datasource template variable + +Declare `DS_SQL` with `type: datasource` and plugin query +`grafana-postgresql-datasource`. Keep it visible, single-select, and without +an all-datasources option. + +Alternative: hard-code a datasource UID in Helm. Rejected because UIDs are +installation-specific and would require another chart setting for a Grafana +runtime concern. + +### Use typed panel datasource references + +Each panel uses: + +```json +{ + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}" +} +``` + +Grafana 12.4.4 defines panel datasources as `{type, uid}` references and +interpolates variables in the UID field. The existing scalar form is legacy +input and does not satisfy the current schema. + +Alternative: leave panel references scalar and only add the variable. +Rejected because it preserves the ambiguous representation that produced the +missing-datasource state. + +### Preserve the existing Helm packaging seam + +Keep dashboard JSON under `dashboards/` and let +`templates/grafana-dashboard.yaml` package it unchanged. A rendered ConfigMap +test proves the runtime variable and typed panel references survive Helm. + +Alternative: generate the variable in the template. Rejected because it +duplicates dashboard structure in Go templates and makes standalone dashboard +validation harder. + +## Risks / Trade-offs + +- **No PostgreSQL datasource exists** → The dropdown has no valid selection; + chart documentation states that operators must provision and select one. +- **A saved selection becomes stale** → The variable remains visible so the + operator can select another ordinary PostgreSQL datasource. +- **A datasource connects but lacks request-log access** → Grafana reports the + database/query error normally; this change only owns datasource resolution. +- **Grafana schema behavior changes** → Focused artifact tests and a real + Grafana 12.4.4 API/browser scenario lock the supported contract. + +## Migration Plan + +1. Upgrade or redeploy the chart with Grafana dashboard sidecar support + enabled. +2. Let the sidecar replace the dashboard ConfigMap payload. +3. Open the TTFT dashboard and select the PostgreSQL datasource that points to + the codex-lb database. + +Rollback is a chart rollback to the previous dashboard JSON. No database, +application, secret, or chart-value migration is involved. + +## Open Questions + +None. Grafana 12.4.4 documentation and source confirm the plugin ID, +datasource-variable flags, typed panel reference, and UID interpolation path. diff --git a/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/proposal.md b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/proposal.md new file mode 100644 index 0000000000..2c53849563 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/proposal.md @@ -0,0 +1,37 @@ +## Why + +The shipped TTFT breakdown dashboard references `${DS_SQL}` on every panel, +but it does not declare that runtime variable. Grafana therefore treats the +literal placeholder as a datasource UID and renders +`Datasource ${DS_SQL} was not found` instead of executing the PostgreSQL +queries. + +## What Changes + +- Declare `DS_SQL` as a visible, single-select runtime datasource variable + restricted to the PostgreSQL datasource plugin. +- Bind all four TTFT panels through typed datasource objects whose UID is the + selected `DS_SQL` value. +- Preserve Helm sidecar ConfigMap packaging and title overrides. +- Document that operators select the PostgreSQL datasource after the sidecar + provisions the dashboard. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `proxy-runtime-observability`: The shipped TTFT dashboard MUST resolve its + SQL panels through an operator-selected PostgreSQL datasource. + +## Impact + +- `deploy/helm/codex-lb/dashboards/ttft-breakdown.json` +- `deploy/helm/codex-lb/README.md` +- focused dashboard-artifact and rendered-ConfigMap tests + +There is no application runtime, API, database schema, chart value, sidecar +label, or navigation change. diff --git a/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/specs/proxy-runtime-observability/spec.md b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/specs/proxy-runtime-observability/spec.md new file mode 100644 index 0000000000..7c15efe1cc --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/specs/proxy-runtime-observability/spec.md @@ -0,0 +1,37 @@ +## MODIFIED Requirements + +### Requirement: 24-hour TTFT breakdown queries are available + +Operators MUST have an OpenSpec context runbook or dashboard artifact with +24-hour TTFT breakdown queries by user agent group, upstream transport, +model/cache ratio, session gap cohort, prompt size cohort, and prewarm +status/outcome. + +The shipped Grafana TTFT dashboard MUST declare a visible, single-select +runtime datasource variable named `DS_SQL` that is restricted to PostgreSQL. +Every SQL panel MUST bind to the selected UID through a typed PostgreSQL +datasource object. The Helm chart MUST preserve the dashboard in its existing +sidecar-discoverable ConfigMap, and chart documentation MUST tell operators to +select the PostgreSQL datasource in Grafana. + +#### Scenario: Operator investigates TTFT regression + +- **WHEN** an operator needs to inspect the last 24 hours of request-log + latency +- **THEN** the repository provides SQL that reports p50, p90, p95 TTFT and + total latency for the requested breakdowns + +#### Scenario: Sidecar-provisioned dashboard resolves the selected database + +- **GIVEN** the Helm chart renders the Grafana dashboard ConfigMap +- **AND** Grafana has a PostgreSQL datasource available +- **WHEN** the operator selects that datasource through `DS_SQL` +- **THEN** all four TTFT panels resolve to the selected datasource UID +- **AND** no panel reports `Datasource ${DS_SQL} was not found` + +#### Scenario: Datasource choice remains explicit and deterministic + +- **WHEN** Grafana loads the TTFT dashboard +- **THEN** `DS_SQL` is visible to the operator +- **AND** it permits exactly one PostgreSQL datasource selection +- **AND** it does not offer an all-datasources selection diff --git a/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/tasks.md b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/tasks.md new file mode 100644 index 0000000000..c06cdf7091 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/tasks.md @@ -0,0 +1,26 @@ +## 1. Regression coverage + +- [x] 1.1 Add dashboard JSON assertions for the visible single-select + PostgreSQL `DS_SQL` variable and all four typed panel bindings. +- [x] 1.2 Add a rendered ConfigMap assertion proving Helm preserves the + runtime datasource contract. + +## 2. Dashboard and operator documentation + +- [x] 2.1 Declare `DS_SQL` and convert all TTFT panels to typed PostgreSQL + datasource objects without changing SQL or layout. +- [x] 2.2 Document Grafana-side PostgreSQL datasource selection while + preserving the existing sidecar deployment model. + +## 3. Verification + +- [x] 3.1 Capture focused RED, implement the minimal artifact fix, and run the + focused tests GREEN. +- [x] 3.2 Run changed-file diagnostics, Ruff, typecheck, Helm architecture and + rendering gates, strict affected OpenSpec validation, and final diff + review. +- [x] 3.3 Provision the exact rendered dashboard into isolated Grafana 12.4.4 + with a synthetic PostgreSQL datasource; inspect the API and 1440x900 + browser surface and capture evidence. +- [x] 3.4 Sync the verified delta into the owning capability and archive the + completed change before publication. diff --git a/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/.openspec.yaml b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/.openspec.yaml new file mode 100644 index 0000000000..41c30bab88 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-19 diff --git a/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/design.md b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/design.md new file mode 100644 index 0000000000..9bfff7fd5f --- /dev/null +++ b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/design.md @@ -0,0 +1,51 @@ +## Context + +See [proposal.md](proposal.md) for the production incident. The shared classifier currently accepts canonical `code = "previous_response_not_found"`, or `code = "invalid_request_error"` only when `param = "previous_response_id"` and the message says the response was not found. The observed upstream frame has neither `code` nor `param`; normalization yields `code = "invalid_request_error"`, and its new ``Invalid `previous_response_id`.`` wording fails the message test. + +Every downstream recovery mechanism already depends on this classifier. Direct WebSocket full resends retain a safe request body without the anchor and can replay transparently. Delta-only Codex-native requests receive a sanitized canonical code that the client uses to resend full local history; public `/v1` traffic receives generic `stream_incomplete` masking. The classifier miss bypasses all of those paths. + +The incident data also contained upstream WebSocket interruptions, downstream disconnects, and connection-limit rotations. Those events explain why otherwise recent response ids can become unusable across connection boundaries, but they do not justify changing cleanup, retry, or transport policy here. The cleanup-budget and phase-attribution changes from upstream PRs #1723 and #1726 are already present on the affected deployment. + +## Goals / Non-Goals + +**Goals:** + +- Recognize the exact newly observed stale-anchor envelope at the shared classification boundary. +- Preserve the existing safety gates that decide between transparent replay, client-assisted full resend, and fail-closed masking. +- Keep false-positive risk bounded with explicit code, parameter, and exact-message checks. + +**Non-Goals:** + +- Do not retry delta-only input without conversation history. +- Do not change WebSocket cleanup budgets, connection lifetime, account routing, health penalties, or retry-circuit policy. +- Do not infer that every generic invalid request is a stale anchor. + +## Decisions + +### Extend the shared semantic classifier + +Add a normalized-message predicate for ``Invalid `previous_response_id``` with zero or one trailing period, and accept it only when the normalized error code is `invalid_request_error` and `param` is absent or already names `previous_response_id`. Reject other trailing punctuation and every different named parameter. Normalize `error.type` at the WebSocket rewrite helper just as its detection and retry-decision callers already do; without that consistency, the first classifier can recognize a code-less frame while the later rewrite still relays it raw. This keeps nested and top-level WebSocket consumers, the HTTP bridge, and compact/error sanitizers on one source of truth. + +Alternative: special-case the raw frame inside the WebSocket relay. Rejected because it would duplicate semantics, miss other existing classifier consumers, and make nested versus top-level envelopes diverge. + +### Reuse existing recovery policy unchanged + +Once classified, the event follows the existing `previous_response_not_found` paths. A self-contained full resend can be replayed without the anchor; a delta-only request cannot. Codex-native clients receive the canonical sanitized code for their controlled full-history retry, while public clients retain generic masking. + +Alternative: drop `previous_response_id` and retry every request. Rejected because the observed first and third failures carried only tool-call output deltas; replaying those without history would silently detach the tool result from its conversation. + +### Treat connection churn as evidence, not patch scope + +The rejected ids were successful on the same account and session 9–17 seconds earlier. That makes long-term retention and cross-account explanations less likely, but does not rule out short retention or reconnect invalidation. The deployment also recorded connection churn, which may make the stale-anchor condition more frequent, but the classifier repair remains correct whether the anchor was invalidated by a reconnect, upstream retention, or another server-side lifecycle boundary. + +Alternative: combine this patch with #1711 transport changes. Rejected because #1711's cleanup warning is observational, its focused fixes are already merged, and the overnight data does not prove one new transport mutation that would eliminate all three failures. + +## Risks / Trade-offs + +- [Upstream reuses the exact message for malformed client ids] → The same recovery remains safe: transparent replay is still gated on a self-contained body, while delta-only clients receive a sanitized request to resend full history. +- [Over-classifying unrelated invalid requests] → Require `invalid_request_error`, reject any different named parameter, and match only the observed message after case/whitespace normalization and optional terminal punctuation. +- [Shared classifier changes non-WebSocket consumers] → Those consumers already treat unusable `previous_response_id` as continuity loss; focused tests cover the classifier plus Codex-native and public route behavior. + +## Migration Plan + +No data or configuration migration is required. Deploy as an application patch; rollback restores the prior raw-400 behavior without changing persisted state. diff --git a/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/proposal.md b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/proposal.md new file mode 100644 index 0000000000..7b45bb1739 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/proposal.md @@ -0,0 +1,27 @@ +## Why + +The ChatGPT-backed Codex WebSocket now emits stale-anchor failures as `invalid_request_error` with no `code` or `param` and the message ``Invalid `previous_response_id`.``. codex-lb does not recognize that observed shape, so it relays the raw 400 instead of entering its existing safe replay or sanitized client-recovery path. + +Production evidence on current upstream `main` recorded three affected Codex sessions in one overnight window. In every case the rejected anchor was a successful response from the same session and account only 9–17 seconds earlier, making this an active compatibility gap rather than an old retained response or account-routing mismatch. + +## What Changes + +- Classify the exact observed parameterless `invalid_request_error` message as a previous-response continuity miss. +- Reuse the existing WebSocket recovery contract: transparently replay self-contained full resends without the anchor, surface sanitized canonical `previous_response_not_found` to Codex-native delta clients, and retain generic masking for public `/v1` clients. +- Preserve classification boundaries for unrelated invalid-request errors and errors naming a different parameter. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: Recognize the parameterless invalid-previous-response error shape emitted by the upstream Codex WebSocket and route it through existing stale-anchor recovery and masking. + +## Impact + +- Shared OpenAI error classification in `app/core/errors.py`. +- Direct Responses WebSocket behavior on `/backend-api/codex/responses` and `/v1/responses` through their existing recovery policies. +- Route-level and classifier regression coverage; no API, schema, migration, dependency, configuration, or dashboard changes. diff --git a/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..ca1560ac13 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/specs/responses-api-compat/spec.md @@ -0,0 +1,31 @@ +## ADDED Requirements + +### Requirement: Parameterless invalid previous-response errors use continuity recovery + +When an upstream Responses WebSocket rejects an anchored request with `type = "invalid_request_error"`, no `code` or `param`, and the normalized message ``Invalid `previous_response_id``` with or without one trailing period, the service MUST classify the frame as a previous-response continuity miss. It MUST apply the same replay, masking, ownership, and account-health rules as the canonical `previous_response_not_found` error and MUST NOT relay the raw invalid-request frame downstream. A different named parameter or any other trailing punctuation MUST NOT match this error shape. + +#### Scenario: Codex-native delta continuation receives the canonical recovery signal + +- **GIVEN** a Codex-native `/backend-api/codex/responses` request carries `previous_response_id` and delta-only tool output that cannot be replayed safely without its anchor +- **WHEN** upstream returns the parameterless ``Invalid `previous_response_id`.`` error before `response.created` +- **THEN** the downstream client receives a sanitized error with `code = "previous_response_not_found"` +- **AND** the raw upstream envelope and previous response id are not exposed + +#### Scenario: Self-contained full resend is replayed without the rejected anchor + +- **GIVEN** an anchored direct WebSocket request retains a self-contained full-resend body that is safe to replay without `previous_response_id` +- **WHEN** upstream returns the parameterless ``Invalid `previous_response_id`.`` error before `response.created` +- **THEN** the service reconnects and replays the retained body without `previous_response_id` +- **AND** the raw upstream error is not sent downstream + +#### Scenario: Public WebSocket retains generic continuity masking + +- **GIVEN** a public `/v1/responses` WebSocket request carries `previous_response_id` but cannot be replayed safely without its anchor +- **WHEN** upstream returns the parameterless ``Invalid `previous_response_id`.`` error +- **THEN** the downstream client receives the existing sanitized `stream_incomplete` continuity failure +- **AND** neither `previous_response_not_found` nor the raw upstream envelope is exposed + +#### Scenario: Unrelated invalid requests retain their original classification + +- **WHEN** upstream returns `invalid_request_error` with a different message or names a parameter other than `previous_response_id` +- **THEN** the service MUST NOT classify that error as a previous-response continuity miss diff --git a/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/tasks.md b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/tasks.md new file mode 100644 index 0000000000..8f05dc0f04 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/tasks.md @@ -0,0 +1,20 @@ +## 1. Regression Coverage + +- [x] 1.1 Add a Codex-native route regression using the exact production frame (`invalid_request_error`, no `code`/`param`, ``Invalid `previous_response_id`.``) and verify it fails by exposing the raw 400 before implementation. + +## 2. Classification Fix + +- [x] 2.1 Extend the shared previous-response classifier with the exact parameterless upstream message, normalize the code-less nested frame consistently at the rewrite call, and reject a different named parameter or unrelated invalid-request message. +- [x] 2.2 Verify the route regression passes and the existing canonical stale-anchor recovery tests remain green. + +## 3. Compatibility Boundaries + +- [x] 3.1 Cover the exact observed frame in the self-contained full-resend replay path and confirm the replay drops `previous_response_id`. +- [x] 3.2 Cover the exact observed frame on public `/v1/responses` and confirm it retains generic `stream_incomplete` masking. +- [x] 3.3 Add focused classifier cases for the observed shape and false-positive boundaries. +- [x] 3.4 Add the stable failure-mode and recovery example to the existing `responses-api-compat` context documentation. + +## 4. Verification + +- [x] 4.1 Run focused OpenAI error and direct WebSocket route tests, then the relevant proxy architecture and formatting/lint/type checks. +- [x] 4.2 Run strict OpenSpec validation, the repository's proportionate final gate, and review the final diff for unrelated changes. diff --git a/openspec/changes/archive/2026-08-19-fix-gpt56-context-window/proposal.md b/openspec/changes/archive/2026-08-19-fix-gpt56-context-window/proposal.md new file mode 100644 index 0000000000..bd006a45c3 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-fix-gpt56-context-window/proposal.md @@ -0,0 +1,23 @@ +## Why + +The GPT-5.6 bootstrap catalog was originally pinned to Codex +`rust-v0.144.1`, whose entries reported a 372,000-token context window. The +upstream bundled catalog corrected Sol, Terra, and Luna to 272,000 tokens in +`rust-v0.145.0`. codex-lb must advertise the corrected upstream input budget in +its bootstrap catalog and normative compatibility contract so startup/offline +clients do not overfill requests before the live registry refreshes. + +## What Changes + +- Re-pin GPT-5.6 bootstrap catalog provenance from Codex `rust-v0.144.1` to + `rust-v0.145.0`. +- Require `context_window` and `max_context_window` of 272,000 for Sol, Terra, + and Luna. +- Update regression-test evidence comments to cite the reproducible upstream + bundled catalog release instead of untracked live-fetch artifacts. + +## Impact + +- No schema, route, or database migration change. +- Before a live registry refresh, bootstrap `/v1/models` and `/backend-api/codex/models` change the GPT-5.6 advertised context budget from 372,000 to 272,000 tokens. +- Affects `model-catalog-compat` documentation, client setup examples, and GPT-5.6 bootstrap regression coverage. diff --git a/openspec/changes/archive/2026-08-19-fix-gpt56-context-window/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-19-fix-gpt56-context-window/specs/model-catalog-compat/spec.md new file mode 100644 index 0000000000..ea158329db --- /dev/null +++ b/openspec/changes/archive/2026-08-19-fix-gpt56-context-window/specs/model-catalog-compat/spec.md @@ -0,0 +1,56 @@ +## MODIFIED Requirements + +### Requirement: GPT-5.6 bootstrap metadata matches the upstream bundled catalog + +The GPT-5.6 bootstrap catalog entries (`gpt-5.6-sol`, `gpt-5.6-terra`, +`gpt-5.6-luna`) MUST mirror the upstream bundled catalog +(`codex-rs/models-manager/models.json` at Codex release `rust-v0.145.0`) +field-for-field for every metadata field codex-lb serves. In particular each +entry MUST carry: `context_window` and `max_context_window` of `272000`; +`minimal_client_version` `"0.144.0"`; `tool_mode` `"code_mode_only"`; +`use_responses_lite` `true`; `apply_patch_tool_type` `"freeform"`; +`web_search_tool_type` `"text_and_image"`; `supports_image_detail_original` +`true`; `truncation_policy` `{ "mode": "tokens", "limit": 10000 }`; +`comp_hash` `"3000"`; `reasoning_summary_format` `"experimental"`; +`default_reasoning_summary` `"none"`; `include_skills_usage_instructions` +`false`; `experimental_supported_tools` `[]` (a field the Codex client's +deserializer requires); `supports_search_tool` `true`; `additional_speed_tiers` +`["fast"]`; the `priority`/`Fast` service tier entry; `shell_type` +`"shell_command"`; `prefer_websockets` `true`; and the 21-plan +`available_in_plans` list upstream advertises (including `edu_plus`, +`edu_pro`, `enterprise_cbp_automation`, and `sci`). `multi_agent_version` MUST +be `"v2"` for Sol and Terra and `"v1"` for Luna. Sol MUST carry the upstream +`availability_nux` message while Terra and Luna carry `null`. Default reasoning +levels MUST be `low` for Sol and `medium` for Terra and Luna, and +reasoning-level descriptions MUST be the verbatim upstream strings. + +The ~16.5 KB upstream `base_instructions` prompt and the personality-templated +`model_messages` object are deliberately NOT bundled in the bootstrap catalog; +the first successful live registry refresh supplies them. This is the only +sanctioned divergence from the upstream GPT-5.6 entries. + +#### Scenario: GPT-5.6 bootstrap entries retain the corrected upstream context budget + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` report + `context_window=272000` and `max_context_window=272000` + +#### Scenario: GPT-5.6 entries expose upstream tool and multi-agent metadata + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` carry `tool_mode: "code_mode_only"`, `use_responses_lite: true`, `experimental_supported_tools: []`, and `minimal_client_version: "0.144.0"` +- **AND** `multi_agent_version` is `"v2"` for Sol and Terra and `"v1"` for Luna + +#### Scenario: GPT-5.6 entries expose upstream reasoning-summary and plan metadata + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** each GPT-5.6 entry carries `default_reasoning_summary: "none"`, `reasoning_summary_format: "experimental"`, and `comp_hash: "3000"` +- **AND** each GPT-5.6 entry's `available_in_plans` includes `edu_plus`, `edu_pro`, `enterprise_cbp_automation`, and `sci` +- **AND** only `gpt-5.6-sol` carries a non-null `availability_nux` message diff --git a/openspec/changes/archive/2026-08-19-fix-gpt56-context-window/tasks.md b/openspec/changes/archive/2026-08-19-fix-gpt56-context-window/tasks.md new file mode 100644 index 0000000000..59c94076f0 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-fix-gpt56-context-window/tasks.md @@ -0,0 +1,23 @@ +## 1. Regression coverage + +- [x] 1.1 Update GPT-5.6 bootstrap catalog test evidence to cite + `codex-rs/models-manager/models.json` at Codex `rust-v0.145.0`. +- [x] 1.2 Run the focused bootstrap metadata tests and verify every GPT-5.6 + entry reports `context_window` and `max_context_window` of 272,000. +- [x] 1.3 Assert the top-level `context_window` and raw + `max_context_window` are 272,000 for every GPT-5.6 bootstrap entry. + +## 2. Specification + +- [x] 2.1 Add a `model-catalog-compat` delta that re-pins the GPT-5.6 bootstrap + catalog source to Codex `rust-v0.145.0` and requires both context-window + fields to be 272,000. +- [x] 2.2 State that operator context-window overrides take precedence over + the default bootstrap budget. + +- [x] 2.3 Correct the documented OpenCode and OpenClaw GPT-5.6 budgets to + 272,000 tokens. +## 3. Validation + +- [x] 3.1 Validate the OpenSpec change and the complete specification set. +- [x] 3.2 Run the focused bootstrap metadata tests after review follow-up. diff --git a/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/.openspec.yaml b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/.openspec.yaml new file mode 100644 index 0000000000..41c30bab88 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-19 diff --git a/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/design.md b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/design.md new file mode 100644 index 0000000000..a7c1d395f9 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/design.md @@ -0,0 +1,89 @@ +## Context + +Image generation and edit reserve limited API-key quota before invoking the +internal Responses pipeline. They intentionally pass no reservation into that +pipeline because image usage comes from `tool_usage.image_gen`, not +`response.usage`. The image adapter therefore owns final settlement. + +The current adapter performs finalization inline after it has already produced +the public image result. A persistence failure rolls the reservation back to +`reserved`; the adapter logs and returns, so no request-scoped owner remains. +The standard Responses settlement path already provides detached task tracking, +cancellation handoff, retrying release, bounded repository concurrency, and +graceful persistence drain. + +## Goals / Non-Goals + +**Goals** + +- Transfer image reservation ownership exactly once to the existing tracked + settlement machinery. +- Finalize captured image tokens when persistence succeeds. +- Preserve the completed public response while failed or cancelled settlement + transfers ownership to the existing retrying release fallback. +- Keep generation/edit and streaming/non-streaming behavior aligned. + +**Non-Goals** + +- Define image-only retries of authoritative token finalization. +- Change repository states, retry timings, concurrency limits, stale-reset + policy, database schema, settings, or external response shapes. +- Give the internal Responses stream a second settlement owner. +- Broaden pre-terminal image cancellation cleanup. + +## Decisions + +### Reuse tracked stream settlement ownership + +Add one image-facing adapter on the API-key usage mixin. The adapter constructs +the existing settlement value from the public image model, captured image +tokens, API-key data, reservation, service tier `None`, and request id, then +delegates to the existing tracked settlement entrypoint. + +This keeps task registration, cancellation callbacks, retrying release, and +persistence drain in one implementation rather than copying lifecycle logic +into the route module. + +### Preserve image-token authority + +When at least one captured image token field is usable, the adapter records a +successful settlement and normalizes missing token fields to zero. When no +captured image usage is usable, it selects the existing non-success settlement +path so the reservation releases instead of recording fabricated usage. + +The internal Responses call continues receiving `api_key_reservation=None`. + +### Transfer ownership before returning the completed result + +All four image completion paths call the same adapter exactly once. The adapter +returns after the settlement task is registered; the public response does not +wait for persistence. If tracked finalization fails or is cancelled, its done +callback transfers ownership synchronously to the retrying release task. + +Exactly-once refers to the terminal database mutation. Retried release attempts +remain safe because repository transitions claim only a still-reserved row. + +## Risks / Trade-offs + +- A failed finalization falls back to release, so successful image usage can be + omitted under persistence failure. This matches existing standard stream + policy and is preferable to keeping quota ownerless. Retrying authoritative + finalization is a broader accounting-policy change and remains separate. +- Reusing a private settlement value couples the adapter to existing settlement + internals. Keeping construction inside the mixin limits that coupling and + avoids route-level task lifecycle duplication. +- Permanent release failure leaves quota conservatively reserved, but the task + remains visible to persistence drain and the stale reaper remains a final + process-restart fallback. + +## Migration Plan + +No migration or rollout setting is required. Existing terminal reservations are +unchanged; new image completions use tracked settlement after deployment. + +Rollback restores inline image finalization behavior without data conversion. + +## Open Questions + +None for this change. Stronger retries of authoritative image finalization are +explicit follow-up scope. diff --git a/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/proposal.md b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/proposal.md new file mode 100644 index 0000000000..65be5b7be2 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/proposal.md @@ -0,0 +1,38 @@ +## Why + +Image generation and edit routes reserve limited API-key quota but deliberately +exclude that reservation from the internal Responses stream settlement path. +Their image-specific finalizer currently logs and abandons the reservation when +persistence fails, leaving quota charged until stale cleanup and leaving +graceful persistence drain unaware of the unresolved work. + +## What Changes + +- Transfer image reservation settlement to the existing tracked, + cancellation-safe stream settlement machinery while preserving captured + `tool_usage.image_gen` tokens as the authoritative usage source. +- Preserve successful public Images JSON and SSE responses when settlement + fails or is cancelled. +- Transfer failed or cancelled finalization to the existing tracked, + retrying release fallback so persistence drain remains aware of unresolved + ownership. +- Keep the internal Responses stream reservation-free to prevent duplicate + settlement across image and standard response paths. + +## Capabilities + +### Modified Capabilities + +- `images-api-compat`: require successful image generation and edit paths to + retain tracked reservation ownership through finalization or fallback release. + +## Impact + +- Affects the image generation/edit settlement handoff in + `app/modules/proxy/api.py` and the reusable API-key settlement seam in + `app/modules/proxy/_service/api_key_usage.py`. +- Adds event-driven integration coverage for finalization failure, + cancellation, release retry, persistence drain, and all four image response + modes. +- Does not change database schema, API-key repository transitions, retry + constants, scheduler policy, external response schemas, or settings. diff --git a/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/specs/images-api-compat/spec.md b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/specs/images-api-compat/spec.md new file mode 100644 index 0000000000..d1eedf4554 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/specs/images-api-compat/spec.md @@ -0,0 +1,35 @@ +## MODIFIED Requirements + +### Requirement: Image routes participate in usage accounting and policy + +The system SHALL apply API-key allowed-model policy and model-scoped usage +limits to `/v1/images/*` using the publicly-requested `gpt-image-*` value as the +effective model. The system SHALL record the publicly-requested `gpt-image-*` +value (not the internal host model) in the request log's `model` column once the +upstream response id becomes known. A successful image generation or edit that +owns a limited API-key reservation SHALL transfer that reservation exactly once +to persistence-drained settlement using captured `tool_usage.image_gen` tokens, +while the internal Responses stream SHALL NOT receive a second settlement +owner. Failed or cancelled finalization SHALL preserve the completed public +image response and transfer ownership to the tracked retrying release fallback. + +#### Scenario: API key allowed-model policy blocks gpt-image-2 + +- **WHEN** an API key's `allowed_models` list does not include `gpt-image-2` +- **THEN** requests to `/v1/images/generations` or `/v1/images/edits` with `model=gpt-image-2` return 403 `model_not_allowed` + +#### Scenario: Request log surfaces the publicly requested image model + +- **WHEN** an `/v1/images/*` request completes successfully against an internal host Responses model (for example `gpt-5.5`) +- **THEN** the resulting `request_logs` row has `model` equal to the publicly requested value (for example `gpt-image-2`) so dashboards and usage views surface the user-visible model rather than the internal host model + +#### Scenario: Failed image-token settlement retains tracked release ownership + +- **GIVEN** a limited API key owns a reservation for a successful image generation or edit request +- **AND** the internal Responses stream receives no API-key reservation +- **AND** the image adapter captures authoritative `tool_usage.image_gen` tokens +- **WHEN** tracked finalization fails or is cancelled while the reservation remains `reserved` +- **THEN** the completed public Images JSON response or SSE completion remains available +- **AND** settlement ownership transfers to a persistence-drained fallback release task +- **AND** transient release failures keep that task tracked and retrying until release succeeds or graceful persistence drain reports timeout +- **AND** a successful fallback restores pre-reserved quota exactly once without recording `response.usage` or starting a second image settlement diff --git a/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/tasks.md b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/tasks.md new file mode 100644 index 0000000000..351b6d5c85 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/tasks.md @@ -0,0 +1,16 @@ +## 1. Regression Coverage + +- [x] 1.1 Replace the unkeyed finalization-failure case with a limited-key integration that proves the completed image response transfers an unresolved reservation to tracked release ownership +- [x] 1.2 Add event-driven coverage for settlement cancellation and retrying release while persistence drain remains pending +- [x] 1.3 Cover generation and edit, streaming and non-streaming, to prove exactly one image settlement handoff and no internal Responses reservation owner + +## 2. Tracked Image Settlement + +- [x] 2.1 Add an image-facing API-key usage adapter that delegates captured image tokens to the existing tracked stream settlement lifecycle +- [x] 2.2 Route all four image completion paths through the adapter while preserving public response availability and public model attribution + +## 3. Verification + +- [x] 3.1 Run focused image and settlement tests, Ruff, type checking, proxy architecture checks, and strict affected OpenSpec validation +- [x] 3.2 Exercise the isolated HTTP image surface with a limited key, gated release retry, persistence drain, and real SQLite state assertions +- [x] 3.3 Verify implementation against this change, synchronize the delta, and archive the verified OpenSpec change diff --git a/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/.openspec.yaml b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/.openspec.yaml new file mode 100644 index 0000000000..41c30bab88 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-19 diff --git a/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/design.md b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/design.md new file mode 100644 index 0000000000..f4ebc97bf1 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/design.md @@ -0,0 +1,58 @@ +## Context + +Compact Responses receives an upstream HTTP response whose body can terminate +with an SSE event. Nested OpenAI-style error objects are parsed through the +shared error parser and preserve their type. A top-level event shaped as +`{"type":"error","error_type":...,"code":...,"message":...}` instead uses a +fallback converter. + +The status-code helper already reads top-level `error_type`, so it can infer +HTTP 400/401/403/429 correctly. The fallback envelope independently hard-codes +`server_error`, producing an internally inconsistent public response. + +## Goals / Non-Goals + +**Goals** + +- Preserve a supplied non-blank top-level `error_type`. +- Retain `server_error` when the field is absent, non-string, or blank. +- Leave nested envelopes and all other mapped fields/statuses unchanged. + +**Non-Goals** + +- Change compact request routing, retries, account selection, or health. +- Infer new status codes or normalize arbitrary upstream error types. +- Change non-compact Responses or nested error parsing. + +## Decisions + +### Fix only the top-level fallback + +The fallback converter reads `payload["error_type"]`. A string containing at +least one non-whitespace character becomes the OpenAI detail `type`; otherwise +the existing `server_error` value remains. + +This keeps the fix at the data-loss seam and avoids changing the shared parser +or status inference that already behave correctly. + +### Preserve supplied type text + +Whitespace is used only to decide whether a value is blank. A non-blank string +is forwarded verbatim, matching the existing field-preservation behavior for +top-level `code`, `message`, and `param`. + +## Risks / Trade-offs + +- Upstream can supply an unfamiliar type. Preserving it is preferable to + fabricating `server_error` and matches OpenAI-compatible passthrough behavior. +- The fallback remains intentionally conservative for absent, non-string, or + whitespace-only values. + +## Migration Plan + +No migration, setting, or rollout step is required. Rollback restores the +previous top-level type substitution. + +## Open Questions + +None. diff --git a/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/proposal.md b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/proposal.md new file mode 100644 index 0000000000..42b1fefad5 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/proposal.md @@ -0,0 +1,31 @@ +## Why + +The compact Responses transport derives the correct HTTP status from a +top-level terminal SSE frame's `error_type`, but its fallback OpenAI envelope +replaces that supplied type with `server_error`. Clients therefore receive a +contradictory response such as HTTP 400 with `error.type=server_error` even +though the upstream classified the failure as `invalid_request_error`. + +## What Changes + +- Preserve a non-empty top-level compact SSE `error_type` in the emitted OpenAI + error envelope. +- Keep `server_error` as the compatibility fallback when the top-level field is + absent or blank. +- Preserve existing nested error-envelope behavior and status, code, message, + and parameter mapping. + +## Capabilities + +### Modified Capabilities + +- `responses-api-compat`: define compact terminal error-envelope behavior for + top-level `type=error` SSE frames. + +## Impact + +- Affects the compact SSE terminal-error converter in + `app/core/clients/proxy.py`. +- Adds focused routed transport tests and fallback/nested controls. +- Does not change request routing, retry behavior, account health, schemas, + settings, dependencies, or non-compact Responses behavior. diff --git a/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..b2c9cd48d4 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/specs/responses-api-compat/spec.md @@ -0,0 +1,22 @@ +## ADDED Requirements + +### Requirement: Compact terminal SSE errors preserve top-level error type + +When the compact Responses upstream terminates with a top-level SSE `type=error` frame, the proxy MUST preserve a supplied non-blank `error_type` in the emitted OpenAI error envelope. If `error_type` is absent, non-string, or blank, the proxy MUST use `server_error`. The proxy MUST preserve existing status, code, message, and parameter mapping, and MUST NOT alter nested OpenAI-style error-envelope behavior. + +#### Scenario: Top-level invalid request type is preserved + +- **WHEN** compact upstream terminates with a top-level `type=error` frame whose `error_type` is `invalid_request_error` +- **THEN** the proxy returns HTTP 400 with `error.type=invalid_request_error` +- **AND** preserves the frame's code, message, and parameter + +#### Scenario: Missing or blank top-level type uses compatibility fallback + +- **WHEN** compact upstream terminates with a top-level `type=error` frame whose `error_type` is absent or blank +- **THEN** the emitted OpenAI error envelope uses `error.type=server_error` +- **AND** existing status, code, message, and parameter mapping remains unchanged + +#### Scenario: Nested compact error envelope remains unchanged + +- **WHEN** compact upstream terminates with a nested OpenAI-style error envelope +- **THEN** the proxy preserves the nested type and all other mapped fields using the existing parser diff --git a/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/tasks.md b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/tasks.md new file mode 100644 index 0000000000..265df4b8ca --- /dev/null +++ b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/tasks.md @@ -0,0 +1,15 @@ +## 1. Regression Coverage + +- [x] 1.1 Add a routed compact top-level terminal SSE regression that preserves `invalid_request_error` +- [x] 1.2 Add missing and blank `error_type` fallback controls and retain the nested-envelope control + +## 2. Compact Error Conversion + +- [x] 2.1 Preserve a supplied non-blank top-level `error_type` in the OpenAI error detail +- [x] 2.2 Keep status, code, message, parameter, nested-envelope, and `server_error` fallback behavior unchanged + +## 3. Verification + +- [x] 3.1 Run focused compact tests, Ruff, type checking, proxy architecture checks, and strict affected OpenSpec validation +- [x] 3.2 Exercise the live compact HTTP route with top-level invalid-request and missing-type upstream terminal frames +- [x] 3.3 Verify implementation against this change, synchronize the delta, and archive the verified OpenSpec change diff --git a/openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/proposal.md b/openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/proposal.md new file mode 100644 index 0000000000..ba518e90ea --- /dev/null +++ b/openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/proposal.md @@ -0,0 +1,54 @@ +## Why + +Upstream raised the GPT-5.6 maximum context window from 272,000 to 872,000 +tokens while leaving the default input budget at 272,000 +(`codex-rs/models-manager/models.json`, openai/codex commit +`2eee483e49f88b868f67364134a658b3298e6c14`, "Raise the GPT-5.6 maximum context +window", openai/codex#39102). codex-lb's bootstrap catalog synthesizes +`max_context_window` as a copy of `context_window` +(`app/core/openai/model_registry.py`), so it advertises a 272,000 ceiling for +Sol, Terra, and Luna. A Codex client pointed at codex-lb before the first live +registry refresh therefore has its `model_context_window` opt-in clamped to +272,000 and cannot reach the window upstream actually serves. + +## What Changes + +- Decouple `max_context_window` from `context_window` for the GPT-5.6 + bootstrap family: `max_context_window` becomes 872,000; `context_window` + stays 272,000. +- Keep the GPT-5.6 base provenance pinned at Codex `rust-v0.145.0`, with a + single tracked exception for `max_context_window`, pinned to the upstream + commit that raised it (no `rust-v*` release tag carries it yet as of + `rust-v0.148.0-alpha.21`). +- Document the Codex CLI opt-in, including the clamp semantics that make + `model_context_window = 1000000` resolve to 872,000 and + `model_auto_compact_token_limit = 900000` a no-op (Codex clamps the + auto-compact limit to 90% of the resolved window). + +## Non-goals + +- `context_window` stays 272,000. It is the tuned default input budget and the + upstream long-context pricing threshold. +- The other post-`rust-v0.145.0` upstream deltas to these entries + (`include_apps_usage_instructions`, `include_plugin_usage_instructions`, the + `base_instructions` relocation to `prompt.md`, `supports_parallel_tool_calls` + now serde-defaulted) are out of scope and need their own compatibility + review. +- No clamp or override logic changes; `/v1` input budget fields keep reporting + the default input budget (see PR #1808 for override plumbing work). + +## Impact + +- No schema, route, or database migration change. +- Before a live registry refresh, `GET /backend-api/codex/models` changes the + GPT-5.6 advertised `max_context_window` from 272,000 to 872,000 tokens; + `context_window` is unchanged. +- `GET /v1/models` is unchanged: it reports the default input budget and does + not promote `raw["max_context_window"]`. +- Operator `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entries and persisted + registry snapshots continue to take precedence over bootstrap values. +- The delta restates the `model-catalog-compat` GPT-5.6 requirement in full, + so it is order-insensitive with respect to the still-unarchived + `fix-gpt56-context-window` delta (issue #1714): applied before or after it, + the merged requirement text is identical. +- Revert cost is one line if upstream reverts before tagging a release. diff --git a/openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/specs/model-catalog-compat/spec.md new file mode 100644 index 0000000000..0d460ab1eb --- /dev/null +++ b/openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/specs/model-catalog-compat/spec.md @@ -0,0 +1,86 @@ +## MODIFIED Requirements + +### Requirement: GPT-5.6 bootstrap metadata matches the upstream bundled catalog + +The GPT-5.6 bootstrap catalog entries (`gpt-5.6-sol`, `gpt-5.6-terra`, +`gpt-5.6-luna`) MUST mirror the upstream bundled catalog +(`codex-rs/models-manager/models.json` at Codex release `rust-v0.145.0`) +field-for-field for every metadata field codex-lb serves, with one tracked +exception: `max_context_window`, which upstream raised from `272000` to +`872000` in openai/codex commit +`2eee483e49f88b868f67364134a658b3298e6c14` (openai/codex#39102) and which no +`rust-v*` release tag carries as of `rust-v0.148.0-alpha.21`. In particular +each entry MUST carry: `context_window` of `272000` and `max_context_window` +of `872000`; `minimal_client_version` `"0.144.0"`; `tool_mode` +`"code_mode_only"`; `use_responses_lite` `true`; `apply_patch_tool_type` +`"freeform"`; `web_search_tool_type` `"text_and_image"`; +`supports_image_detail_original` `true`; `truncation_policy` `{ "mode": +"tokens", "limit": 10000 }`; `comp_hash` `"3000"`; `reasoning_summary_format` +`"experimental"`; `default_reasoning_summary` `"none"`; +`include_skills_usage_instructions` `false`; `experimental_supported_tools` +`[]` (a field the Codex client's deserializer requires); `supports_search_tool` +`true`; `additional_speed_tiers` `["fast"]`; the `priority`/`Fast` service tier +entry; `shell_type` `"shell_command"`; `prefer_websockets` `true`; and the +21-plan `available_in_plans` list upstream advertises (including `edu_plus`, +`edu_pro`, `enterprise_cbp_automation`, and `sci`). `multi_agent_version` MUST +be `"v2"` for Sol and Terra and `"v1"` for Luna. Sol MUST carry the upstream +`availability_nux` message while Terra and Luna carry `null`. Default reasoning +levels MUST be `low` for Sol and `medium` for Terra and Luna, and +reasoning-level descriptions MUST be the verbatim upstream strings. + +`context_window` is the default input budget and `max_context_window` is the +ceiling a client may opt into; the two MUST NOT be collapsed into one value +for these entries. + +The ~16.5 KB upstream `base_instructions` prompt and the personality-templated +`model_messages` object are deliberately NOT bundled in the bootstrap catalog; +the first successful live registry refresh supplies them. This is the only +sanctioned divergence from the upstream GPT-5.6 entries beyond the +`max_context_window` exception above. + +#### Scenario: GPT-5.6 bootstrap entries retain the corrected upstream context budget + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` report + `context_window=272000` + +#### Scenario: GPT-5.6 bootstrap entries advertise the raised upstream ceiling + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` report + `context_window=272000` +- **AND** each reports `max_context_window=872000` + +#### Scenario: OpenAI-compatible metadata keeps the default input budget + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /v1/models` +- **THEN** each GPT-5.6 entry reports `context_window=272000` and + `input_context_window=272000` +- **AND** the raised Codex-native ceiling is not promoted into the + OpenAI-compatible input budget fields + +#### Scenario: GPT-5.6 entries expose upstream tool and multi-agent metadata + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` carry `tool_mode: "code_mode_only"`, `use_responses_lite: true`, `experimental_supported_tools: []`, and `minimal_client_version: "0.144.0"` +- **AND** `multi_agent_version` is `"v2"` for Sol and Terra and `"v1"` for Luna + +#### Scenario: GPT-5.6 entries expose upstream reasoning-summary and plan metadata + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** each GPT-5.6 entry carries `default_reasoning_summary: "none"`, `reasoning_summary_format: "experimental"`, and `comp_hash: "3000"` +- **AND** each GPT-5.6 entry's `available_in_plans` includes `edu_plus`, `edu_pro`, `enterprise_cbp_automation`, and `sci` +- **AND** only `gpt-5.6-sol` carries a non-null `availability_nux` message diff --git a/openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/tasks.md b/openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/tasks.md new file mode 100644 index 0000000000..9c9ff4efb5 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/tasks.md @@ -0,0 +1,49 @@ +## 1. Bootstrap catalog + +- [x] 1.1 Add `max_context_window: 872_000` to `_gpt56_raw()` so it overrides + the `_bootstrap_model` synthesis for all three GPT-5.6 slugs at once. +- [x] 1.2 Leave `context_window` at 272,000 for Sol, Terra, and Luna. +- [x] 1.3 Amend the `_gpt56_raw()` docstring to record `max_context_window` as + the one tracked divergence from the `rust-v0.145.0` base pin, citing the + upstream commit. + +## 2. Regression coverage + +- [x] 2.1 Assert `max_context_window == 872_000` and `context_window == + 272_000` for every GPT-5.6 entry in the shared unit-test loop. +- [x] 2.2 Assert the same pair through `GET /backend-api/codex/models` in the + shared integration-test loop. +- [x] 2.3 Assert `max_context_window > context_window` in both loops so a + future re-unification of the two fields fails loudly. +- [x] 2.4 Assert `GET /v1/models` still reports 272,000 for the GPT-5.6 input + budget fields on the bootstrap path. +- [x] 2.5 Re-pin both evidence comments to `rust-v0.145.0` plus the upstream + commit for `max_context_window`. + +## 3. Documentation + +- [x] 3.1 Distinguish the 272k default budget from the 872k maximum in the + client-setup model lineup summary. +- [x] 3.2 Document the Codex CLI opt-in with correct clamp semantics: values + above `max_context_window` are clamped, and the auto-compact limit + resolves to 90% of the window, so larger values are no-ops. +- [x] 3.3 Leave the OpenCode and OpenClaw examples at 272,000, matching what + `/v1/models` advertises. + +## 4. Specification + +- [x] 4.1 Add a `model-catalog-compat` delta requiring `context_window` + 272,000 and `max_context_window` 872,000 for the GPT-5.6 bootstrap + entries. +- [x] 4.2 Restate the requirement and surviving scenarios in full so the delta + is order-insensitive against the unarchived `fix-gpt56-context-window` + delta. +- [x] 4.3 Carry GIVEN clauses on context-budget scenarios excluding refreshed + snapshots, persisted snapshots, and operator context-window overrides. + +## 5. Validation + +- [x] 5.1 `openspec validate raise-gpt56-max-context-window --strict` +- [x] 5.2 `openspec validate --specs` +- [x] 5.3 Focused unit + integration model-catalog tests, ruff, and + `mkdocs build --strict`. diff --git a/openspec/changes/attribute-websocket-scope-cleanup-phase/proposal.md b/openspec/changes/attribute-websocket-scope-cleanup-phase/proposal.md new file mode 100644 index 0000000000..4ade0403fa --- /dev/null +++ b/openspec/changes/attribute-websocket-scope-cleanup-phase/proposal.md @@ -0,0 +1,33 @@ +## Why + +When WebSocket scope cleanup exceeds its cleanup budget, the warning reports the +timeout and total background cleanup task count but not the operation that is +still blocked. Operators cannot distinguish an upstream-close stall from +reader observation, request finalization, or lease release without reproducing +the incident under instrumentation. + +## What Changes + +- Track the current WebSocket scope cleanup phase locally while the existing + finalization sequence runs. +- Add that fixed, low-cardinality phase to the existing timeout warning. +- Keep cleanup ordering, timeout budgets, retries, and ownership unchanged. +- Do not log request ids, account ids, payloads, credentials, or exception + content in the phase field. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `proxy-runtime-observability`: WebSocket scope cleanup timeout warnings MUST + identify the blocked cleanup phase with a fixed low-cardinality value. + +## Impact + +`app/modules/proxy/_service/websocket/mixin.py` and its route-level WebSocket +cleanup regression coverage. No API, schema, setting, timeout, or dashboard +change. diff --git a/openspec/changes/attribute-websocket-scope-cleanup-phase/specs/proxy-runtime-observability/spec.md b/openspec/changes/attribute-websocket-scope-cleanup-phase/specs/proxy-runtime-observability/spec.md new file mode 100644 index 0000000000..20feb5e9d7 --- /dev/null +++ b/openspec/changes/attribute-websocket-scope-cleanup-phase/specs/proxy-runtime-observability/spec.md @@ -0,0 +1,35 @@ +# proxy-runtime-observability Delta + +## ADDED Requirements + +### Requirement: WebSocket scope cleanup timeout identifies its blocked phase + +When WebSocket scope finalization exceeds its cleanup budget, the proxy MUST +include the current cleanup phase in the existing warning. The phase MUST be a +fixed low-cardinality value that identifies the cleanup operation and MUST NOT +contain request ids, account ids, request payloads, credentials, or exception +content. This diagnostic MUST NOT change cleanup ordering, timeout budgets, +retry behavior, or task ownership. + +The phase MUST be one of `not_started`, `upstream_close`, `upstream_reader`, +`retired_create_lease`, `unsent_request`, `replay_request`, `pending_requests`, +`connection_lease`, or `complete`. `not_started` is the fallback before the +first cleanup operation begins. `complete` records finished cleanup and MUST NOT +appear in a timeout warning. Missing or unrecognized phases MUST fall back to +`not_started`; implementations MUST NOT derive a phase from request or exception +data. + +#### Scenario: Pending request finalization exceeds the cleanup budget + +- **GIVEN** a cancelled WebSocket scope whose pending request finalization does + not finish within the cleanup budget +- **WHEN** the proxy emits the cleanup-budget warning +- **THEN** the warning includes `cleanup_phase=pending_requests` +- **AND** the cleanup remains owned by the existing background drain + +#### Scenario: Diagnostic phase remains low-cardinality + +- **WHEN** any WebSocket scope cleanup phase exceeds the cleanup budget +- **THEN** the warning identifies only a fixed cleanup phase +- **AND** the phase contains no request id, account id, payload, credential, or + exception content diff --git a/openspec/changes/attribute-websocket-scope-cleanup-phase/tasks.md b/openspec/changes/attribute-websocket-scope-cleanup-phase/tasks.md new file mode 100644 index 0000000000..668b52e9e1 --- /dev/null +++ b/openspec/changes/attribute-websocket-scope-cleanup-phase/tasks.md @@ -0,0 +1,12 @@ +## 1. Implementation + +- [x] 1.1 Track the current fixed WebSocket scope cleanup phase. +- [x] 1.2 Include the phase in the existing cleanup-budget warning without + changing cleanup control flow or timeout behavior. + +## 2. Validation + +- [x] 2.1 Add a route-level regression proving a blocked request-finalization + cleanup is attributed to `pending_requests`. +- [x] 2.2 Run focused WebSocket tests, proxy integration tests, lint, type + checks, architecture checks, and strict OpenSpec validation. diff --git a/openspec/changes/auto-recover-proofed-live-codex-turns/design.md b/openspec/changes/auto-recover-proofed-live-codex-turns/design.md deleted file mode 100644 index d9afcb05d6..0000000000 --- a/openspec/changes/auto-recover-proofed-live-codex-turns/design.md +++ /dev/null @@ -1,100 +0,0 @@ -# Design: proof-gated live semantic rebase - -## Eligibility - -Automatic recovery is narrower than ordinary rowless capture. It requires all -existing rowless capture and root-identity checks plus a valid official -`x-codex-turn-metadata` carrier whose request kind is `turn` and whose -`workspace_kind` is a nonblank bounded string. Body and direct carriers must -agree. Parent, fork, and subagent lineage remains ineligible. - -The complete request must have a canonical account-neutral projection, a -self-contained history, and a direct-call ledger with zero unresolved, -duplicate, orphaned, or type-mismatched calls. External refetchable images and -account-scoped resources remain ineligible. It must retain at least one prior -assistant message or agent message followed by a fresh user turn. A request -that contains only the incremental user input is insufficient proof that -completed output and settled call results were retained, even if it is -otherwise self-contained. - -## Live first-attempt recovery - -When upstream rejects the explicit anchor before `response.created` and before -any response event, the reader still owns the original downstream request and -its exact serialized body. It atomically creates and preflight-claims an -automatic authority, retaining only content-free hashes in the database. The -anchor-free projected body remains in memory. - -The current bridge reconnects on the same selected account. Before the second -physical send, it binds the durable replacement-session UNKNOWN journal and -persists the send marker. A wire mismatch, setup failure, or cancellation -before the send primitive uses the existing physically-unsent rollback. Once -the send primitive may have been reached, the authority remains UNKNOWN unless -terminal settlement proves completion. - -The stale upstream failure is never delivered when the in-place replay was -successfully submitted. The original HTTP/SSE request receives the recovered -events and `response.completed`, so Codex Desktop records the new anchor on the -same local turn instead of entering its generic reconnect path. - -## Unsent generation replacement - -An existing CAPTURED or APPROVED authority may be replaced only inside one -transaction that locks both the authority and any bound durable marker. The -transition requires: - -- the same API-key scope, stable task authority, task/session identity, - selected account, rejected anchor, and marker origin; -- no replacement session, dispatch request, wire claim, send timestamp, or - consumed timestamp; -- no marker attempt fingerprint; and -- a new request that independently satisfies automatic eligibility. - -If the request contract changed, the transaction writes a content-free audit -record for the old generation, increments the generation, installs the new -capture hashes, and clears the old operator challenge/receipt fields. It then -records an automatic authorization proof and transitions directly to UNKNOWN -with the new request/wire preflight claim. Concurrent requests lose the CAS. - -UNKNOWN and CONSUMED authorities are never replaceable. An operator-approved -generation that already started any dispatch is also never replaceable. - -Official Codex can omit `previous_response_id` while a hard-session durable -marker still requires reattachment. After owner and marker validation inject -the stale anchor, automatic preflight may use that verified anchor while -retaining the original anchorless payload as the request-contract evidence. -Once the semantic rebase claims its anchor-free wire, the temporary injection -state is cleared so store-context trimming cannot rewrite the claimed wire or -discard its authority settlement identity. - -## Authorization provenance - -Two nullable columns distinguish `operator_checkpoint` from -`automatic_live_request` authorization and bind a SHA-256 proof. Existing -approved rows are backfilled as operator checkpoint authorization. Dispatch -accepts either mode only when its corresponding proof is present. - -The automatic proof hashes the authority identity, generation and nonce, -stable task authority, captured input/contract/tool-ledger/projected-wire -fingerprints, selected account, and exact wire fingerprint. It contains no -request content, prompt text, anchor, credentials, or tool output. - -## Failure semantics - -- A successful in-place replay suppresses the original stale-anchor terminal - event. -- A physically proven pre-send failure restores an automatically authorized - generation to APPROVED so an exact later retry can be claimed without a - dashboard action. -- An ambiguous post-send outcome remains UNKNOWN and is non-replayable. -- Requests outside automatic eligibility retain the existing explicit - administrator flow and stable non-retryable HTTP 400 errors. -- A database uniqueness race at either pre-commit flush is rolled back and - mapped to the stable automatic proof-conflict error before any upstream send. - -## Rollback floor - -The status endpoint reports active automatic UNKNOWN or APPROVED authorities. -While any exist, `preAutomaticRecoveryImageCompatible` is false and the minimum -rollback capability is `rowless_automatic_recovery_v3`. Older images cannot -interpret automatic authorization provenance or its preflight rollback rules. diff --git a/openspec/changes/auto-recover-proofed-live-codex-turns/proposal.md b/openspec/changes/auto-recover-proofed-live-codex-turns/proposal.md deleted file mode 100644 index 80aaa4ae1f..0000000000 --- a/openspec/changes/auto-recover-proofed-live-codex-turns/proposal.md +++ /dev/null @@ -1,50 +0,0 @@ -# Proposal: auto-recover proofed live Codex turns - -## Problem - -The rowless semantic-rebase fence currently returns an administrator-approval -error after an eventless stale-anchor rejection. Official Codex Desktop then -finishes the local turn and may rebuild a different request on retry. The -original exact request is lost even when the gateway already proved that the -live request is a root task, self-contained, account-neutral, and has a fully -settled direct-call ledger. Requiring a dashboard click does not repair that -request loss and exposes a deterministic HTTP 400 as a misleading reconnect. - -An older APPROVED authority can have the same problem: if it has never reached -dispatch, a later official live request may contain a newer complete checkpoint -that cannot match the old whole-input fingerprint. Keeping the old generation -forever makes the original task unrecoverable even though no upstream semantic -rebase was attempted. - -## Change - -- Treat a validated official root-turn metadata carrier as eligibility for an - automatic live semantic rebase. The request must still pass every existing - self-contained, account-neutral, exact-identity, zero-unresolved-call, and - eventless stale-anchor proof. Its complete resend must also retain prior - assistant or agent output before a fresh user follow-up; an incremental-only - input remains on the operator path. -- Keep the exact anchor-free wire in memory and use the existing durable - authority, preflight CAS, replacement journal, send marker, and terminal - settlement before performing one in-place replay on the same downstream - request. -- Allow a CAPTURED or APPROVED generation to be replaced by a newer live - capture only when the old generation is physically proven never dispatched: - no dispatch request, no replacement binding, no send marker, no recovery - journal, and no durable-marker attempt claim. Increment the generation and - write a content-free audit record before claiming the new wire. -- Record whether dispatch authorization came from an operator checkpoint or - an automatic live-request proof. Preserve the existing operator APIs for - legacy and forensic use. -- Keep UNKNOWN and CONSUMED generations monotonic. An ambiguous send, identity - conflict, child lineage, incomplete input, unresolved tool call, changed - account, or changed wire remains fail closed and is never auto-replayed. - -## Non-goals - -- Disabling the exactly-once fence. -- Trusting logs, UI retry state, or unvalidated client metadata as authority. -- Automatically retrying an UNKNOWN or post-send request. -- Importing a different task or creating a replacement Codex thread. -- Treating a new user message without retained completed output as a complete - replayable history. diff --git a/openspec/changes/auto-recover-proofed-live-codex-turns/specs/responses-api-compat/spec.md b/openspec/changes/auto-recover-proofed-live-codex-turns/specs/responses-api-compat/spec.md deleted file mode 100644 index 227205c4a0..0000000000 --- a/openspec/changes/auto-recover-proofed-live-codex-turns/specs/responses-api-compat/spec.md +++ /dev/null @@ -1,78 +0,0 @@ -# Delta: Responses API compatibility - -## ADDED Requirements - -### Requirement: Proofed official live turns recover without employee approval - -The service MUST automatically rebase the same live downstream turn when an -official root Codex turn supplies valid, mutually consistent turn metadata and -an eventless stale-anchor rejection occurs, but only when the request is -self-contained, account-neutral, has zero unresolved direct calls, and can be -fenced by the existing durable at-most-once authority. The request MUST retain -prior completed assistant or agent output before its fresh user follow-up. The -employee MUST NOT need to visit the dashboard or retry the turn. - -#### Scenario: First stale rejection recovers in place - -- **GIVEN** an official root turn with a complete safe resend and a stale anchor -- **WHEN** upstream rejects the anchor before any response event -- **THEN** the service persists and claims one automatic recovery generation -- **AND** reconnects on the same account with the exact anchor-free projection -- **AND** binds the replacement journal and send marker before physical send -- **AND** returns the recovered response events on the original downstream turn -- **AND** does not emit an administrator-approval or reconnectable error. - -#### Scenario: Unsent old generation is replaced safely - -- **GIVEN** a CAPTURED or APPROVED authority with no dispatch request, - replacement binding, send marker, journal, or marker-attempt claim -- **AND** a newer eligible complete request for the same root task, anchor, - account, scope, and marker -- **WHEN** the complete request no longer matches the old input fingerprint -- **THEN** the service records a content-free supersession audit -- **AND** increments the generation and atomically claims the new exact wire -- **AND** the old generation cannot dispatch afterward. - -#### Scenario: Ambiguous and unsafe requests remain fail closed - -- **GIVEN** an UNKNOWN or CONSUMED authority, a child task, conflicting - identity metadata, an unresolved tool call, incomplete history, changed - account, changed wire, or a dispatch that may have started -- **WHEN** automatic recovery is considered -- **THEN** the service MUST NOT replace, approve, or replay that generation -- **AND** MUST preserve the existing stable fail-closed error and durable fence. - -#### Scenario: Incremental-only input keeps the operator gate - -- **GIVEN** an official root turn whose request contains a new user message but - does not retain any prior completed assistant or agent output -- **WHEN** upstream rejects its stale anchor before any response event -- **THEN** the service MUST NOT automatically claim or replay the request -- **AND** MUST preserve the operator-authorized semantic-rebase flow. - -#### Scenario: Marker-backed anchorless retry uses the verified stale anchor - -- **GIVEN** an eligible official root turn omits `previous_response_id` -- **AND** a hard-session durable marker verifies the task, account, and stale - anchor before the gateway injects that anchor for reattach -- **WHEN** a physically unsent CAPTURED or APPROVED authority is superseded by - the live request -- **THEN** automatic preflight MUST bind the marker's verified stale anchor -- **AND** the exact claimed anchor-free projection MUST reach upstream once -- **AND** later store-context processing MUST NOT trim or rebuild that wire -- **AND** `response.completed` MUST consume the authority and clear the marker. -#### Scenario: Active automatic authority raises the rollback floor - -- **GIVEN** at least one automatically authorized APPROVED or UNKNOWN authority -- **WHEN** an operator reads the rowless recovery status -- **THEN** `preAutomaticRecoveryImageCompatible` MUST be false -- **AND** the minimum rollback capability MUST be - `rowless_automatic_recovery_v3`. - -#### Scenario: Automatic completion publishes ordinary continuity - -- **GIVEN** one automatic live rebase reached its send primitive -- **WHEN** upstream publishes `response.completed` -- **THEN** the response anchor, complete input fingerprint, aliases, recovery - journal, and CONSUMED authority MUST commit atomically -- **AND** the next ordinary follow-up MUST use that new anchor. diff --git a/openspec/changes/auto-recover-proofed-live-codex-turns/tasks.md b/openspec/changes/auto-recover-proofed-live-codex-turns/tasks.md deleted file mode 100644 index 51669b2bbb..0000000000 --- a/openspec/changes/auto-recover-proofed-live-codex-turns/tasks.md +++ /dev/null @@ -1,16 +0,0 @@ -# Tasks - -- [x] Add authorization provenance columns and migration/backfill coverage. -- [x] Add repository CAS for automatic live capture, unsent generation replacement, and preflight claim. -- [x] Replay an eligible first stale-anchor rejection in the same downstream request. -- [x] Replace an eligible unsent legacy generation before account selection. -- [x] Preserve replacement journal, send-marker, rollback, UNKNOWN, and completion invariants. -- [x] Add unit, integration, concurrency, cancellation, ambiguous-send, and migration tests. -- [x] Require retained completed output before a fresh user follow-up and keep - incremental-only requests on the operator path. -- [x] Carry a verified marker-injected stale anchor through anchorless - automatic preflight without trimming or rebuilding the claimed wire. -- [x] Expose the automatic-authority rollback floor and map all uniqueness - flush races to a stable fail-closed rejection. -- [x] Validate OpenSpec, lint, type checks, and focused/full test gates. -- [ ] Release to production and recover the three original tasks serially. diff --git a/openspec/changes/background-account-deletion/design.md b/openspec/changes/background-account-deletion/design.md new file mode 100644 index 0000000000..2de8e239db --- /dev/null +++ b/openspec/changes/background-account-deletion/design.md @@ -0,0 +1,334 @@ +## Context + +Reference commit: `origin/main` 0c8d9219. Verified surfaces: +`AccountsRepository.delete()` and its fold-state lock comment, +`app/modules/accounts/usage_rollup.py` (lifetime fold, `lock_fold_state`), +`app/modules/accounts/usage_time_rollup.py` (hourly/demand/error/conversation +folds, lifecycle mirrors, history-rewrite discipline), `app/core/retention/` +(chunked prune precedent, BATCH_SIZE=10k), duplicate-account consolidation +(`_reconcile_chatgpt_identity_duplicates`, same fold lock), scheduler + leader +election pattern, Alembic single head `20260812_120000_add_sticky_abandonment_scope`. + +Production measurements (10.0.0.113): account deletion = single transaction +holding the fold-state lock for the whole drain; ~93k `usage_history` rows ≈ +11.6 s each for two accounts; ~133k `request_logs` soft-detach = 313 s (18 +indexes ≈ 8.3 GB, every row non-HOT); fold blocked; 73 pool timeouts in 36 h; +HTTP client timeout on the DELETE call. + +## Goals / Non-Goals + +**Goals:** + +- DELETE API returns in milliseconds; the account is immediately invisible to + listings and unroutable. +- Bulk row work proceeds in bounded background transactions (1k rows each, + batch selection pinned to account-leading indexes) so the fold, the pool, + and vacuum are never blocked for minutes. +- Same end state as the synchronous delete for both `delete_history` + variants, including the folded-bucket lifecycle mirrors and the + "rollup row deleted with the account row" invariant. +- Restart-safe resume, idempotent repeat requests, explicit supersede path. + +**Non-Goals:** + +- No change to duplicate-account consolidation (still synchronous under the + fold lock; its row volume is bounded by the duplicate's history and it must + stay atomic with the identity swap). +- No change to retention, fold cadence, or watermark semantics. +- No new settings; no dashboard UI for drain progress (the account is simply + gone; the worker logs outcomes). + +## Decisions + +### D1: Terminal mark = existing `DEACTIVATED` status + marker columns, not a new enum value + +Serving-path exclusion lists are written as denylists +(`status not in (REAUTH_REQUIRED, DEACTIVATED, PAUSED)` in +`proxy/helpers.py`, `load_balancer.py`, `account_cache.py`, `proxy/api.py`, +`realtime_live.py`): a new `DELETING` enum value would be *routable* until +every denylist was found and extended, and would require a PostgreSQL enum +migration. Reusing `DEACTIVATED` inherits every existing exclusion (sticky +purge, bridge close, selection caches) with zero new status handling. The +pending-deletion state itself lives in `accounts.delete_requested_at` +(authoritative marker + queue ordering) and `delete_history_requested` +(variant, frozen at request time); `deactivation_reason="pending_deletion"` +is operator-facing only. + +The marker also fences ordinary status writers (`update_status`, +`update_status_if_current` gain `delete_requested_at IS NULL`): a stale +in-flight settlement — e.g. a 429 for a request selected before the DELETE — +would otherwise replace `DEACTIVATED` with `RATE_LIMITED` and make the +account selectable again for direct-by-id paths mid-drain. Credential +replacement does not go through these writers (it writes fields directly and +clears the marker in the same transaction), so the supersede path is +unaffected. Pre-upgrade replicas' writers are unfenced during a rolling +deploy, so every drain chunk additionally self-heals: when the marked row +drifted (non-terminal status or a recreated API-key assignment) without a +credential replacement, the chunk re-asserts the terminal status and +re-removes the assignments under the row lock it already holds — a DB +trigger was rejected as disproportionate machinery for a drift window that +is already bounded to one chunk transaction (seconds). + +Repeat DELETE requests short-circuit on an unlocked marker+wipe read before +entering the writer section / row lock: a drain chunk holds the account row +(and, on SQLite, the writer section) for seconds at a time, and the +fast-path contract must hold throughout the drain. The short-circuit falls +through to the full path when the credentials were replaced without +clearing the marker, so an explicit re-delete after a legacy replacement +re-wipes and re-arms the deletion. + +`begin_delete` additionally produces the two projections the synchronous +delete's row removal produced instantly: it deletes the account's +`ApiKeyAccountAssignment` rows (key listings and pooled-usage reads exclude +the account immediately; the key's persisted `account_assignment_scope_enabled` +flag keeps the key scoped, exactly as after the FK cascade) and overwrites +the access/refresh/id token ciphertext with empty-credential ciphertext. +The wipe is what keeps the rolling upgrade honest: a pre-upgrade replica's +export endpoints read the row without knowing the marker, and must not be +able to hand out usable credentials during the drain window. Token rotation +is CAS-guarded on the pre-wipe refresh ciphertext (a stale rotation misses), +and every supersede path writes complete fresh ciphertext. The wipe must +not break the reauth supersede path itself: targeted reauthentication +verifies the seat against `chatgpt_user_id` or — on legacy rows where it +was never backfilled — the stored id-token claims, so `begin_delete` +backfills `chatgpt_user_id` from those claims (non-secret identity) in the +same transaction before destroying them. + +The wipe doubles as the supersede signal for pre-upgrade replicas: a +replacement handled by old code writes fresh ciphertext but cannot clear +marker columns its ORM does not know. Every marker re-check (chunk and +finalization) therefore also inspects ALL THREE token ciphertexts — +non-wiped or undecryptable material in any field of a marked row means a +replacement happened (a legal replacement may carry an empty refresh token +while providing fresh access/id material, so a refresh-only check would +finalize a freshly replaced account), and the worker clears the marker +itself (under the row lock) instead of draining further or finalizing. API-key assignment validation +(`ApiKeysRepository.list_accounts_by_ids`) likewise rejects marked +accounts, and `replace_account_assignments` locks the target account rows (`FOR SHARE` +on PostgreSQL, which conflicts with `begin_delete`'s row update) BEFORE +touching any assignment row — the same account-then-assignment order +`begin_delete` uses, so the race serializes instead of deadlocking — and +then inserts through a conditional `INSERT … SELECT … WHERE +delete_requested_at IS NULL`. A key create/update whose validation raced +the DELETE therefore cannot recreate an assignment that would re-surface +the account in key listings: either it commits first and `begin_delete`'s +assignment cleanup removes its rows, or the marker is visible to the +insert and the account is skipped. + +### D2: The account row is the queue (no new table) + +The marker columns make the `accounts` row its own durable work item: the +worker scans `delete_requested_at IS NOT NULL`, progress is the shrinking +`WHERE account_id = :id` predicates, and finalization's row delete is the +dequeue. Restart resume and idempotency need no extra state machine; a +crash between any two chunk transactions loses nothing. + +### D3: Chunks do NOT take the fold-state lock; only finalization does + +The single-transaction delete held the fold lock to prevent an in-flight +fold slice from committing pre-delete attribution after the mirrors ran +(resurrecting folded rows). The chunked drain preserves that invariant with +lock-free chunks because: + +1. Chunk transactions touch only raw rows; they never write a rollup table + or move a watermark (`usage_history` tables are not fold-governed at all). +2. An interleaved fold slice aggregates either still-attached rows (folded + under the account dimension) or already-detached rows (folded under the + orphaned-deleted dimension — the soft-path end state). Both converge at + finalization: it takes `lock_fold_state()`, detaches/deletes residual raw + rows, and runs the lifecycle mirrors, which move or remove EVERY folded + row carrying the account dimension — including rows folded mid-drain. +3. Every fold slice holds the fold-state row lock (`FOR UPDATE` on the + `account_usage_rollup_state` row) from before it reads raw rows until its + commit. A slice therefore commits strictly before finalization (its + output is mirrored) or strictly after (it sees no attributed raw rows). + Post-finalization resurrection is impossible. + +Per-chunk fold-lock acquisition was considered and rejected: it adds fold +stalls proportional to drain length while providing nothing the finalization +lock does not already guarantee (the mirrors are a pure dimension move over +whatever is folded at mirror time). + +The letter of the history-rewrite discipline in `usage_time_rollup.py` +("mutations of folded dimensions below the watermark take the fold lock and +mirror or skip **in the same transaction**") is relaxed for this one path: +mid-drain, folded buckets may still attribute to an account whose raw rows a +chunk already detached. That intermediate state never double- or +under-counts a read (folded side serves below-watermark, raw tail above; +the watermark folds each raw row exactly once, and already-folded rows are +never re-read), and on the deletion path it is bounded by drain duration: +the end state is byte-identical to the synchronous path, with the module +docstring of `deletion.py` documenting this as the single sanctioned +exception, converged by finalization. + +When a supersede lands after a partial drain, finalization never runs and +the divergence for rows drained before the supersede is the PERMANENT, +intended end state: folded buckets keep attributing that traffic to the +revived account (it is the account's true pre-delete history — nothing is +added or inflated), while the raw rows stay detached (soft) or deleted +(`delete_history`), exactly as the "rows already drained stay detached" +trade-off promises. Reads stay consistent for the same reason as mid-drain: +below-watermark reads are folded-only, drained rows below the watermark are +never re-folded, and drained rows above the watermark fold once under the +orphaned dimension. Reconciling instead (running the lifecycle mirrors at +supersede time) was rejected: it would drag the fold lock and per-row delta +mirroring into every credential-replacement path to "fix" attribution that +is already historically correct. + +### D4: Finalization reuses `AccountsRepository.delete()` with a marker guard + +`delete(only_pending=True)` is the historical transaction verbatim — +identity-membership lock (PostgreSQL), fold-state lock, residual +usage-history delete, residual detach/delete + mirrors, sticky + rollup + +account row — plus: it aborts (touching nothing) unless the marker is still +set, and it reads the `delete_history` variant from the persisted flag +rather than the caller. The identity-membership `FOR NO KEY UPDATE` row lock +(PostgreSQL) keeps the marker stable through the transaction; on SQLite the +writer section serializes writers. Lock order (identity → fold) matches +consolidation, so no new deadlock ordering is introduced. Residual rows also +cover stragglers: a stream that started before the mark settles its +request-log row at stream end, possibly after every chunk ran. + +After the fold lock, finalization upgrades the account row to a full +`FOR UPDATE` lock (PostgreSQL) before the residual sweeps. `FOR UPDATE` +conflicts with the `KEY SHARE` a request-log FK insert takes, so an +in-flight stream's insert either commits before the sweep (and is swept) or +blocks until the transaction commits and then fails its FK against the +deleted row — the same outcome a post-delete insert always had. Without the +upgrade, an insert could commit between the sweep and the account-row +delete, where `ON DELETE SET NULL` would leave a live (`deleted_at IS +NULL`) orphan on the soft path or surviving raw history under +`delete_history`. The lock order (identity → fold → row exclusive) matches +the historical transaction, whose final `DELETE` acquired the same +exclusive lock after the fold lock. + +### D5: Supersede-by-replacement, first-request-wins idempotency + +`_apply_account_updates` (every credential replacement: re-import, reauth, +slot reuse) clears the marker: account ids are deterministic, so +delete-then-reimport lands on the marked row, and letting the worker delete +a just-reimported account would be data loss. Every chunk transaction and +finalization re-read the marker under the account row lock (PostgreSQL +`FOR NO KEY UPDATE`, compatible with the `KEY SHARE` taken by concurrent +rollup FK inserts; on SQLite the writer section serializes writers), so a +replacement either commits before the marker read (the chunk sees the +cleared marker and stops) or blocks until the chunk commits — no chunk can +mutate rows after a replacement has successfully returned, and a superseded +account is never finalized (rows already drained stay detached — history +loss was requested by the earlier delete). The chunk takes only the account +row lock and touches only that account's child rows, so no new lock ordering +is introduced. Marked accounts are also absent from the credential-export +endpoints: the synchronous delete made exports 404 immediately, and the +asynchronous drain window must not keep decrypted tokens retrievable after +a successful DELETE. Repeat DELETE requests return +success without escalating `delete_history` (first request wins), matching +the synchronous world where a second DELETE arrived after the account was +already gone. `reactivate_account` treats a marked account as not found +rather than racing the worker back to ACTIVE. + +### D6: Worker = leader-gated 30 s tick + local wake, cheap pre-check, round-robin pass + +Same scheduler shape as retention. Each tick runs one `LIMIT 1` existence +probe *before* leader election — served by the partial index +`idx_accounts_delete_requested_at` (`WHERE delete_requested_at IS NOT +NULL`), which is empty in the steady state — so a tick with nothing to do +costs one tiny index probe. `delete_account` wakes the local worker after +commit: on the leader (the single-replica common case) draining starts +immediately; a follower's wake is a no-op and the leader's tick picks the +request up within 30 s. Batch size 1k: measured ~1.2 s/10k `usage_history` +deletes and ~23 s/10k `request_logs` detaches (18 indexes, non-HOT updates) +bound the worst table at ~2.3 s per transaction — and every chunk holds the +account row lock (`FOR NO KEY UPDATE`) for its full duration, so a supersede +or fenced settlement waits for at most one chunk. Between row-touching +rounds the pass sleeps a fraction of the round's own duration (capped), so +a multi-hundred-chunk drain leaves the 2-vCPU database headroom instead of +running chunk transactions back-to-back. + +Chunk batch selection is planner-pinned to the account-leading indexes +(`idx_usage_account_time`, `ix_additional_usage_distinct_labels`, +`idx_logs_account_kind_deleted_latest` — the last one covering, so the +request-log batch is an index-only scan): the batch subquery selects with an +`account_id >= :id AND account_id <= :id` range (equivalent rows, but the +range keeps `account_id` out of the constant-equivalence class and thus in +the sort pathkeys) ordered by the target index's exact column order, making +that index the only sort-free plan. A plain `account_id = :id LIMIT n` shape +was verified on the production planner to run as a LIMIT-terminated Seq +Scan for exactly the large accounts this change targets — with an unbounded +dead-prefix re-scan mid-drain and a guaranteed full heap scan for every +empty probe once per-account statistics go stale. With the pinned shape, +chunk scan work is bounded by the account's own remaining rows and a +drained-table probe is a single index descent. Within one pass, a table +observed empty is not re-probed on later rounds (rows settling mid-drain +are converged by finalization's residual sweep anyway). + +A deletion pass round-robins: each round advances every pending account by +at most one NONEMPTY chunk transaction (a round stops at the first chunk +that touched rows, not the first batch-size-full one, so small tables cannot +stack several row-touching transactions into one round) and the pending set +is re-scanned between rounds. A multi-minute drain (the measured 133k-row +account is ~133 chunks) therefore cannot starve another marked account, and +a DELETE that lands mid-pass is picked up by the next round's re-scan rather +than waiting for the whole pass to finish. + +### D7: API contract unchanged (`{"status": "deleted"}`) + +The dashboard's delete mutation only toasts and refetches the listing, which +already excludes the marked account — the operator-visible contract ("after +DELETE, the account is gone from the list") holds exactly. Returning a new +`"deleting"` status would break any consumer comparing against "deleted" +while conveying nothing actionable: the deletion is irrevocable (modulo +re-import) once the API returns. The spec states row purge is asynchronous. + +## Risks / Trade-offs + +- **Mid-drain visibility**: statistics pages may briefly attribute folded + history to the (invisible) account while raw rows are already detached. + Bounded by drain duration; strictly better than the previous minutes-long + fold outage. +- **Interleaved folds vs hard delete**: a fold slice between hard-delete + chunks may fold rows (account and API-key aggregates) that the + single-transaction path would have deleted first. This is inherent fold + timing (a fold 1 s before the DELETE captured them under the old code + too); the account side is removed by the mirrors, and API-key folded sums + keeping settled traffic is the documented behavior for folded history. +- **Supersede after partial drain**: a re-import that lands mid-drain keeps + the account but its already-detached rows stay detached (and + already-deleted rows stay deleted), while folded buckets keep attributing + the pre-supersede-drained traffic to the revived account — permanently, + since finalization's mirrors never run. This is historically correct + attribution (the folded numbers pre-existed the delete), never double- or + under-counts a read (see D3), and is the documented consequence of the + operator asking for deletion first. +- **Alembic head races**: the revision sits on the current single head; + parallel PRs adding revisions require the usual head merge. + +## Migration + +`20260816_000000_add_account_pending_deletion`: adds +`accounts.delete_requested_at` (nullable DateTime), +`accounts.delete_history_requested` (Boolean, `server_default false`), and +the partial queue index `idx_accounts_delete_requested_at` +(`(delete_requested_at, id) WHERE delete_requested_at IS NOT NULL`), with +existence guards and a symmetric downgrade. Existing rows are +untouched (no pending deletions can predate the feature). Rolling upgrade: an +old replica neither sets nor reads the marker; a delete handled by an old +replica is simply the old synchronous delete, and a delete handled by a new +replica leaves old replicas nothing exploitable — the fast path wipes the +token ciphertext, so old export/read paths that do not know the marker can +only produce empty credentials until finalization removes the row (old +replicas may transiently show the account in listings during the mixed +window; it is unroutable via the terminal status either way). + +One mixed-window caveat follows directly from "old replica = old delete": a +repeat DELETE routed to a pre-upgrade replica while the row is still marked +runs the legacy synchronous delete with its caller-provided +`delete_history` variant, which can differ from the frozen first-request +choice (either direction). The legacy delete is still a complete, +fold-locked, mirror-correct deletion — only the history-policy choice +diverges, only inside the deploy window, and only when the operator issues +contradictory repeat requests inside it. Fencing was rejected: new code +cannot retrofit a fence into binaries that predate the marker columns, a +database trigger is disproportionate machinery for the window, and a +"defer background marks until the fleet is upgraded" gate would add a +permanent setting for a transient condition (single-replica deployments — +the production topology — have no mixed window at all). diff --git a/openspec/changes/background-account-deletion/proposal.md b/openspec/changes/background-account-deletion/proposal.md new file mode 100644 index 0000000000..4c1292cabf --- /dev/null +++ b/openspec/changes/background-account-deletion/proposal.md @@ -0,0 +1,56 @@ +## Why + +`DELETE /api/accounts/{id}` detaches (or deletes) the account's entire raw +history in one transaction while holding the fold-state lock. Measured on +production: ~11.6 s to delete ~93k `usage_history` rows and **313 s** to +soft-detach ~133k `request_logs` rows (18 indexes, every row non-HOT). For +those minutes the fold is blocked, one pool connection is pinned (contributing +to `QueuePool` timeouts), the HTTP client times out, and the long transaction +delays vacuum. + +## What Changes + +- `DELETE /api/accounts/{id}` becomes a fast mark: the account turns terminal + (`DEACTIVATED` + a pending-deletion marker), disappears from listings and + serving immediately, and the API returns within milliseconds with the + existing `{"status": "deleted"}` contract. +- A new leader-gated background worker drains the account's bulk rows + (`usage_history`, `additional_usage_history`, `request_logs`) in bounded + chunks (5k rows per transaction, no fold-state lock), then finalizes in one + fold-state-locked transaction with the exact shape of the old synchronous + delete: residual rows, folded-bucket lifecycle mirrors, sticky/rollup rows, + account row. +- Deletion is restart-safe (all progress in the database), idempotent + (repeat DELETE requests succeed without escalating the frozen + `delete_history` choice), supports both `delete_history` variants, and is + superseded by a credential replacement (re-import/reauth) that clears the + marker. +- Schema: two new nullable-safe columns on `accounts` + (`delete_requested_at`, `delete_history_requested`), Alembic revision + `20260816_000000_add_account_pending_deletion` on the current single head. + +## Capabilities + +### New Capabilities + +- `account-deletion`: asynchronous account deletion lifecycle — fast terminal + mark, immediate listing/serving exclusion, chunked background drain with + fold-interleave safety, restart-safe finalization, idempotency, and + supersede-by-replacement semantics. + +### Modified Capabilities + +(none — the query-caching lifecycle requirement "rollup row deleted in the +same transaction as the account deletion" continues to hold: the finalization +transaction removes both together.) + +## Impact + +- `app/modules/accounts/repository.py` (`begin_delete`, marker-guarded + `delete(only_pending=True)`, listing filters, replacement clears marker), + `app/modules/accounts/deletion.py` (new worker + scheduler), + `app/modules/accounts/service.py`, `app/main.py` (scheduler wiring), + `app/db/models.py`, one Alembic revision. +- No API schema change (`AccountDeleteResponse` unchanged), no frontend + change (the listing refetch after delete already sees the account gone), + no new settings. diff --git a/openspec/changes/background-account-deletion/specs/account-deletion/spec.md b/openspec/changes/background-account-deletion/specs/account-deletion/spec.md new file mode 100644 index 0000000000..ed6dd89fad --- /dev/null +++ b/openspec/changes/background-account-deletion/specs/account-deletion/spec.md @@ -0,0 +1,321 @@ +# account-deletion Delta + +## ADDED Requirements + +### Requirement: Account deletion requests return fast and hide the account immediately + +`DELETE /api/accounts/{account_id}` MUST NOT perform the account's bulk row +work (raw request-log detach/delete, usage-history removal) on the request +path. The request MUST only stamp a durable pending-deletion marker in a +short transaction: terminal `DEACTIVATED` status, the pending-deletion +marker (`delete_requested_at`), the frozen `delete_history` choice +(`delete_history_requested`), sticky-session removal, bridge-session +closure, API-key account-assignment removal (the projection the synchronous +delete's FK cascade produced — key listings and pooled-usage reads exclude +the account immediately while the key's persisted assignment-scope flag is +untouched), and an overwrite of the stored access/refresh/id token +ciphertext with empty-credential ciphertext so that NO reader of the +surviving row — including a pre-upgrade replica during a rolling deploy, +whose export endpoints do not know the marker — can produce usable +credentials during the drain window. Repeat DELETE requests MUST +short-circuit before taking the account row lock or the SQLite writer +section, so the millisecond contract holds even while a drain chunk +transaction is holding the row. Because targeted reauthentication (a +supersede path) verifies the seat against `chatgpt_user_id` or, on legacy +rows where it was never backfilled, the stored id-token claims, the fast +path MUST preserve the non-secret seat identity before the wipe by +backfilling `chatgpt_user_id` from those claims when it is absent. The +response contract remains `{"status": "deleted"}` with 200 for an existing +account and 404 otherwise; row purge is asynchronous. + +Accounts carrying the pending-deletion marker MUST be excluded from account +listings (`GET /api/accounts` and every listing-derived read) and MUST be +excluded from proxy serving via the terminal status. EVERY ID-based account +surface MUST report a marked account as not found (or absent) — reads +(trends, reset-credit views), mutations (account update, alias, +limit-warmup, routing policy, upstream-proxy binding), action routes +(pause, probe, reset-credit consumption on both the dashboard and +rate-limit route families, `/v1` reset-credit redemption, reactivation), +and the credential-export endpoints (account export, auth export, opencode +auth export) — because the synchronous delete returned 404 on all of them +once the row was removed, and a successful DELETE MUST NOT leave decrypted +tokens retrievable during the background drain window. Only +credential-replacement paths (re-import, reauthentication) may address the +marked row. + +Ordinary status writes MUST NOT modify a marked account: a stale in-flight +settlement (for example a 429 landing after the DELETE for a request +selected before it) must not replace the terminal `DEACTIVATED` state and +make the account selectable mid-drain. Only a credential replacement — +which clears the marker — may change a marked account's state. Because +pre-upgrade replicas' status writers are unfenced during a rolling deploy, +every drain chunk transaction MUST re-assert the terminal status (and +re-remove any recreated API-key assignments) under the account row lock +when the marked row has drifted without a credential replacement, bounding +such drift to one chunk transaction; after the repairing chunk commits, the +worker MUST propagate the same cache invalidation as the delete request +(routing unavailability, selection/API-key snapshots, routing-change bump) +so replicas that cached the drift stop serving it. + +Marked accounts MUST be rejected by API-key account-assignment validation +and excluded from API-key pooled-usage projections, and assignment +insertion MUST re-check the marker atomically with the write (a conditional +insert; on PostgreSQL additionally serialized against the delete mark by a +`FOR SHARE` lock on the account rows, acquired BEFORE any assignment-row +mutation so the lock order matches the delete path's account-then-assignment +order and the race serializes instead of deadlocking), so an assignment +created or updated after — or racing — the DELETE cannot re-surface the +account in key listings before finalization. + +#### Scenario: Delete responds without draining rows + +- **GIVEN** an account with raw request-log and usage-history rows +- **WHEN** `DELETE /api/accounts/{id}` returns 200 `{"status": "deleted"}` +- **THEN** the account no longer appears in `GET /api/accounts` +- **AND** the account row still exists, terminal and marked, with its raw + rows untouched until the background worker drains them + +#### Scenario: Marked account cannot be reactivated + +- **GIVEN** an account marked for background deletion +- **WHEN** `POST /api/accounts/{id}/reactivate` is called +- **THEN** the response is 404 `account_not_found` + +#### Scenario: All ID-based routes report the marked account as gone + +- **GIVEN** an account marked for background deletion whose rows are not yet + drained +- **WHEN** any ID-based account route (trends, reset-credit read/consume, + probe, pause, update, alias, limit-warmup, routing policy) is called +- **THEN** the response is 404 `account_not_found` + +#### Scenario: Marked account no longer serves credential exports + +- **GIVEN** an account marked for background deletion whose rows are not yet + drained +- **WHEN** any credential-export endpoint is called for the account +- **THEN** the response is 404 and no token material is returned +- **AND** the row's stored token ciphertext decrypts to empty credentials + (nothing usable remains for readers that do not know the marker) + +#### Scenario: Seat identity survives the token wipe for reauth supersede + +- **GIVEN** a legacy account whose `chatgpt_user_id` is unset (seat identity + lives only in the stored id-token claims) +- **WHEN** `DELETE /api/accounts/{id}` marks the account and wipes the token + ciphertext +- **THEN** `chatgpt_user_id` is backfilled from the id-token claims in the + same transaction, so a targeted reauthentication can still verify the + seat and supersede the deletion + +#### Scenario: Deleted account leaves API-key listings immediately + +- **GIVEN** an account assigned to an API key +- **WHEN** `DELETE /api/accounts/{id}` returns +- **THEN** the key's listed assigned-account ids no longer contain the + account and its pooled-usage projection excludes it +- **AND** the key's assignment-scope flag remains enabled + +#### Scenario: Marked account cannot be assigned to an API key + +- **GIVEN** an account marked for background deletion +- **WHEN** an API-key create or update names the account in its assigned + account ids +- **THEN** the request is rejected as referencing an unknown account + +#### Scenario: Stale settlement cannot resurrect a marked account + +- **GIVEN** an account marked for background deletion +- **WHEN** an ordinary status write (e.g. a late rate-limit settlement) + targets the account +- **THEN** the write is rejected and the account stays terminal and marked + +#### Scenario: Drift written by an unfenced pre-upgrade replica is re-fenced + +- **GIVEN** a marked account whose status was replaced (or whose API-key + assignment was recreated) by a pre-upgrade replica's unfenced writer, + with the token ciphertext still wiped +- **WHEN** the next drain chunk transaction runs +- **THEN** the terminal status and reason are re-asserted and the recreated + assignment is removed, in the same chunk transaction + +#### Scenario: Repeat delete stays fast during an active drain + +- **GIVEN** a marked account whose drain chunk transaction currently holds + the account row lock +- **WHEN** a repeat `DELETE /api/accounts/{id}` arrives +- **THEN** it returns success without waiting for the chunk transaction + +### Requirement: Background worker drains marked accounts in bounded chunks + +A leader-gated background worker MUST drain each marked account's +`usage_history`, `additional_usage_history`, and `request_logs` rows in +bounded per-transaction chunks (at most `DELETE_BATCH_SIZE` rows per +transaction) without holding the fold-state lock, and MUST then finalize in +ONE fold-state-locked transaction that detaches or deletes residual raw rows +(including request-log rows settled mid-drain by in-flight streams), runs +the folded-bucket lifecycle mirrors, and removes the sticky, lifetime-rollup, +and account rows together. Finalization MUST serialize against in-flight +raw-row inserts (on PostgreSQL by upgrading the account row to a full lock +that conflicts with the FK's `KEY SHARE` before the residual sweep), so a +log row committed by an in-flight stream is either swept by finalization or +its insert fails against the already-deleted account — finalization may +leave behind neither a live orphan row (soft variant) nor surviving raw +history (`delete_history` variant). The soft variant MUST detach raw rows +(`account_id=NULL, deleted_at` set); the `delete_history` variant MUST +delete them. The worker MUST start a drain promptly after a delete request +on the leader replica and within one worker interval otherwise. A deletion +pass MUST round-robin across pending accounts — at most one nonempty chunk +transaction per account per round — and re-scan for newly marked accounts +between rounds, so one account's long drain cannot delay another marked +account's drain start by more than one chunk transaction per pending +account. Chunk batch selection MUST be served by an account-leading index +on every drain table (on PostgreSQL: `idx_usage_account_time`, +`ix_additional_usage_distinct_labels`, and the covering +`idx_logs_account_kind_deleted_latest`), so per-chunk scan work is bounded +by the account's own remaining rows: it MUST NOT degrade to sequential +scans when per-account statistics are large or stale mid-drain, and a probe +of an already-drained table MUST terminate on the index without scanning +the heap. Within one pass, the worker MUST NOT re-probe a drain table it +already observed empty for an account (rows that land after that +observation are swept by finalization's residual pass), and it MUST pause +between consecutive row-touching rounds in proportion to the round's +duration so a long drain does not run chunk transactions back-to-back. + +#### Scenario: Chunked drain reaches the synchronous end state (soft) + +- **GIVEN** a marked account whose raw rows exceed one chunk +- **WHEN** the worker completes the drain and finalization +- **THEN** every raw request-log row is detached and soft-deleted, usage + snapshots are removed, and the sticky, lifetime-rollup, and account rows + are deleted in the finalization transaction + +#### Scenario: Chunked drain reaches the synchronous end state (delete_history) + +- **GIVEN** an account marked with the `delete_history` variant +- **WHEN** the worker completes the drain and finalization +- **THEN** the account's raw request-log rows are deleted and its folded + time-axis buckets are removed + +#### Scenario: In-flight log insert cannot escape finalization + +- **GIVEN** a marked account whose drain is complete and an in-flight stream + holding an uncommitted request-log insert for it +- **WHEN** finalization runs +- **THEN** finalization waits for the insert to commit and sweeps the late + row (or the insert fails against the deleted account), leaving no live + orphan and no surviving history + +#### Scenario: A long drain does not starve other marked accounts + +- **GIVEN** one marked account whose drain spans many chunks +- **WHEN** another account is marked for deletion (before or during the pass) +- **THEN** the second account's drain starts within one chunk round and both + accounts finalize + +#### Scenario: Chunk selection stays on the account index + +- **GIVEN** a marked account whose per-account row estimate is large (or + stale after a partial drain) +- **WHEN** a drain chunk selects its batch on PostgreSQL +- **THEN** the batch subquery is planned as a scan of the account-leading + index on each drain table (index-only for `request_logs`), not a + sequential scan, and an empty probe terminates on the index + +#### Scenario: Drained tables are not re-probed within a pass + +- **GIVEN** a marked account whose usage tables drained while its request + logs still span further chunks +- **WHEN** subsequent rounds of the same pass advance the account +- **THEN** the drained tables' chunk transactions do not run again and + finalization still sweeps any rows that landed after the empty + observation + +### Requirement: Interleaved fold slices never resurrect a deleted account's folded rows + +Fold passes MUST remain able to run between drain chunks. Because every fold +slice holds the fold-state row lock from before reading raw rows until its +commit, and finalization takes the same lock before running the lifecycle +mirrors over whatever is folded at that moment, a fold slice MUST either +commit before finalization (its account-attributed output is moved or +removed by the mirrors) or after (it observes no raw rows attributed to the +account). After finalization commits, no folded row in any rollup table may +carry the deleted account's dimension, and under the soft variant the +orphaned-deleted dimension MUST preserve the account's full folded history. + +#### Scenario: Fold between chunks is converged by finalization + +- **GIVEN** a marked account with part of its raw history already detached + by drain chunks and part still attached +- **WHEN** a fold pass commits between chunks (attributing the still-attached + rows to the account) and the worker then completes finalization +- **THEN** no rollup table contains rows under the account's dimension +- **AND** (soft variant) the orphaned-deleted dimension carries the account's + complete folded history +- **AND** fold passes run after finalization add nothing under the account's + dimension + +### Requirement: Deletion is restart-safe, idempotent, and superseded by credential replacement + +All drain progress MUST live in the database so a worker restart resumes an +interrupted deletion with no separate recovery step. Repeat DELETE requests +for a marked account MUST succeed idempotently and MUST NOT change the +frozen `delete_history` choice (first request wins). The first-request-wins +invariant is scoped to replicas running this revision: during a rolling +deploy, a repeat DELETE routed to a pre-upgrade replica performs the legacy +synchronous delete with its caller-provided variant (exactly the pre-change +behavior — a complete, mirror-correct deletion whose variant choice may +differ from the frozen one). This window is bounded by the deploy itself, +requires the operator to issue contradictory repeat requests inside it, and +is accepted: new code cannot fence binaries that predate the marker, and a +deployment gate would add permanent configuration for a transient window. A credential +replacement (re-import or reauthentication landing on the marked row) MUST +clear the marker and supersede the deletion: every drain chunk and the +finalization transaction MUST re-check the marker under the account row lock +(PostgreSQL `FOR NO KEY UPDATE`; the SQLite writer section) before mutating +rows, so no chunk commits row work after a replacement committed and a +superseded account is never finalized (rows already drained stay detached). + +After a supersede that followed a partial drain, rows drained before the +replacement keep their drained end state (detached under the soft variant, +deleted under `delete_history`), and folded rollups keep attributing the +pre-supersede-drained traffic to the revived account: it is the account's +true pre-delete history, and reads MUST NOT double- or under-count as a +result (below-watermark reads are folded-only; drained rows above the +watermark fold exactly once, under the orphaned dimension). + +A credential replacement handled by a pre-upgrade replica during a rolling +deploy writes fresh credentials but cannot clear marker columns unknown to +its ORM. The worker MUST therefore treat non-wiped (or undecryptable) +ciphertext in ANY of the access/refresh/id token fields of a marked row as +a credential replacement (a legal replacement may carry an empty refresh +token while providing fresh access/id material): it MUST clear the marker +itself under the account row lock and abandon the deletion without mutating +any further rows. + +#### Scenario: Restart resumes a partial drain + +- **GIVEN** a marked account whose drain was interrupted after some chunks +- **WHEN** a fresh worker pass runs +- **THEN** the drain resumes from the database state and finalizes normally + +#### Scenario: Repeat delete does not escalate the variant + +- **GIVEN** an account marked by a request without `delete_history` +- **WHEN** a second `DELETE` request arrives with `delete_history=true` +- **THEN** the request succeeds and the frozen choice remains the soft variant + +#### Scenario: Re-import supersedes a pending deletion + +- **GIVEN** a marked account mid-drain +- **WHEN** a credential replacement lands on the row and clears the marker +- **THEN** the worker abandons the deletion without removing the account row +- **AND** rows detached before the replacement remain detached + +#### Scenario: Legacy-replica replacement supersedes without clearing the marker + +- **GIVEN** a marked account mid-drain whose credentials were replaced by a + pre-upgrade replica (fresh ciphertext, marker still set) +- **WHEN** the worker's next chunk or finalization re-checks the row +- **THEN** the worker clears the marker, abandons the deletion, and the + fresh credentials survive diff --git a/openspec/changes/background-account-deletion/tasks.md b/openspec/changes/background-account-deletion/tasks.md new file mode 100644 index 0000000000..246ec37486 --- /dev/null +++ b/openspec/changes/background-account-deletion/tasks.md @@ -0,0 +1,105 @@ +## 1. Schema + +- [x] 1.1 Add `accounts.delete_requested_at` and + `accounts.delete_history_requested` columns (model + Alembic revision + `20260816_000000_add_account_pending_deletion` on the current head, + guarded upgrade/downgrade). +- [x] 1.2 Partial queue index `idx_accounts_delete_requested_at` + (`(delete_requested_at, id) WHERE delete_requested_at IS NOT NULL`) so + the per-interval pending probe and the queue-order scan never touch the + full accounts table. + +## 2. Fast delete path + +- [x] 2.1 `AccountsRepository.begin_delete`: terminal `DEACTIVATED` mark + + pending marker + sticky/bridge cleanup in one short transaction; + idempotent, first request freezes the `delete_history` choice. +- [x] 2.2 Hide marked accounts from `list_accounts` / `list_accounts_by_ids`; + block reactivation of marked accounts; keep the DELETE response + contract (`{"status": "deleted"}`). +- [x] 2.3 Clear the marker in `_apply_account_updates` so credential + replacement supersedes a pending deletion. +- [x] 2.4 Hide marked accounts from the credential-export endpoints (account + export, auth export, opencode auth export): 404 during the drain + window, matching the synchronous delete's contract. +- [x] 2.5 Wipe the stored token ciphertext in `begin_delete` so readers that + do not know the marker (pre-upgrade replicas during a rolling deploy) + cannot export usable credentials mid-drain; backfill the non-secret + seat identity (`chatgpt_user_id`) from the id-token claims first so + targeted reauthentication can still verify and supersede. +- [x] 2.6 Remove the account's API-key assignments in `begin_delete` (the + projection the synchronous FK cascade produced): key listings and + pooled-usage reads exclude the account immediately, scope flag intact. +- [x] 2.7 Fence ordinary status writers (`update_status`, + `update_status_if_current`) on `delete_requested_at IS NULL` so stale + in-flight settlements cannot resurrect a marked account. +- [x] 2.8 Treat non-wiped credential ciphertext on a marked row as a + supersede (replacement by a pre-upgrade replica that cannot clear the + marker): chunks and finalization clear the marker and abandon. +- [x] 2.9 Reject marked accounts in API-key assignment validation + (`ApiKeysRepository.list_accounts_by_ids`) so post-DELETE key updates + cannot recreate assignments for the account. +- [x] 2.10 Atomic marker re-check in `replace_account_assignments` + (conditional INSERT…SELECT, `FOR SHARE` on PostgreSQL) so validation + that raced the DELETE cannot recreate an assignment. +- [x] 2.11 Finalization upgrades the account row to `FOR UPDATE` before the + residual sweeps (PostgreSQL) so in-flight FK inserts are either swept + or fail post-delete — no live orphans, no surviving history. +- [x] 2.12 Per-chunk self-heal of drift written by unfenced pre-upgrade + replicas: re-assert terminal status and re-remove recreated API-key + assignments under the chunk's row lock. +- [x] 2.13 Repeat DELETE short-circuits on an unlocked marker+wipe read so + the millisecond contract holds while a drain chunk holds the account + row lock / SQLite writer section. +- [x] 2.14 Treat marked accounts as absent on every ID-based account surface + (trends, reset-credit read/consume on both route families, probe, + pause, update, alias, limit-warmup, routing policy, upstream-proxy + binding, `/v1` reset-credit redemption) via a marker-aware fetch or an + atomic write predicate; filter the marker in the unscoped API-key pool + query (`list_all_accounts`) as well. +- [x] 2.15 Propagate the delete-request cache invalidation after a chunk + repairs pre-upgrade-replica drift, so cached drift stops being served. + +## 3. Background worker + +- [x] 3.1 `app/modules/accounts/deletion.py`: chunked drain + (usage_history, additional_usage_history, request_logs; 1k rows per + transaction, marker re-check under the account row lock per chunk, no + fold-state lock) for both variants; round-robin at most one nonempty + chunk per pending account per round with a pending re-scan between + rounds. +- [x] 3.1a Pin chunk batch selection to the account-leading indexes + (`account_id` range predicate + index-order ORDER BY → + `idx_usage_account_time`, `ix_additional_usage_distinct_labels`, + covering `idx_logs_account_kind_deleted_latest`), verified against the + production planner; skip re-probing tables observed empty within a + pass; pause between row-touching rounds proportionally to round + duration. +- [x] 3.2 Finalization via `AccountsRepository.delete(only_pending=True)`: + historical transaction shape (identity lock → fold-state lock → + residual rows → mirrors → sticky/rollup/account) plus marker guard and + persisted-variant read. +- [x] 3.3 Leader-gated scheduler (30 s tick, cheap pending pre-check before + leader election, local wake from the delete path), wired into the app + lifespan; post-finalization cache invalidation mirroring the old + synchronous path. + +## 4. Validation + +- [x] 4.1 Integration coverage: chunk-boundary drain (both variants), fold + pass interleaved between chunks (no folded-row resurrection, orphaned + dimension preserves history), restart resume, straggler row settled + mid-drain, repeat-request idempotency without variant escalation, + supersede by replacement (including the drain/finalize race), fast-path + API contract (immediate hide, 404 reactivate, 404 credential exports), + round-robin interleave and mid-pass pickup of newly marked accounts. +- [x] 4.2 Update the existing delete API tests to drive the worker pass; + keep the direct synchronous `AccountsRepository.delete` coverage. +- [x] 4.3 `ruff check` + `ruff format` + architecture checks + focused + account/rollup/migration test suites + strict OpenSpec validation. +- [x] 4.4 Alembic round-trip coverage for + `20260816_000000_add_account_pending_deletion` (parent -> revision -> + downgrade -> guarded upgrade -> head), wired into the PostgreSQL CI + target list; downgrade REFUSES while any deletion is queued (the + marker columns are the queue's only durable state) and the refusal is + covered by the round-trip test. diff --git a/openspec/changes/bound-account-summary-live-tail/context.md b/openspec/changes/bound-account-summary-live-tail/context.md new file mode 100644 index 0000000000..48589fc819 --- /dev/null +++ b/openspec/changes/bound-account-summary-live-tail/context.md @@ -0,0 +1,107 @@ +# Context: measurements behind the 2h fold lag + +All numbers from the reference production deployment (PostgreSQL 18, +2 vCPU), 2026-08-16, read-only. + +## Insert-visibility skew (what the lag must actually cover) + +`requested_at` is assigned inside `RequestLogsRepository.add_log` as +`requested_at or utcnow()`; a repo-wide audit found **no live caller passing +an explicit value** (the parameter exists for tests; the +`limit_warmup` Protocol declares it but its call site does not use it). The +log write happens at stream end and is dated at the write, so the +stream-start-dating threat the original 24h comment guarded against does not +exist — and has not existed since the initial commit. + +Frontier-lag measurement (how far below the running `max(requested_at)` by +insert order — `id` — a row lands at insert): + +| window | rows | p99 | p99.9 | max | +|---|---|---|---|---| +| last 2M rows (~3.6 days) | 1,999,999 | 4.6ms | 46ms | 5.36s | +| full history | 5,992,511 | — | 30ms | **7.89s** | + +This bound covers every insert path (normal, warmup, duplicates), because it +is computed over every row. 2h ≈ 900x the all-history worst case, with room +for replica clock skew, paused VMs, and event-loop stalls far beyond +anything observed. + +Operational bound made explicit by this change: writer-replica wall clocks +(the `utcnow()` used by `add_log`) must stay within one fold lag of the fold +leader's clock, and no insert transaction may stay open that long. This +requirement is not new — it existed at 24h and only its margin changed. All +replicas share one database and one NTP discipline; a clock trailing by two +hours implies TLS/OAuth breakage long before rollup drift, and the DB pool +recycles connections far below the lag. The residual exposure (a 2–24h +trailing clock that the old lag absorbed) is accepted; deployments that +cannot bound clock skew should not shorten further. A hard fence (stamping +`requested_at` from the database clock) would cost the write path's +no-refresh optimization and is left as a follow-up if ever needed. + +Post-insert mutators are fenced independently of the lag: + +- `update_model_for_request` selects only rows strictly above the lifetime + watermark and at/above the hourly watermark, under the fold-state lock. +- Account deletion and duplicate-identity consolidation reassign request + logs under the fold-state lock and mirror folded sums + (`merge_rollups_into`, time-rollup mirrors). + +## Cache invalidation race (generation fence) + +A summary fill that is between its two statements when deletion or +consolidation commits could otherwise store its pre-commit result *after* +the lifecycle clear, serving stale attribution for a full TTL. Fills capture +a generation counter before their first await; `_clear_...` bumps it and +stores are discarded on mismatch. The clear runs synchronously right after +the lifecycle commit (no await between them), so on the single event loop +every store either precedes the commit (wiped by the clear) or observes the +bumped generation. Regression: +`test_summary_cache_fill_discarded_when_invalidated_mid_flight`. + +## Read-path cost of the 24h tail + +- Tail at measurement time: 655,804 of 5,992,447 rows (11%). +- Listing aggregate (`deduped_usage_aggregate_stmt` above the watermark), + production `EXPLAIN (ANALYZE, BUFFERS)`: + - **24h watermark (actual)**: 59.96s cold — the planner abandons the + covering index at ~656k tail rows and degrades to a parallel seq scan + (external-merge sort, 28MB spill) hash-joined against a hash of the + entire 5.99M-row table (46,474kB hash memory, 16 batches, 75k temp + pages). Warm-cache production average for the same call family: 2.1s + over 1,459 calls, max 131s. + - **2h parameter, same query shape**: 63ms — index scan on + `idx_logs_dash_usage_covering` (9,124 rows), hash-agg dedupe, nested + loop over `request_logs_pkey`. + - **1h parameter**: 18ms. +- No index or query-shape change is needed once the tail is bounded; the + existing covering index already serves the bounded range. The residual + per-render cost is then amortized by the 30s summary cache. + +## Upgrade path + +`run_fold_pass` folds toward `now - FOLD_LAG` in bounded `FOLD_SLICE` (7d) +transactions, so the one-time 22h watermark jump after this change lands in +a single ordinary slice; `test_fold_absorbs_widened_watermark_gap` pins the +totals across the jump. The watermark only advances, so a rollback to the +24h constant simply pauses folding until `now - 24h` catches up with the +already-advanced watermark — reads stay correct throughout (rows above the +watermark are always live-tail-served). + +## Retention interaction + +The retention job's raw-prune floor (`watermark - FOLD_LAG`) and freshness +gate (`watermark` within `2 * FOLD_LAG` of now) tighten with the constant. +The floor exists so no rollup is robbed of raw it has not folded and so +concurrent readers holding a slightly older watermark lose nothing; both +need seconds. Consequence for retention-enabled deployments: raw becomes +physically prunable at ~4h age instead of ~48h when the configured retention +period is that short; sub-hour partial-window reads (non-hour-aligned +`since`/`until`) over pruned history hit their documented raw-degrade path +sooner. Hour-aligned reads are rollup-served and unaffected. + +The rollup/retention parity corpus +(`tests/integration/test_request_usage_rollup_parity.py`) was authored +against the 24h lag (TARGET_W at BASE+9d, prune floor at BASE+8d, unaligned +windows and boundary rows placed between them); it now pins +`CORPUS_FOLD_LAG = 24h` explicitly because the parity semantics it proves +are lag-independent. diff --git a/openspec/changes/bound-account-summary-live-tail/proposal.md b/openspec/changes/bound-account-summary-live-tail/proposal.md new file mode 100644 index 0000000000..b164f9cdda --- /dev/null +++ b/openspec/changes/bound-account-summary-live-tail/proposal.md @@ -0,0 +1,68 @@ +# Bound the account-summary live tail: 2h fold lag + short summary TTL cache + +## Why + +The account listing recomputes its request-usage summaries by deduping and +re-aggregating every raw `request_logs` row above the lifetime fold watermark +on each dashboard accounts load. The watermark trails `now` by the fold lag, +so the lag directly sizes that always-rescanned tail. + +The lag has been 24h since the rollup shipped, justified by a premise the +write path never had: the sizing comment (and the fold spec scenario) claim a +log row is *dated at request start but inserted at stream end*, so the lag +must exceed the maximum request duration. In this codebase `requested_at` is +stamped **inside `RequestLogsRepository.add_log` at write time** +(`requested_at or utcnow()`, and no live caller passes an explicit value — +the parameter exists for tests). A row can land below the `requested_at` +frontier only through replica clock skew, the single-row insert transaction's +commit latency, or a process stall. Measured over one full production history +(6.0M rows), the worst insert landed 7.9s below the frontier (p99.9 = 30ms). +Post-insert mutators need no lag allowance either: `update_model_for_request` +skips rows at/below the watermarks, and account consolidation/deletion +reassign logs under the fold-state lock while mirroring the folded sums. + +The oversized lag is expensive: on the reference deployment the 24h tail is +~660k rows (11% of the table), the listing aggregate was measured at 1,459 +calls averaging 2.1s (max 131s), and the cold plan degrades to a full +seq-scan hash join over the whole table (measured 60s). With a 2h bound the +identical query runs in 63ms via the covering-index nested-loop plan. + +Independently, the listing recomputes the summaries on every accounts load +even though the displayed lifetime totals tolerate short staleness — the same +shape the request-log COUNT cache already addresses (issue #1340). + +## What Changes + +- `FOLD_LAG` drops from 24h to 2h. 2h keeps a ~900x margin over the worst + insert-visibility skew ever observed while bounding the raw tail every + account and API-key summary read must re-aggregate. The hourly and + conversation fold targets and the retention min-gate derive from the same + constant and tighten with it; the retention floor's purpose (protect raw + the folds have not consumed and concurrent readers holding a slightly + older watermark) needs seconds, not hours. +- On upgrade, the first fold pass absorbs the 22h watermark jump as ordinary + bounded backfill slices; totals are unchanged (fold moves rows from the + live tail into the persisted sums). +- The fold-lag spec scenario is corrected to state the real invariant + (insert-visibility skew), replacing the false stream-start-dating premise. +- Account request-usage summaries gain a process-local fixed-TTL (30s) cache + keyed by the account-id signature, mirroring the request-log COUNT cache. + Account deletion and duplicate-identity consolidation clear it because they + re-attribute usage rather than merely append; a non-positive TTL bypasses + the cache (the test suite runs with TTL 0). + +No schema change, no new settings, no API change. + +## Impact + +- Affected specs: `query-caching` (fold safety-lag requirement, account + summary read requirement). +- Affected code: `app/modules/accounts/usage_rollup.py` (constant + sizing + rationale), `app/modules/accounts/repository.py` (summary TTL cache + + invalidation hooks), `tests/`. +- Behavior visible to operators: account/API-key summary reads get a 2h raw + tail instead of 24h; listing summaries may be up to 30s stale (matching the + dashboard's 30s poll cadence); with retention enabled, raw rows become + physically prunable once they are one fold lag (now 2h) below the + watermark, so sub-hour partial-window raw reads over pruned history reach + their documented degrade path sooner. diff --git a/openspec/changes/bound-account-summary-live-tail/specs/query-caching/spec.md b/openspec/changes/bound-account-summary-live-tail/specs/query-caching/spec.md new file mode 100644 index 0000000000..8a2555c4fc --- /dev/null +++ b/openspec/changes/bound-account-summary-live-tail/specs/query-caching/spec.md @@ -0,0 +1,91 @@ +# query-caching (delta) + +## MODIFIED Requirements + +### Requirement: Account request usage summaries combine a persistent rollup with a bounded live tail + +Account request-usage summaries MUST NOT aggregate the full `request_logs` history per read. The read MUST combine persisted per-account rollup sums with a live aggregate constrained to rows newer than the rollup watermark, while preserving existing dedupe semantics (latest row id per `(account_id, request_id, requested_at)`) and existing filters (warmup kinds and soft-deleted rows excluded) on the live portion. + +The merged summaries MAY be served from a process-local cache keyed by the requested account-id signature for a small fixed TTL, because the displayed lifetime totals tolerate short staleness. Account deletion and duplicate-identity consolidation MUST clear the cache in the process that performed them (they re-attribute or remove usage rather than append to it). A non-positive TTL MUST bypass the cache entirely so tests and precision-sensitive callers observe exact totals. + +#### Scenario: Summary read does not scan folded history + +- **GIVEN** rollup rows exist with watermark `folded_through = T` +- **WHEN** account request-usage summaries are loaded +- **THEN** the live request-log aggregate MUST constrain to `requested_at > T` +- **AND** the returned totals MUST equal the persisted rollup sums plus the live-tail aggregate per account +- **AND** the cached-input clamp (`cached_input_tokens ≤ input_tokens`) MUST apply to the merged totals + +#### Scenario: Summary before the first fold matches legacy behavior + +- **GIVEN** no rollup rows exist yet +- **WHEN** account request-usage summaries are loaded +- **THEN** the live aggregate MUST cover all non-deleted, non-warmup request-log history +- **AND** the returned totals MUST equal the pre-rollup query results + +#### Scenario: Folding does not change reported totals + +- **GIVEN** a set of request-log rows including duplicate rows sharing `(account_id, request_id, requested_at)` +- **WHEN** a fold pass folds part of that history and summaries are read afterwards +- **THEN** the totals MUST equal the totals the legacy full-history dedupe aggregate would report for the same rows + +#### Scenario: Summary read is snapshot-consistent with a concurrent fold commit + +- **GIVEN** a fold slice may commit at any point during a summary read +- **WHEN** the read fetches rollup sums and the watermark +- **THEN** both MUST come from a single database snapshot (one statement) +- **AND** no qualifying request-log row's contribution may be absent from both the rollup sums and the live-tail aggregate of that read + +#### Scenario: Cached summaries are served within the TTL per signature + +- **GIVEN** a positive summary cache TTL +- **AND** summaries were computed for one account-id signature +- **WHEN** the same signature is requested again within the TTL +- **THEN** the cached summaries MAY be returned without touching the database +- **AND** a different account-id signature MUST NOT be served from that entry + +#### Scenario: Account deletion invalidates cached summaries + +- **GIVEN** cached summaries that include an account +- **WHEN** that account is deleted, or a duplicate-identity consolidation removes it +- **THEN** the cache MUST be cleared so the next read reflects the new attribution +- **AND** a summary computation already in flight when the invalidation happens MUST NOT re-populate the cache with its pre-invalidation result + +### Requirement: A background fold job advances the account usage rollup safely + +A periodic background job MUST fold request-log rows into `account_usage_rollups` and advance the watermark. Folding MUST be restricted to rows older than a safety lag, MUST apply the dedupe and filtering semantics of the summary query within the folded window, MUST run on at most one instance at a time, and MUST be idempotent under repeated or concurrent invocation. + +#### Scenario: Fold boundary respects the safety lag + +- **WHEN** a fold pass runs at time `now` +- **THEN** it MUST NOT fold any row with `requested_at > now − lag` +- **AND** rows younger than the lag remain covered by the live-tail aggregate +- **AND** the lag MUST exceed the maximum possible distance between a row's `requested_at` and the moment its insert becomes visible — `requested_at` is stamped at write time inside the log insert path, so this distance is bounded by replica clock skew, insert-commit latency, and process stalls, not by request duration — because a row landing below the watermark would otherwise vanish from totals +- **AND** post-insert mutations of folded rows MUST NOT rely on the lag: they are fenced by the watermark (skipped below it) or run under the fold-state lock while mirroring the folded sums + +#### Scenario: Widening the lag gap is absorbed as ordinary backfill + +- **GIVEN** a deployment whose persisted watermark trails `now − lag` by more than one fold cadence (for example after the lag constant is shortened) +- **WHEN** the next fold passes run +- **THEN** the gap MUST be folded in the bounded backfill slices with reported totals unchanged + +#### Scenario: Duplicate rows never split across the fold boundary + +- **GIVEN** duplicate request-log rows sharing the same `(account_id, request_id, requested_at)` +- **WHEN** a fold pass selects its window by `requested_at` +- **THEN** all rows of the duplicate group MUST land on the same side of the boundary +- **AND** only the latest row id of the group MUST contribute to the folded sums + +#### Scenario: Fold is idempotent and single-writer + +- **GIVEN** a fold pass has committed sums through watermark `T` +- **WHEN** another fold pass runs for the same window (repeat invocation or a second instance) +- **THEN** it MUST observe watermark `T` inside its transaction and fold no row at or before `T` +- **AND** no request-log row's contribution appears twice in the rollup + +#### Scenario: Historical backfill is sliced and non-blocking + +- **GIVEN** a deployment with existing request-log history and no rollup rows +- **WHEN** the first fold passes run +- **THEN** history MUST be folded in bounded time slices, each committed in its own transaction +- **AND** summary reads issued during backfill MUST return correct totals (rollup so far plus remaining live tail) diff --git a/openspec/changes/bound-account-summary-live-tail/tasks.md b/openspec/changes/bound-account-summary-live-tail/tasks.md new file mode 100644 index 0000000000..e27a34be4b --- /dev/null +++ b/openspec/changes/bound-account-summary-live-tail/tasks.md @@ -0,0 +1,42 @@ +# Tasks + +## 1. Fold lag + +- [x] 1.1 Shorten `FOLD_LAG` to 2h and rewrite the sizing comment around the + actual invariant: `requested_at` is stamped at insert time inside + `add_log`, so the lag bounds insert-visibility skew (clock skew + + commit latency + stalls), not request duration; note the measured + production worst case (7.9s over 6.0M rows) and the fenced post-insert + mutators +- [x] 1.2 Verify the watermark jump after the lag change is absorbed by the + ordinary backfill path with totals unchanged (regression test + `test_fold_absorbs_widened_watermark_gap`) + +## 2. Summary TTL cache + +- [x] 2.1 Cache `list_request_usage_summary_by_account` results per + account-id signature for a fixed 30s TTL with a bounded entry count, + mirroring the request-log COUNT cache; non-positive TTL bypasses +- [x] 2.2 Clear the cache on account deletion and on duplicate-identity + consolidation (both re-attribute usage), alongside the existing + `_clear_bulk_history_since_sqlite_cache()` call sites +- [x] 2.3 Zero the TTL for the test suite via an autouse conftest fixture so + summaries stay exact within a test (same pattern as the COUNT cache) + +## 3. Tests + +- [x] 3.1 Cache behavior: staleness within TTL, per-signature keying, + invalidation on delete and on consolidation +- [x] 3.2 Re-anchor the rollup/retention parity corpus on an explicit + `CORPUS_FOLD_LAG = 24h` pin — its 10-day geometry (TARGET_W, prune + floor, unaligned windows) was authored against the old lag and the + parity semantics are lag-independent +- [x] 3.3 Fix the lag-coupled backdated insert in + `test_account_delete_removes_rollup_row` (`now - FOLD_LAG / 2`) +- [x] 3.4 `uv run pytest`, `uv run ruff check`, `uv run ruff format --check` + +## 4. Spec + +- [x] 4.1 Correct the fold safety-lag scenario in `query-caching` and add the + summary-cache allowance to the account summary requirement +- [x] 4.2 `openspec validate bound-account-summary-live-tail --strict` diff --git a/openspec/changes/bound-sqlite-wedged-teardown/proposal.md b/openspec/changes/bound-sqlite-wedged-teardown/proposal.md new file mode 100644 index 0000000000..f6a1e4dfd6 --- /dev/null +++ b/openspec/changes/bound-sqlite-wedged-teardown/proposal.md @@ -0,0 +1,21 @@ +## Why + +Issue #1682, part 2 of the plan. Session teardown shields rollback/close unboundedly (`app/db/session.py`), so a wedged teardown pins SQLite's single writer slot with nothing to reclaim it: every writer — including the `scheduler_leader` INSERT that would re-establish leadership — surfaces `database is locked` until the wedge spontaneously resolves (~17 minutes in the report). Part 1 (`report-sqlite-long-write-holders`) made the holder attributable; the teardown itself must now be bounded. Crucially, abandoning the wedged await alone releases nothing — the aiosqlite worker thread still holds the lock — so the bound must come with reclaiming the connection. + +## What Changes + +- The shielded rollback/close teardown gets a hard deadline on file-backed SQLite (one-sixth of the busy timeout, 5s — reclaimed well before other writers exhaust their 30s busy timeout). PostgreSQL teardown semantics are untouched, and in-memory SQLite keeps the unbounded path: its one shared StaticPool connection is the whole database (invalidation would destroy it) and cannot starve other writers. +- A teardown that misses the deadline is reclaimed, not merely abandoned: the driver connection is interrupted (aborting the C-level call the aiosqlite worker is stuck in) and the connection is invalidated — terminating it at the pool disposes the worker and hard-closes the underlying `sqlite3` connection, which releases the writer slot and guarantees the connection is never handed out again. +- The reclaim report carries part 1's watchdog identifiers (held duration, owning task, first/last write statements), including when the watchdog had already deferred them into its pending report because the wedge is inside the transaction-ending call itself — invalidation would otherwise suppress that deferred report. +- A wedged session is fenced: later teardown attempts return immediately instead of driving the session concurrently with the abandoned work, and once the abandoned teardown finishes late the session is closed for bookkeeping — a deferred task owned until completion and drained at `close_db`, never fire-and-forget. +- No new settings: the deadline derives from the existing busy timeout. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `database-backends`: a wedged SQLite session teardown is bounded and its connection reclaimed so the writer slot is released. diff --git a/openspec/changes/bound-sqlite-wedged-teardown/specs/database-backends/spec.md b/openspec/changes/bound-sqlite-wedged-teardown/specs/database-backends/spec.md new file mode 100644 index 0000000000..b234160df7 --- /dev/null +++ b/openspec/changes/bound-sqlite-wedged-teardown/specs/database-backends/spec.md @@ -0,0 +1,40 @@ +## ADDED Requirements + +### Requirement: Wedged SQLite session teardown is bounded and reclaimed + +Session teardown (rollback and close) on file-backed SQLite MUST complete within a hard deadline derived from the busy timeout while remaining shielded from the caller's cancellation. A teardown that misses the deadline MUST NOT be merely abandoned — the aiosqlite worker thread would keep holding the writer slot — it MUST be reclaimed: the driver connection is interrupted to abort the call the worker is stuck in, and the connection is invalidated so the worker is disposed, the underlying `sqlite3` connection is hard-closed releasing the writer slot, and the connection can never be handed out again. The reclaim MUST be reported with the long-write watchdog's identifiers where available (held duration, owning task, first and last write statements), including identifiers the watchdog already deferred into its pending report. A session whose teardown was abandoned MUST be fenced from further teardown attempts, the abandoned work finishing late MUST NOT surface unretrieved errors, and the deferred bookkeeping close MUST be owned until completion (drained at database shutdown, never fire-and-forget). PostgreSQL teardown semantics MUST remain unchanged, and in-memory SQLite — whose single shared connection is the entire database and cannot starve other writers — MUST keep the unbounded teardown and never be reclaimed. + +#### Scenario: A wedged rollback no longer starves every other writer + +- **GIVEN** a session holding an open SQLite write transaction whose rollback wedges during teardown +- **WHEN** the teardown deadline passes +- **THEN** teardown returns, the connection is interrupted and invalidated, and another writer — such as the leader-election `scheduler_leader` INSERT — acquires the writer slot immediately instead of surfacing `database is locked` + +#### Scenario: The reclaim is attributed with the watchdog's identifiers + +- **GIVEN** a wedged teardown whose transaction ran write statements tracked by the long-write watchdog +- **WHEN** the connection is reclaimed +- **THEN** the report names the held duration, owning task, and first/last write statements, even though invalidation prevents the watchdog's own deferred report from firing + +#### Scenario: A wedged session cannot be driven concurrently + +- **GIVEN** a session whose teardown was abandoned as wedged +- **WHEN** teardown is attempted again +- **THEN** it returns immediately, and the session is closed for bookkeeping only after the abandoned teardown finishes late + +#### Scenario: PostgreSQL teardown is untouched + +- **GIVEN** a session bound to a non-SQLite dialect +- **WHEN** its rollback or close outlives the SQLite deadline +- **THEN** the teardown still awaits completion unboundedly and no connection is reclaimed + +#### Scenario: The shared in-memory SQLite connection is never reclaimed + +- **GIVEN** a session bound to an in-memory SQLite database, whose one shared connection is the entire database +- **WHEN** its teardown outlives the deadline +- **THEN** the teardown still awaits completion unboundedly and the connection is never invalidated, preserving schema and data for later sessions + +#### Scenario: The bound never abandons healthy teardown + +- **WHEN** rollback and close complete within the deadline +- **THEN** teardown behaves exactly as before, including re-raising the completed call's exception to the existing swallow points diff --git a/openspec/changes/bound-sqlite-wedged-teardown/tasks.md b/openspec/changes/bound-sqlite-wedged-teardown/tasks.md new file mode 100644 index 0000000000..0ccedd5515 --- /dev/null +++ b/openspec/changes/bound-sqlite-wedged-teardown/tasks.md @@ -0,0 +1,14 @@ +## 1. Bounded teardown and reclaim + +- [x] 1.1 Bound the shielded rollback/close teardown for file-backed SQLite sessions with a deadline derived from the busy timeout, preserving the shield against caller cancellation; PostgreSQL and in-memory SQLite (one shared StaticPool connection is the whole database — reclaim would destroy it, and it cannot starve other writers) keep the unbounded path +- [x] 1.2 On a missed deadline, interrupt the driver connection and invalidate it so the aiosqlite worker is disposed, the writer slot is released, and the connection can never be handed out again; report the reclaim with the long-write watchdog's identifiers (including ones already deferred into its pending report) +- [x] 1.3 Fence the wedged session against further teardown, consume the abandoned task's late failure, and close the session for bookkeeping once the abandoned teardown finishes; own the deferred close until completion and drain it at close_db so shutdown cannot abandon it + +## 2. Tests + +- [x] 2.1 A wedged sqlite rollback: close_session returns within the bound (fails on pre-fix unbounded teardown), the driver is interrupted, the connection invalidated, the reclaim log carries the watchdog identifiers, and an independent writer succeeds immediately while the wedge is still pending +- [x] 2.2 A wedged session is fenced from further teardown; the abandoned teardown finishing late is observed and followed by the bookkeeping close +- [x] 2.3 The bounded shield completes fast work, abandons at the deadline without cancelling, and absorbs caller cancellation like the unbounded shield +- [x] 2.4 Non-sqlite sessions keep the unbounded teardown: a slow rollback/close beyond the sqlite bound still runs to completion and is never reclaimed +- [x] 2.5 A wedged close without a transaction is bounded and fenced too +- [x] 2.6 In-memory SQLite keeps the unbounded teardown: a slow teardown is never reclaimed, the shared connection is never invalidated, and schema/data survive for later sessions diff --git a/openspec/changes/cap-projection-history-rows/proposal.md b/openspec/changes/cap-projection-history-rows/proposal.md new file mode 100644 index 0000000000..7ef6c21f52 --- /dev/null +++ b/openspec/changes/cap-projection-history-rows/proposal.md @@ -0,0 +1,58 @@ +## Why + +On the reference PostgreSQL deployment the dashboard projections bulk +usage-history read returns ~307k rows per call (540 calls over 10 days, +avg 2.54 s, max 56 s, `usage_history` at ~3M rows / 2.8 GB). Live snapshot +ingestion appends usage rows per proxied request, so one busy account's +7-day secondary window holds tens of thousands of rows — yet the projection +consumers only read the recent tail: EWMA depletion/burn rates decay a +sample's contribution by 0.6^n within a few dozen newer samples, the +weekly-pace recent-burn window is 6 hours, and the pace smoothing mean is at +most 240 minutes. Fetching every in-window row burns database reads, row +transfer, and Python row-building for values no consumer can observe. + +## What Changes + +- Bound the PostgreSQL projections bulk usage-history read to each + account's newest rows (newest-first per-account row cap) inside the + existing per-account cutoffs, via one lateral top-N probe per account + over the existing covering indexes (backward index-only scan that stops + at the cap or cutoff). +- Exempt rows inside the configured pace-smoothing window from the cap + (uncapped recent floor supplied by the projections caller): ingestion + writes per proxied request whenever the usage fingerprint changes, so a + write burst could out-write any fixed cap inside the smoothing window, + and the smoothing mean weighs every in-window sample equally. The probe + splits into two disjoint branches over the same covering index — the + time-bounded floor branch returns in full, the cap bounds the older + remainder. +- The dashboard projections caller supplies the cap, sized so the + tail-weighted consumers are unchanged (covers the 6-hour recent-burn + EWMA window at the ingestor's 5-second per-account write throttle + floor; EWMA contributions decay by 0.6^n well inside the cap). +- SQLite keeps its shared-floor snapshot cache and ignores the cap, the + same way it ignores per-account cutoffs. +- No schema change: the existing covering indexes already serve the capped + probes index-only. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `query-caching`: the projections history bulk read MUST additionally + bound each account's slice to its newest in-cutoff rows on PostgreSQL, + MUST exempt rows inside the configured pace-smoothing window from the + cap, and under-cap accounts MUST return slices equal to the shared-floor + fetch after per-account trimming. + +## Impact + +`app/modules/usage/repository.py` (capped PostgreSQL fetch shape), +`app/modules/dashboard/repository.py` / `app/modules/dashboard/service.py` +(cap plumbed from the projections caller), repository/plan/unit regression +coverage. No API, response-schema, setting, migration, or dashboard UI +change. diff --git a/openspec/changes/cap-projection-history-rows/specs/query-caching/spec.md b/openspec/changes/cap-projection-history-rows/specs/query-caching/spec.md new file mode 100644 index 0000000000..b3d0767008 --- /dev/null +++ b/openspec/changes/cap-projection-history-rows/specs/query-caching/spec.md @@ -0,0 +1,54 @@ +# query-caching Delta + +## MODIFIED Requirements + +### Requirement: Projection history reads are bounded per account +The dashboard projections history fetch MUST NOT widen every account's +lookback to the widest account window. On PostgreSQL the bulk usage-history +read MUST bound rows per account by that account's own window cutoff, and +MUST additionally bound each account's slice to a newest-first per-account +row cap supplied by the projections caller. Because live snapshot ingestion +writes a row per proxied request whenever the usage fingerprint changes, no +fixed row cap alone can guarantee coverage of a fixed time window; the +fetch MUST therefore exempt rows inside the configured pace-smoothing +window (the projections caller supplies its start as an uncapped recent +floor) so every row the equal-weight smoothing mean consumes is returned +regardless of write density, while the cap MUST still bound the rows older +than that floor. The cap MUST be sized to cover the remaining tail-weighted +consumers' lookback (the recent-burn EWMA window) at the ingestor's minimum +per-account write interval. Returned slices MUST keep the newest in-cutoff +rows and MUST remain ordered oldest-first. For accounts whose in-cutoff +rows do not exceed the cap, the returned histories MUST equal the previous +shared-floor fetch after the existing per-account trimming; for accounts +over the cap, the returned history MUST be exactly the union of every +in-cutoff row at or after the uncapped recent floor and the newest cap-many +in-cutoff rows older than the floor. + +#### Scenario: One weekly account does not widen the fetch for short-window accounts +- **GIVEN** one account with a 7-day window and several accounts with 5-hour windows +- **WHEN** the projections history fetch runs on PostgreSQL +- **THEN** rows for the 5-hour accounts MUST be bounded by their own cutoff in SQL +- **AND** each account's resulting history slice MUST equal the slice the shared-floor fetch produced after per-account trimming + +#### Scenario: A dense account returns only its newest rows +- **GIVEN** an account whose in-cutoff usage-history rows exceed the per-account row cap +- **WHEN** the projections history fetch runs on PostgreSQL +- **THEN** the account's slice MUST be exactly the in-cutoff rows at or after the uncapped recent floor plus the newest cap-many in-cutoff rows older than the floor, ordered oldest-first +- **AND** accounts whose in-cutoff rows do not exceed the cap MUST return their full trimmed slice unchanged + +#### Scenario: A write burst inside the smoothing window is never truncated +- **GIVEN** an account that wrote more usage-history rows inside the configured pace-smoothing window than the per-account row cap +- **WHEN** the projections history fetch runs on PostgreSQL +- **THEN** every in-cutoff row at or after the smoothing-window start MUST be returned +- **AND** the weekly-pace smoothed values MUST equal the values the uncapped fetch would produce + +#### Scenario: Capped probes stay index-only +- **GIVEN** usage history rows for multiple accounts and a populated visibility map +- **WHEN** the capped per-account probe shape is EXPLAINed on PostgreSQL with sequential and bitmap scans disabled +- **THEN** the plan MUST serve each probe as an Index Only Scan over the covering indexes with no sequential scan of `usage_history` + +#### Scenario: SQLite snapshot cache keeps the shared floor +- **GIVEN** the SQLite backend serves the projections history fetch through its snapshot cache +- **WHEN** per-account cutoffs and a per-account row cap are supplied +- **THEN** the SQLite read MAY keep the shared floor and MAY ignore the row cap +- **AND** per-account trimming in the caller MUST still bound each account's slice diff --git a/openspec/changes/cap-projection-history-rows/tasks.md b/openspec/changes/cap-projection-history-rows/tasks.md new file mode 100644 index 0000000000..ba91d9aa84 --- /dev/null +++ b/openspec/changes/cap-projection-history-rows/tasks.md @@ -0,0 +1,27 @@ +## 1. Implementation + +- [x] 1.1 Add a newest-first per-account row cap to the PostgreSQL bulk + usage-history read (lateral top-N probe per account, composed with the + existing per-account cutoffs; oldest-first slices preserved). +- [x] 1.2 Pass the cap from the dashboard projections history fetch, sized + so the tail-weighted EWMA consumers (depletion, weekly-pace burn) see + identical inputs. +- [x] 1.3 Exempt the configured pace-smoothing window from the cap + (uncapped recent floor plumbed from the projections caller; disjoint + floor + capped-tail branches in the lateral probe) so a per-request + write burst can never truncate the equal-weight smoothing mean. +- [x] 1.4 Keep the SQLite snapshot-cache path on the shared floor (cap + ignored, like cutoffs). + +## 2. Validation + +- [x] 2.1 Regression: capped slices equal the newest rows of the uncapped + fetch, compose with per-account cutoffs, leave under-cap accounts + untouched, and never drop rows at or after the uncapped recent floor; + SQLite ignores the cap. +- [x] 2.2 PostgreSQL plan tests: the capped lateral probes (with and + without the floor branch) stay index-only on the covering indexes. +- [x] 2.3 Unit test: the projections fetch supplies the cap and the + smoothing-window floor. +- [x] 2.4 Run lint, type checks, sqlite + PostgreSQL test slices, and strict + OpenSpec validation. diff --git a/openspec/changes/clarify-routing-quota-help-copy/proposal.md b/openspec/changes/clarify-routing-quota-help-copy/proposal.md new file mode 100644 index 0000000000..d2f950adb1 --- /dev/null +++ b/openspec/changes/clarify-routing-quota-help-copy/proposal.md @@ -0,0 +1,52 @@ +## Why + +Issue #1708: the dashboard collapses several routing concepts into similar +wording, so operators cannot tell what codex-lb will do during a failure +without reading source code. `Sticky threads` sounds like it controls all +affinity (it does not control hard Codex continuation ownership), the sticky +thresholds are percent **used** while account pages show percent +**remaining**, primary/secondary quota windows are never named, and +`Prefer earlier reset` and `Limit warm-up` do not say what they change. +`Active` also reads as "will serve the next request" even though eligibility +is decided per request. + +## What Changes + +- `Sticky threads` copy states it is a soft preference and adds a note that + hard Codex continuation affinity (turn state, previous responses, uploaded + files) is not disabled by the toggle. +- A `Primary vs secondary quota` explainer names the 5-hour and + weekly/monthly windows and states the used-vs-remaining unit split. +- Sticky threshold descriptions name the window and unit (percent used) and + render a live `X% used · equivalent to Y% remaining` hint whose two values + always sum to 100. +- `Prefer earlier reset` copy describes the actual selection behavior + (earliest reset bucket of the selected window, day-bucketed weekly + comparison, applied under capacity weighted, usage weighted, and fill + first). +- `Limit warm-up` copy states that a probe is one small real request using + the configured model/prompt and consumes a small amount of quota. +- The `Active` status badge on the accounts list carries a hint that the + displayed status is not per-request eligibility. +- `docs/routing.md` gains a routing/quotas/eligibility explainer section. +- i18n keys added/updated for `en`, `ko`, and `zh-CN`. + +Deferred (needs selector plumbing, out of scope here): per-account +"why wasn't this account selected" inspection and an actionable +`No available accounts` breakdown. + +## Capabilities + +### New Capabilities + +- None + +### Modified Capabilities + +- `frontend-architecture`: routing/quota help copy, threshold unit hints, and + the Active-status eligibility hint. + +## Impact + +Dashboard SPA copy and help text plus `docs/routing.md`; i18n +(`en`/`ko`/`zh-CN`). No API, database, proxy, or routing behavior changes. diff --git a/openspec/changes/clarify-routing-quota-help-copy/specs/frontend-architecture/spec.md b/openspec/changes/clarify-routing-quota-help-copy/specs/frontend-architecture/spec.md new file mode 100644 index 0000000000..e28fa8c70f --- /dev/null +++ b/openspec/changes/clarify-routing-quota-help-copy/specs/frontend-architecture/spec.md @@ -0,0 +1,76 @@ +MUST make routing, sticky affinity, quota thresholds, warm-up, and account +eligibility understandable from dashboard copy alone. + +## ADDED Requirements + +### Requirement: Sticky-threads copy distinguishes soft routing from hard continuation affinity + +The routing settings SHALL describe `Sticky threads` as a soft preference and +SHALL state that disabling it does not disable hard Codex continuation +affinity for requests that carry continuation state. + +#### Scenario: Sticky threads help copy + +- **WHEN** the routing settings section renders +- **THEN** the sticky-threads description identifies the toggle as a soft preference +- **AND** an adjacent note states that hard Codex continuation affinity is not disabled by the toggle + +### Requirement: Sticky thresholds are presented in percent used with a remaining equivalent + +The sticky reallocation threshold controls SHALL name the quota window they +apply to, SHALL state that the value is percent used, and SHALL show the +quota-arithmetic remaining percent for a valid threshold value, qualified so +it is not presented as the exact account-page remaining figure (routing adds +temporary in-flight pressure on top of reported usage). + +#### Scenario: Threshold unit hint + +- **GIVEN** the sticky secondary threshold input holds the valid value `70` +- **WHEN** the routing settings section renders +- **THEN** a hint shows `70% used` alongside `30% remaining` in quota terms +- **AND** the quota-window explainer states that in-flight work counts as temporary extra usage + +#### Scenario: Quota window explainer + +- **WHEN** the routing settings section renders +- **THEN** an explainer identifies primary quota as the 5-hour window and secondary quota as the longer weekly window (monthly on plans without a weekly window) +- **AND** it states that account pages show percent remaining while the thresholds are percent used + +### Requirement: Prefer-earlier-reset and limit warm-up copy describe actual behavior + +The routing settings SHALL describe `Prefer earlier reset` as preferring +otherwise-eligible accounts whose selected quota window resets sooner, and +SHALL describe limit warm-up as sending one small probe request that consumes +a small amount of quota when an opted-in account's quota window is confirmed +to have newly reset. + +#### Scenario: Prefer earlier reset help copy + +- **WHEN** the routing settings section renders +- **THEN** the prefer-earlier-reset description says selection prefers accounts whose selected quota window resets sooner +- **AND** it names the strategies the preference applies to (capacity weighted, usage weighted, and fill first) + +#### Scenario: Limit warm-up help copy + +- **WHEN** the routing settings section renders +- **THEN** the limit warm-up description says a probe is sent when an opted-in account's quota window is confirmed to have newly reset +- **AND** it states that probes are real requests and consume a small amount of quota + +### Requirement: Active status is presented as displayed status, not per-request eligibility + +The accounts list SHALL present a visible note that displayed status does not +guarantee per-request eligibility, and SHALL annotate active rows and their +status badges with the same hint for pointer and assistive-technology users. + +#### Scenario: Accounts list eligibility note + +- **GIVEN** the accounts list contains at least one account +- **WHEN** the list renders +- **THEN** a visible note states that individual requests can still skip an `Active` account + +#### Scenario: Active badge eligibility hint + +- **GIVEN** an account whose status is `active` +- **WHEN** its accounts-list entry renders +- **THEN** the focusable account row and the status badge carry the hint as a native tooltip and accessible description +- **AND** non-active statuses do not carry that hint diff --git a/openspec/changes/clarify-routing-quota-help-copy/tasks.md b/openspec/changes/clarify-routing-quota-help-copy/tasks.md new file mode 100644 index 0000000000..a1462cf639 --- /dev/null +++ b/openspec/changes/clarify-routing-quota-help-copy/tasks.md @@ -0,0 +1,17 @@ +## 1. Routing settings copy + +- [x] 1.1 Sticky-threads soft-preference copy plus hard-affinity note +- [x] 1.2 Primary-vs-secondary quota explainer block +- [x] 1.3 Threshold descriptions in percent used with live remaining hint +- [x] 1.4 Prefer-earlier-reset and limit warm-up behavior descriptions +- [x] 1.5 Add `en` / `ko` / `zh-CN` keys for the new copy + +## 2. Accounts surface + +- [x] 2.1 Active-status eligibility hint on the account list badge + +## 3. Docs and validation + +- [x] 3.1 Extend `docs/routing.md` with the routing/quotas/eligibility explainer +- [x] 3.2 Unit tests for the new copy, hint computation, and badge hint +- [x] 3.3 `openspec validate clarify-routing-quota-help-copy --strict` diff --git a/openspec/changes/classify-bridge-recovery-error-frames/.openspec.yaml b/openspec/changes/classify-bridge-recovery-error-frames/.openspec.yaml new file mode 100644 index 0000000000..f774115be7 --- /dev/null +++ b/openspec/changes/classify-bridge-recovery-error-frames/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-20 diff --git a/openspec/changes/classify-bridge-recovery-error-frames/proposal.md b/openspec/changes/classify-bridge-recovery-error-frames/proposal.md new file mode 100644 index 0000000000..588cf25415 --- /dev/null +++ b/openspec/changes/classify-bridge-recovery-error-frames/proposal.md @@ -0,0 +1,29 @@ +# Classify Bridge Recovery Error Frames + +## Why + +The HTTP responses session bridge can wedge a session permanently (issue #1830). After one genuine mid-turn interruption the bridge rebinds to its stored durable anchor and re-injects it on every attempt. When upstream rejects that anchor with a classifiable previous-response error, two gaps keep the session unrecoverable: + +1. The bridge-local recovery gate reads raw error codes without the normalization the WebSocket path gained in the `classify-invalid-previous-response-id` change (#1818): a frame that carries its classifiable code only in `type`, or the terse parameterless ``Invalid `previous_response_id`.`` shape, falls through to the ambiguous-transport class instead of previous-response recovery. +2. Anchor poisoning only counts `stream_idle_timeout` failures, and only on the reader path when admission waiters exist. The wedge observed in production fails eventlessly with `stream_incomplete` (the bridge's masked form of an upstream previous-response rejection), so the retry circuit opens and cools down forever while `http_responses_session_bridge_anchor_poison_failure_threshold` never fires. Operators had to wipe the `http_bridge_*` tables to free sessions. + +## What Changes + +- Route the bridge-local previous-response recovery gate through the same error-code normalization as the WebSocket rewrite path (code falls back to `type`; the terse parameterless invalid-previous-response shape classifies as a continuity miss). +- Count both ambiguous eventless transport classes — `stream_incomplete` and `stream_idle_timeout` (with its aliased diagnostics) — toward anchor poison, so consecutive same-anchor failures self-heal even when the frame is genuinely unclassifiable. `clean_close` still never poisons. +- Evaluate anchor poison at the shared retirement boundary as well, so a wedged anchored session that fails without admission waiters also clears its poisoned durable anchor once the threshold is reached. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: Normalize error frames at the bridge-local recovery gate and widen anchor poisoning to all consecutive eventless same-anchor failures, including the waiterless retirement path. + +## Impact + +- HTTP bridge recovery gate (`app/modules/proxy/_service/http_bridge/helpers.py`), anchor-poison accounting (`app/modules/proxy/_service/http_bridge/upstream_events.py`, `app/modules/proxy/_service/http_bridge/request_submit.py`, `app/modules/proxy/_service/http_bridge/retry_circuit.py`). +- No API, schema, migration, dependency, configuration, or dashboard changes; the existing poison threshold setting and its default of seven are unchanged. diff --git a/openspec/changes/classify-bridge-recovery-error-frames/specs/responses-api-compat/spec.md b/openspec/changes/classify-bridge-recovery-error-frames/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..17b86e6a4b --- /dev/null +++ b/openspec/changes/classify-bridge-recovery-error-frames/specs/responses-api-compat/spec.md @@ -0,0 +1,74 @@ +# responses-api-compat Delta + +## ADDED Requirements + +### Requirement: Bridge-local previous-response recovery classifies normalized error frames + +When the HTTP bridge evaluates whether a failed anchored request may enter bridge-local previous-response recovery, it MUST classify the error frame with the same normalization as the WebSocket rewrite path: a missing or empty `code` MUST fall back to the error `type` before classification, and the parameterless ``Invalid `previous_response_id`.`` invalid-request shape MUST classify as a previous-response continuity miss. A classifiable previous-response rejection MUST route into previous-response recovery and MUST NOT be treated as an ambiguous transport failure that only feeds the retry-circuit cooldown. + +#### Scenario: Terse parameterless rejection enters local recovery + +- **GIVEN** an anchored HTTP bridge request fails with `type = "invalid_request_error"`, no `code`, no `param`, and the message ``Invalid `previous_response_id`.`` +- **WHEN** the bridge evaluates bridge-local previous-response recovery for that failure +- **THEN** the failure classifies as a previous-response continuity miss +- **AND** the bridge attempts previous-response recovery instead of the ambiguous-transport path + +#### Scenario: Code carried only in the error type classifies + +- **GIVEN** an anchored HTTP bridge request fails with no `code` and `type = "previous_response_not_found"` +- **WHEN** the bridge evaluates bridge-local previous-response recovery for that failure +- **THEN** the failure enters previous-response recovery instead of the ambiguous-transport class + +#### Scenario: Unrelated errors keep their classification + +- **WHEN** a failed anchored request carries an error whose normalized code, param, and message do not match a previous-response continuity miss +- **THEN** the bridge MUST NOT classify it as a previous-response continuity miss + +## MODIFIED Requirements + +### Requirement: Repeated zero-event idle failures poison dead anchors + +For hard HTTP bridge keys, repeated zero-event failures MUST use the existing durable retry-circuit counter to identify an anchor that should no longer remain addressable; the counter resets on a completed response, so a run of consecutive failures proves the anchor never advanced. Both ambiguous eventless transport classes — `stream_idle_timeout` (including its aliased diagnostics) and `stream_incomplete` — MUST be able to trigger anchor poisoning at the threshold; a `clean_close` outcome MUST NOT itself trigger anchor poisoning. When an eligible eventless failure reaches the configured poison threshold for the same hard bridge key, the proxy MUST abandon durable continuity for that session and retire the bridge even when admission waiters exist, and the shared retirement boundary MUST clear the poisoned durable anchor even when no admission waiter exists, while the session still owns its durable lease. If the clear cannot be confirmed on the waiterless retirement path, the proxy MUST re-attempt it when a later eligible eventless failure at or above the threshold retires the session. The default threshold MUST be no greater than seven failures. + +#### Scenario: Admission waiters cannot defer anchor poisoning forever + +- **GIVEN** a hard durable bridge key has admission waiters +- **AND** repeated zero-event idle failures for that same key reach the poison + threshold +- **WHEN** the reader failure path would normally defer retirement for the + admission waiter +- **THEN** the proxy clears the durable continuity anchors +- **AND** retires the session despite the admission waiter +- **AND** the next attach starts from fresh durable state rather than the + poisoned previous-response anchor + +#### Scenario: Repeated eventless stream_incomplete failures poison the anchor + +- **GIVEN** a hard durable bridge key has a stored durable anchor +- **AND** every anchored attempt fails eventlessly with `stream_incomplete` (for example a masked upstream previous-response rejection) +- **WHEN** consecutive failures for that key reach the poison threshold +- **THEN** the proxy clears the durable continuity anchors under the session's owner epoch +- **AND** the next attach starts from fresh durable state instead of looping through retry-circuit cooldown + +#### Scenario: Waiterless retirement poisons the anchor at the threshold + +- **GIVEN** a hard durable bridge key fails eventlessly with no admission waiters +- **WHEN** the shared retirement boundary records the eventless failure that reaches the poison threshold +- **THEN** the proxy clears the durable continuity anchors before releasing the durable lease + +#### Scenario: Failed waiterless clear is re-attempted on the next threshold failure + +- **GIVEN** the waiterless retirement path reached the poison threshold but the durable continuity clear could not be confirmed +- **WHEN** the next eligible eventless failure for the same key retires the session +- **THEN** the proxy re-attempts the durable continuity clear under the new session's owner epoch + +#### Scenario: Clean closes never trigger anchor poisoning + +- **WHEN** a `clean_close` retry-circuit outcome is recorded for a hard bridge key, at any consecutive-failure count +- **THEN** that outcome does not clear the durable continuity anchors + +#### Scenario: Lease liveness comparison is timezone-safe +- **GIVEN** a durable bridge session whose `lease_expires_at` was read from a `timestamptz` column (offset-aware) on PostgreSQL +- **WHEN** the dead-owner classifier evaluates lease liveness against the application's naive-UTC clock +- **THEN** both timestamps MUST be normalized to naive UTC before comparison +- **AND** the anchored-lookup path MUST NOT raise on mixed-awareness datetimes diff --git a/openspec/changes/classify-bridge-recovery-error-frames/tasks.md b/openspec/changes/classify-bridge-recovery-error-frames/tasks.md new file mode 100644 index 0000000000..7934025565 --- /dev/null +++ b/openspec/changes/classify-bridge-recovery-error-frames/tasks.md @@ -0,0 +1,21 @@ +# Tasks + +## 1. Regression Coverage + +- [x] 1.1 Add gate regressions for the terse parameterless ``Invalid `previous_response_id`.`` frame and a frame carrying the classifiable code only in `type`, verifying both misclassify (no recovery) before the fix. +- [x] 1.2 Add anchor-poison regressions: consecutive eventless `stream_incomplete` reader failures with an admission waiter, and consecutive eventless failures through the shared retirement boundary without waiters, verifying neither poisons the anchor before the fix. + +## 2. Classifier Routing + +- [x] 2.1 Normalize the error code (falling back to `type`) in the bridge-local previous-response recovery gate before all classification checks, matching the WebSocket rewrite path from `classify-invalid-previous-response-id`. + +## 3. Anchor Poison Counting + +- [x] 3.1 Map both ambiguous eventless retry-circuit classes (`stream_incomplete`, `stream_idle_timeout` and its aliases) to anchor-poison details; keep `clean_close` excluded. +- [x] 3.2 Widen the deferred reader-path poison branch to both classes and thread the poison detail into the poisoned-anchor observability events. +- [x] 3.3 Evaluate the poison threshold at the shared retirement boundary and clear the poisoned durable anchor while the session still owns its durable lease. + +## 4. Verification + +- [x] 4.1 Run the touched bridge unit and integration suites, ruff, and type checks. +- [x] 4.2 Run strict OpenSpec validation for this change and review the final diff for unrelated changes. diff --git a/openspec/changes/close-nonstream-chat-collect/context.md b/openspec/changes/close-nonstream-chat-collect/context.md new file mode 100644 index 0000000000..753ce20938 --- /dev/null +++ b/openspec/changes/close-nonstream-chat-collect/context.md @@ -0,0 +1,28 @@ +# Close non-stream Chat Completions collect + +## Purpose + +Keep non-streaming `/v1/chat/completions` on the same settlement and error-status +contract as `/v1/responses` collect. + +## Decision + +Hold a reference to the `stream_responses` generator and `_aclose_stream` it in +`finally` after collect. The startup probe already closes the generator when it +sees an error event; this covers the probe-timeout path that then splits the +stream with `__anext__` + `_prepend_first`. + +Reuse `_mask_previous_response_not_found_error` so status mapping cannot drift +from Responses. + +## Failure mode + +If collect returns on the prepended first `response.failed` event, `_prepend_first` +never enters `async for` on the live generator, so Python does not aclose it. +Reservation release lives in that generator's `finally`. + +## Example + +Client: `POST /v1/chat/completions` `{ "model": "gpt-5.2", "messages": [...], "stream": false }` +after the 2s startup probe timed out. First event is `response.failed` / +`rate_limit_exceeded`. Response is `429` and the reservation row is `released`. diff --git a/openspec/changes/close-nonstream-chat-collect/proposal.md b/openspec/changes/close-nonstream-chat-collect/proposal.md new file mode 100644 index 0000000000..29208cc0ba --- /dev/null +++ b/openspec/changes/close-nonstream-chat-collect/proposal.md @@ -0,0 +1,30 @@ +## Why + +Non-streaming `POST /v1/chat/completions` takes the first SSE item off the +upstream generator, then collects through a prepend wrapper. When that first +item is `response.failed`, collect returns without closing the original +generator. The stream `finally` (API-key reservation release and lease +cleanup) does not run, and the route maps every collected error to HTTP 502 +except a small unavailable-selection set. The same failure on +`POST /v1/responses` drains the generator and uses `_status_for_error` +(429 for `rate_limit_exceeded`). + +## What Changes + +- Close the upstream `stream_responses` generator after non-stream chat + collect, including early `response.failed` / `error` returns. +- Map collected Chat Completions error envelopes with the same HTTP status + helper as non-stream `/v1/responses`. + +## Capabilities + +### Modified Capabilities + +- `chat-completions-compat`: non-stream chat settles the upstream generator + and returns the Responses-aligned error status. + +## Impact + +Streaming Chat Completions and `/v1/responses` collect are unchanged. Clients +that already treated every non-stream chat error as 502 will now see 429/401/ +400 when the envelope code already implied that. diff --git a/openspec/changes/close-nonstream-chat-collect/specs/chat-completions-compat/spec.md b/openspec/changes/close-nonstream-chat-collect/specs/chat-completions-compat/spec.md new file mode 100644 index 0000000000..b06d81d1bd --- /dev/null +++ b/openspec/changes/close-nonstream-chat-collect/specs/chat-completions-compat/spec.md @@ -0,0 +1,24 @@ +## ADDED Requirements + +### Requirement: Non-streaming chat collect closes the upstream generator + +When `stream` is `false` or omitted, `POST /v1/chat/completions` MUST close the upstream Responses generator after collect returns or raises, including when the first consumed event is `response.failed` or `error`. Closing MUST run the generator finalizer so an open API-key reservation is released or settled before the HTTP response is returned. + +#### Scenario: First-event rate limit releases the reservation + +- **WHEN** the startup probe did not consume the stream +- **AND** the first upstream event is `response.failed` with + `code=rate_limit_exceeded` +- **AND** the request reserved API-key usage +- **THEN** the reservation is released before the error response is returned + +### Requirement: Non-streaming chat errors use the Responses status map + +When non-streaming `POST /v1/chat/completions` returns an OpenAI error envelope collected from the upstream Responses stream, the HTTP status MUST match the non-streaming `/v1/responses` mapping for that envelope (`429` for `rate_limit_exceeded`, `503` for unavailable-selection codes, `401`/`400` where that path already maps them). The envelope body MUST remain an OpenAI error object. + +#### Scenario: Collected rate limit is 429 + +- **WHEN** non-streaming chat collect returns + `{ "error": { "code": "rate_limit_exceeded", ... } }` +- **THEN** the HTTP status is `429` +- **AND** the body is that OpenAI error envelope diff --git a/openspec/changes/close-nonstream-chat-collect/tasks.md b/openspec/changes/close-nonstream-chat-collect/tasks.md new file mode 100644 index 0000000000..b806730d7e --- /dev/null +++ b/openspec/changes/close-nonstream-chat-collect/tasks.md @@ -0,0 +1,18 @@ +## 1. Implementation + +- [x] 1.1 Close the original `stream_responses` generator after non-stream + chat collect, including `__anext__` errors and early collect return. +- [x] 1.2 Map collected chat error envelopes with `_status_for_error` / + `_mask_previous_response_not_found_error`. + +## 2. Regression coverage + +- [x] 2.1 Assert first-event `response.failed` closes the upstream generator. +- [x] 2.2 Assert non-stream chat `rate_limit_exceeded` returns 429 and + releases the API-key reservation. + +## 3. Validation + +- [x] 3.1 Run the new chat collect regressions and existing chat completion + suites. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/openspec/changes/coalesce-sticky-same-owner-refresh/proposal.md b/openspec/changes/coalesce-sticky-same-owner-refresh/proposal.md new file mode 100644 index 0000000000..d043b49d10 --- /dev/null +++ b/openspec/changes/coalesce-sticky-same-owner-refresh/proposal.md @@ -0,0 +1,65 @@ +# Coalesce same-owner sticky refresh writes + +## Why + +The sticky-session upsert is the single most expensive statement in a production +deployment: over 10 days of `pg_stat_statements` it accounted for 44% of total +database execution time (1,203,934 calls, mean 32.1ms, stddev 259.9ms, max 23.2s, +zero I/O time). The table itself is small (26MB / 27k rows) and healthy; the cost +is row-lock serialization. Every request that retains its pinned owner on a +TTL-based mapping (`prompt_cache`) re-executes +`INSERT ... ON CONFLICT (key, kind) DO UPDATE SET account_id = ..., updated_at = now(), ...` +purely to advance `updated_at`. Concurrent requests of one hot session hit the +same `(key, kind)` row and queue on its row lock through each other's commits, +which produces the heavy tail (stddev 8x the mean) and burns wall-clock time on +the TTFT-critical selection path. + +## What Changes + +- The owner lookup that selection already performs per request now also reports + a refresh-skip deadline when the row was observed fresh: `updated_at` within + `min(15s, 1% of the mapping TTL)`, not stamped in the future, AND no + abandonment marker in either `continuity_abandoned_at` or + `continuity_abandonment_scope`. The deadline is + `observed_updated_at + skip window`. +- When selection retains the same pinned owner, the mutation carries that + deadline to the persist site, which revalidates it against the clock at the + moment the statement would be issued and only then omits the same-owner + refresh upsert — no statement, no row lock. A deadline that lapsed during + admission or account-state persistence writes through, so the mapping's + effective expiry never moves earlier by more than the skip window. The next + request after the window closes performs the normal write-through refresh. +- Every state-changing write is unaffected and still immediate: rebinding to a + different account, deleting a mapping, restoring after failed admission, + clearing an abandonment tombstone, seeding a new mapping (including a thread + retention that must initialize a missing process seed — that write is the + seed-initialization carrier and is never skipped), and the raw legacy owner + paths. A row carrying any abandonment marker is never skippable because the + upsert also clears those marker columns. +- The deadline is DB-observed within the same request (no cross-request cache), + so it is correct with any number of workers or replicas: a replica can only + skip a write whose freshness it just read from the shared database. + +## Freshness window rationale + +`updated_at` on `prompt_cache` mappings is consumed by two TTL clocks, both +driven by `openai_cache_affinity_max_age_seconds` (default 1800s): the read-path +expiry in the owner lookup and the background cleanup loop. Skipping a refresh +while the row is younger than `min(15s, TTL * 0.01)` means a mapping's effective +expiry can move at most that window earlier — at most 1% of the TTL it protects, +and never more than 15 seconds. Sessions with request gaps longer than the +window (the overwhelming majority) still refresh on every request; only bursts +faster than the window coalesce, and those bursts re-refresh within the window +by construction. Durable kinds (`codex_session`, `sticky_thread` without TTL) +never used the refresh-on-retention write and are untouched. + +## Impact + +- Affected specs: `sticky-session-operations` +- Affected code: `app/modules/proxy/sticky_repository.py`, + `app/modules/proxy/_load_balancer/sticky_selection.py`, + `app/modules/proxy/load_balancer.py` +- No new settings, no migration, no dashboard surface. Routing decisions are + byte-identical; only the redundant same-owner freshness write is coalesced. +- Operators see `updated_at` in the dashboard sticky-session list advance in + steps of up to the skip window on hot sessions instead of per request. diff --git a/openspec/changes/coalesce-sticky-same-owner-refresh/specs/sticky-session-operations/spec.md b/openspec/changes/coalesce-sticky-same-owner-refresh/specs/sticky-session-operations/spec.md new file mode 100644 index 0000000000..828b742e49 --- /dev/null +++ b/openspec/changes/coalesce-sticky-same-owner-refresh/specs/sticky-session-operations/spec.md @@ -0,0 +1,82 @@ +## ADDED Requirements + +### Requirement: Same-owner sticky refresh writes are coalesced + +When selection retains the existing pinned owner of a TTL-based sticky mapping, the +mapping write exists only to advance the mapping's freshness timestamp. The system +MUST skip that write when the same request's owner lookup already observed the row +with a freshness timestamp younger than a bounded skip window, so concurrent requests +of one hot session do not serialize on the same row's lock. + +The skip window MUST NOT exceed 1% of the mapping's configured TTL and MUST NOT +exceed 15 seconds, so a mapping's effective expiry — on both the read-path TTL check +and the background cleanup loop — moves at most that window earlier than today's +write-per-request behavior. + +The skip decision MUST be derived from row state observed in the current request's +database lookup, not from cross-request in-process state, so any number of workers or +replicas remain correct. The lookup MUST report the skip as a deadline (the observed +freshness timestamp plus the skip window), and the write path MUST revalidate that +deadline against the clock at the moment the write would otherwise be issued — a +deadline that lapsed while the request was being admitted no longer authorizes a +skip. A row whose observed freshness timestamp lies in the future (clock skew or a +restored row) MUST NOT be skippable at all. + +A skip MUST apply only to a pure freshness rewrite. The following writes MUST remain +immediate and unconditional: rebinding the mapping to a different account, deleting +the mapping, restoring a provisional owner after failed admission, initializing a +seed mapping, and any upsert against a row carrying an abandonment marker (whose +write also clears the marker columns). In particular, a retention write that would +initialize a missing seed mapping MUST NOT be skipped even when the retained row +itself was observed fresh, because the seed initialization piggybacks on that write. +A raw legacy owner that shadows the namespaced row MUST NOT inherit the namespaced +row's freshness observation. + +#### Scenario: Hot same-owner retention skips the redundant refresh write + +- **GIVEN** a `prompt_cache` mapping pinned to an eligible account +- **AND** the request's owner lookup observed the row fresher than the skip window + with no abandonment marker +- **WHEN** selection retains the pinned account +- **THEN** the request routes to the pinned account +- **AND** no sticky-session write is issued for the retention + +#### Scenario: Retention outside the skip window refreshes write-through + +- **GIVEN** a `prompt_cache` mapping pinned to an eligible account +- **AND** the row's freshness timestamp is older than the skip window but inside the TTL +- **WHEN** selection retains the pinned account +- **THEN** the mapping's freshness timestamp is advanced by a write + +#### Scenario: Rebind is never coalesced + +- **GIVEN** a soft mapping whose row was observed fresher than the skip window +- **WHEN** selection rebinds the mapping to a different account +- **THEN** the rebind is persisted immediately + +#### Scenario: A skipped refresh does not clobber a concurrent rebind + +- **GIVEN** a request that observed a fresh same-owner row and skipped its refresh write +- **AND** a concurrent request rebinds the same mapping to another account +- **WHEN** both requests complete +- **THEN** the mapping's owner is the rebind target + +#### Scenario: A retention that must initialize a missing seed is never skipped + +- **GIVEN** a thread mapping observed fresher than the skip window +- **AND** the corresponding process seed mapping does not exist +- **WHEN** selection retains the thread mapping's pinned account +- **THEN** the retention write is issued and the seed mapping is initialized + +#### Scenario: A deadline that lapsed during admission writes through + +- **GIVEN** a request whose lookup observed the row inside the skip window +- **AND** admission latency carried the request past the observed skip deadline +- **WHEN** the retention write would be issued +- **THEN** the deadline is revalidated and the freshness write is performed + +#### Scenario: A future freshness timestamp is never skippable + +- **GIVEN** a mapping whose freshness timestamp lies ahead of the current clock +- **WHEN** the owner lookup evaluates the skip window +- **THEN** no skip deadline is reported and retention writes through diff --git a/openspec/changes/coalesce-sticky-same-owner-refresh/tasks.md b/openspec/changes/coalesce-sticky-same-owner-refresh/tasks.md new file mode 100644 index 0000000000..5ff96ff431 --- /dev/null +++ b/openspec/changes/coalesce-sticky-same-owner-refresh/tasks.md @@ -0,0 +1,47 @@ +# Tasks + +## 1. Repository freshness observation + +- [x] 1.1 Extend `StickyOwnerLookup` with `refresh_skip_deadline` + (`observed_updated_at + skip window`), computed only on the fresh-row TTL + lookup path: `updated_at` within `min(15s, 1% of TTL)`, not in the future, + and both abandonment marker columns NULL +- [x] 1.2 Keep the deadline unset on the stale-delete recovery path, on lookups + without a TTL, on rows carrying any abandonment marker, and on rows whose + `updated_at` is ahead of the clock + +## 2. Selection wiring + +- [x] 2.1 Thread the deadline from `run_sticky_selection_path`'s per-attempt owner + lookup through `_select_with_stickiness` onto the retention mutation; reset it + when the raw legacy owner shadows the namespaced row, when the inner helper + re-resolves the owner itself, and when the process seed is still missing + (seed initialization piggybacks on the retention write) +- [x] 2.2 Revalidate the deadline at the persist site (`_sticky_refresh_write_skippable`) + and only then omit the same-owner refresh statement — on both the non-probe + persist path and the recovery-probe admission path (whose compensating + restores are skipped symmetrically when nothing was written); rebinds, + deletes, restores of actually-written rows, and seed-initializing writes + keep writing immediately + +## 3. Verification + +- [x] 3.1 Unit tests: skip on fresh same-owner retention, write-through when the + deadline is unset or lapsed at persist time, rebind/departed-owner writes never + suppressed, grace-period retention honors the window, internal re-resolution + resets the deadline, persist-time gate guards deletes/seed writes/non-datetime + deadlines +- [x] 3.2 Balancer-level tests: fresh same-owner retention issues no write when the + seed exists, a fresh thread row with a missing seed still writes and initializes + the seed, an expired deadline writes through, and a fresh retention of a + due-probing pinned owner skips the write on the probe admission path while + the probe reservation still commits +- [x] 3.3 Integration tests: deadline conditions against the real repository (fresh + row, TTL-scaled window, marker disqualification, future timestamp, no-TTL + lookup), concurrent upserts on one `(key, kind)` keep RETURNING/self-write and + single-row semantics, a skipped refresh never clobbers a concurrent rebind +- [x] 3.4 `uv run pytest tests/unit/test_select_with_stickiness.py + tests/unit/test_load_balancer_concurrency.py + tests/integration/test_proxy_sticky_sessions.py`, `uv run ruff check`, + `uv run ruff format --check`, `make typecheck` +- [x] 3.5 `openspec validate coalesce-sticky-same-owner-refresh --strict` diff --git a/openspec/changes/configure-model-source-reasoning-efforts/proposal.md b/openspec/changes/configure-model-source-reasoning-efforts/proposal.md new file mode 100644 index 0000000000..2004c1ea45 --- /dev/null +++ b/openspec/changes/configure-model-source-reasoning-efforts/proposal.md @@ -0,0 +1,41 @@ +## Why + +PR #1661 already landed the backend contract for operator-declared model-source +reasoning efforts. PR #1675 still carries the useful dashboard part of that +feature, but its original branch also duplicated the backend parser/spec work +and baked in assumptions that #1661 explicitly rejected (`none` filtering and a +fixed effort enum). + +The dashboard still needs a way to configure the metadata that the backend now +honors, and it needs to preserve arbitrary provider-specific effort slugs rather +than forcing one hardcoded vocabulary. + +## What Changes + +- Add model-source dashboard controls for the reasoning-effort metadata that + `#1661` already reads from `raw_metadata_json`. +- Store supported efforts as an operator-edited list of slugs instead of a + fixed checkbox enum, so the UI can round-trip `none` and provider-specific + effort names. +- Keep the existing reasoning toggle, seed reasonable defaults when an operator + enables it for the first time, and normalize stale defaults back onto the + configured effort list during edit/save. +- Localize the new dashboard copy in `en`, `ko`, and `zh-CN`. + +## Capabilities + +### New Capabilities + +- None. + +### Modified Capabilities + +- `frontend-architecture`: model-source create/edit dialogs can configure and + preserve supported reasoning-effort metadata for source models. + +## Impact + +- Dashboard only: model-source form state, create/edit dialogs, i18n, and + focused frontend regression tests. +- No API contract, database, backend parser, proxy routing, or request-policy + behavior changes in this PR; those remain owned by #1661. diff --git a/openspec/changes/configure-model-source-reasoning-efforts/specs/frontend-architecture/spec.md b/openspec/changes/configure-model-source-reasoning-efforts/specs/frontend-architecture/spec.md new file mode 100644 index 0000000000..1229e111bc --- /dev/null +++ b/openspec/changes/configure-model-source-reasoning-efforts/specs/frontend-architecture/spec.md @@ -0,0 +1,38 @@ +## MODIFIED Requirements + +### Requirement: Model-source reasoning metadata editor + +The dashboard MUST let operators configure the reasoning metadata stored on +model-source models without assuming one global effort vocabulary. + +#### Scenario: Edit arbitrary supported reasoning efforts + +- **GIVEN** a model source whose `raw_metadata_json` contains + `supports_reasoning: true` +- **AND** its `supported_reasoning_levels` include values such as `none` or a + provider-specific slug +- **WHEN** the dashboard opens the model-source create or edit form +- **THEN** the reasoning controls MUST show those effort slugs without dropping + or rewriting them +- **AND** saving the form MUST write the edited effort list back into + `supported_reasoning_levels`. + +#### Scenario: Normalize stale defaults during save + +- **GIVEN** a model source whose configured default effort is no longer present + in the edited supported-effort list +- **WHEN** the operator saves the form +- **THEN** the dashboard MUST replace the stale default with one of the + configured supported efforts +- **AND** it MUST NOT leave `default_reasoning_level` pointing at a removed + value. + +#### Scenario: Seed a first-time reasoning configuration + +- **GIVEN** an operator enables reasoning for a model source that previously had + no configured supported-effort list +- **WHEN** the dashboard reveals the reasoning metadata controls +- **THEN** the form MUST seed an editable default effort list and default value + so the operator can save a valid initial configuration +- **AND** the operator MUST still be able to replace that seed with arbitrary + effort slugs before saving. diff --git a/openspec/changes/configure-model-source-reasoning-efforts/tasks.md b/openspec/changes/configure-model-source-reasoning-efforts/tasks.md new file mode 100644 index 0000000000..ea2bfc87aa --- /dev/null +++ b/openspec/changes/configure-model-source-reasoning-efforts/tasks.md @@ -0,0 +1,15 @@ +## 1. Model-source dashboard + +- [x] 1.1 Extend model-source form state so it can parse, round-trip, and save + supported reasoning efforts plus the default effort from raw metadata. +- [x] 1.2 Add create/edit dialog controls for the effort list and default + selector without constraining operators to a fixed enum. +- [x] 1.3 Keep the existing reasoning toggle and preserve unrelated raw metadata + keys during edits. + +## 2. Verification + +- [x] 2.1 Add focused frontend regression coverage for default seeding, + arbitrary effort round-tripping, and stale-default normalization. +- [x] 2.2 Run focused frontend checks and strict OpenSpec validation for this + change. diff --git a/openspec/changes/customize-dashboard-request-log-columns/.openspec.yaml b/openspec/changes/customize-dashboard-request-log-columns/.openspec.yaml new file mode 100644 index 0000000000..8e7013b8b1 --- /dev/null +++ b/openspec/changes/customize-dashboard-request-log-columns/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-27 diff --git a/openspec/changes/customize-dashboard-request-log-columns/design.md b/openspec/changes/customize-dashboard-request-log-columns/design.md new file mode 100644 index 0000000000..2ba9649b5e --- /dev/null +++ b/openspec/changes/customize-dashboard-request-log-columns/design.md @@ -0,0 +1,44 @@ +## Context + +The dashboard renders request logs through `RecentRequestsTable`, with a fixed +set of columns and browser-managed horizontal overflow. The feature is entirely +presentational and requires no backend contract changes. + +## Goals / Non-Goals + +**Goals:** + +- Let operators select visible request-log columns from the existing dashboard. +- Let pointer and keyboard users resize each visible column independently. +- Persist and safely restore the layout per browser. +- Preserve existing request filtering, pagination, row details, and defaults. + +**Non-Goals:** + +- Creating another dashboard route or navigation item. +- Changing request-log APIs, schemas, database records, or server settings. +- Synchronizing layout preferences between browsers or users. + +## Decisions + +- Keep column metadata and bounded default widths in one typed frontend module + so the chooser, table, and preference validation share a single source of + truth. +- Store only column identifiers and widths in versioned `localStorage`. + Defensive parsing ignores unknown columns and malformed widths. +- Extend `RecentRequestsTable` with optional presentation props. Its existing + defaults remain all columns, preserving other callers and tests. +- Render an accessible separator in each visible header. Pointer movement sets + the width continuously; Left/Right arrow keys adjust it by a fixed step. +- Set table minimum width to the sum of visible widths so the existing + horizontal scroll container handles overflow without a global width slider. + +## Risks / Trade-offs + +- Browser-local settings can become stale after future columns are added. + → Validate stored identifiers and provide a restore-default action. +- Very wide user-selected columns require horizontal scrolling. + → Keep bounded widths and retain the existing overflow container. +- Drag handles can interfere with header content. + → Restrict pointer behavior to a narrow trailing-edge separator with an + explicit resize cursor and accessible name. diff --git a/openspec/changes/customize-dashboard-request-log-columns/proposal.md b/openspec/changes/customize-dashboard-request-log-columns/proposal.md new file mode 100644 index 0000000000..df5cc96cb7 --- /dev/null +++ b/openspec/changes/customize-dashboard-request-log-columns/proposal.md @@ -0,0 +1,35 @@ +## Why + +The dashboard request log contains many fields, but operators cannot currently +prioritize the fields they use or allocate more horizontal space to values that +need it. Configurable visibility and per-column sizing make the existing table +usable across different workflows and screen sizes. + +## What Changes + +- Add a column chooser to the existing dashboard Request Logs section. +- Allow each visible request-log column to be resized by dragging its header + separator, with keyboard adjustment for accessibility. +- Persist visible columns and individual widths in browser-local storage. +- Preserve at least one visible column and provide a control that restores the + default column layout. +- Derive the table's minimum width from its visible columns so wide layouts + remain horizontally scrollable. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `frontend-architecture`: Define configurable visibility, resizable headers, + persistence, reset behavior, and accessibility for dashboard request logs. + +## Impact + +- Dashboard-only frontend changes under `frontend/src/features/dashboard/`. +- A small browser-local preference module; no API, database, authentication, + routing, or deployment changes. +- Focused component, preference, and dashboard integration tests. diff --git a/openspec/changes/customize-dashboard-request-log-columns/specs/frontend-architecture/spec.md b/openspec/changes/customize-dashboard-request-log-columns/specs/frontend-architecture/spec.md new file mode 100644 index 0000000000..9bb4f7cea5 --- /dev/null +++ b/openspec/changes/customize-dashboard-request-log-columns/specs/frontend-architecture/spec.md @@ -0,0 +1,53 @@ +## ADDED Requirements + +### Requirement: Configurable dashboard request-log columns + +The dashboard SHALL let operators show or hide request-log columns and MUST +preserve at least one visible column. Column choices MUST be stored locally per +browser and restored on later visits. Malformed or stale stored choices MUST +fall back to supported defaults without preventing the dashboard from +rendering. A restore-default action MUST clear customized visibility and width +values. + +#### Scenario: Choose visible request-log columns + +- **WHEN** an operator selects or deselects columns in the dashboard request-log column chooser +- **THEN** the corresponding request-log headers and cells are shown or hidden +- **AND** the choice is restored when that browser revisits the dashboard + +#### Scenario: Preserve a usable table + +- **WHEN** only one request-log column remains visible +- **THEN** the dashboard prevents that final column from being hidden + +#### Scenario: Recover from invalid stored preferences + +- **WHEN** stored request-log preferences are malformed or contain unsupported column identifiers +- **THEN** the dashboard renders with supported default columns and widths + +### Requirement: Resizable dashboard request-log columns + +The dashboard SHALL render a vertical resize separator at the trailing edge of +each visible request-log header. Dragging a separator MUST adjust that column +within bounded minimum and maximum widths without changing other configured +columns. Individual widths MUST be stored locally per browser and restored on +later visits. Separators MUST support keyboard adjustment, and the table's +minimum width MUST be derived from its visible column widths so overflow +remains horizontally scrollable without a global table-width control. + +#### Scenario: Resize a request-log column by dragging + +- **WHEN** an operator drags a request-log header separator horizontally +- **THEN** the corresponding header and body column change width +- **AND** the selected width is restored on a later dashboard visit in the same browser + +#### Scenario: Resize a request-log column with the keyboard + +- **WHEN** a focused request-log header separator receives a Left or Right arrow key +- **THEN** the corresponding column width decreases or increases by the documented step within its bounds + +#### Scenario: Wide columns remain reachable + +- **WHEN** the sum of visible request-log column widths exceeds the available viewport +- **THEN** the table remains horizontally scrollable +- **AND** no separate global table-width control is displayed diff --git a/openspec/changes/customize-dashboard-request-log-columns/tasks.md b/openspec/changes/customize-dashboard-request-log-columns/tasks.md new file mode 100644 index 0000000000..2a7e4e17e4 --- /dev/null +++ b/openspec/changes/customize-dashboard-request-log-columns/tasks.md @@ -0,0 +1,24 @@ +## 1. Column layout model + +- [x] 1.1 Add typed request-log column metadata, default widths, and width bounds. +- [x] 1.2 Add a versioned browser-local preference hook with defensive parsing, persistence, final-column protection, and restore-default behavior. +- [x] 1.3 Add focused preference tests for persistence, malformed data, bounds, and reset behavior. + +## 2. Resizable request-log table + +- [x] 2.1 Extend `RecentRequestsTable` with optional visible-column and column-width props while preserving all-column defaults. +- [x] 2.2 Render only selected headers and cells, and derive table minimum width from visible column widths. +- [x] 2.3 Add accessible pointer and keyboard resize separators to visible headers. +- [x] 2.4 Add component tests for visibility, pointer resizing, keyboard resizing, bounds, and horizontal overflow sizing. + +## 3. Dashboard integration + +- [x] 3.1 Add the column chooser and restore-default action to the existing dashboard Request Logs section. +- [x] 3.2 Connect saved visibility and width preferences to `RecentRequestsTable` without changing filters, pagination, or account/dashboard content. +- [x] 3.3 Add dashboard integration coverage for column selection, resizing, persistence, and absence of a global width control. + +## 4. Validation + +- [x] 4.1 Run frontend type checking, lint, focused tests, and the full frontend suite. +- [x] 4.2 Run the production frontend build and strict OpenSpec validation. +- [x] 4.3 Review the final diff to confirm it contains no Compact route, navigation, backend, deployment, secret, or machine-specific changes. diff --git a/openspec/changes/dashboard-first-run-empty-states/.openspec.yaml b/openspec/changes/dashboard-first-run-empty-states/.openspec.yaml new file mode 100644 index 0000000000..b6b2d1f67c --- /dev/null +++ b/openspec/changes/dashboard-first-run-empty-states/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-13 diff --git a/openspec/changes/dashboard-first-run-empty-states/context.md b/openspec/changes/dashboard-first-run-empty-states/context.md new file mode 100644 index 0000000000..d84422e853 --- /dev/null +++ b/openspec/changes/dashboard-first-run-empty-states/context.md @@ -0,0 +1,36 @@ +# Context: dashboard first-run empty states + +Normative requirements live in the change delta spec. This note records the +first-run review that produced the change. + +## Purpose + +A brand-new dashboard (no accounts, no API keys, no request logs) should tell +the operator what to do next. Filter-empty copy and zero-line charts imply +data exists and was hidden. + +## Decisions + +- Copy is the product fix; Accounts/APIs already expose add/create above the + list, so they do not get a second CTA. +- Dashboard empty-account cards/list are the one place that needs a link, + because those surfaces have no add control. +- Reports no-data is keyed off an empty `daily` array. Gap-filling zeros stay + for sparse-but-present series. +- `/firewall` stays as a compatibility route. The redirect target is richer; + `/settings` without query/hash stays collapsed so first-paint still skips + Advanced self-fetches. + +## Example + +Empty fleet, operator opens `/accounts`: "No accounts yet" / "Add an account +to start routing." After importing one account and searching for a missing +email: "No matching accounts" / "Adjust filters." + +Opening `/firewall` lands on Settings with Advanced open and the Firewall +heading in view. + +## Non-goals + +Guest write access, skeleton loading, and session-fail-as-admin stay out of +this change. diff --git a/openspec/changes/dashboard-first-run-empty-states/design.md b/openspec/changes/dashboard-first-run-empty-states/design.md new file mode 100644 index 0000000000..8c7e2a0ac3 --- /dev/null +++ b/openspec/changes/dashboard-first-run-empty-states/design.md @@ -0,0 +1,59 @@ +## Context + +First-run operators land on empty Accounts, APIs, Dashboard, and Reports +surfaces. Automations already splits first-run vs filtered empty copy. Settings +keeps Advanced collapsed by default so first-paint skips firewall/quota +self-fetches. The legacy `/firewall` route only redirects to `/settings`. + +## Goals / Non-Goals + +**Goals:** + +- First-run empty lists describe setup, not filter mismatch. +- Filtered-empty lists keep "no matches / adjust filters" copy. +- Dashboard empty-account surfaces link to `/accounts`. +- Reports line charts with no daily rows show a no-data state instead of a + zero-filled series. +- `/firewall` expands Advanced and scrolls to the firewall section. +- Plain `/settings` stays collapsed by default. + +**Non-Goals:** + +- Guest write-gating, infinite skeletons, or session-fail-as-admin. +- Conversation empty-copy, donut empty-copy, or daily-table zero-fill changes. +- New Settings query parameters beyond `advanced=1` and the existing `#firewall` + hash. +- API, schema, or nav-budget changes. + +## Decisions + +1. **Reuse the Automations empty-vs-filtered key pattern.** Lists key off + source-array emptiness (`accounts.length === 0`), not only the filtered + array. Request logs take a `filtersApplied` flag from non-default filters + (search, timeframe other than `all`, account/API-key/model/status + selections, or conversation pin). +2. **Optional EmptyState action slot.** Dashboard empty-account cards and list + render a `Link` to `/accounts`. Accounts/APIs already have add/create + buttons above the list, so they only change copy. +3. **Reports no-data uses the raw `daily` payload.** + `buildContinuousDailyRows` still fills gaps when any daily row exists. + Empty `data` skips the chart. +4. **Firewall deeplink is a redirect target, not a new page.** + `/firewall` → `/settings?advanced=1#firewall`. Advanced opens when + `advanced=1` or the hash is `#firewall`. After mount, scroll to + `id="firewall"`. Unmount-while-collapsed is unchanged for plain `/settings`. + +## Risks / Trade-offs + +- Opening Advanced on the deeplink issues the same self-fetching section + requests as a manual expand. Acceptable: the operator asked for firewall. +- `filtersApplied` treats any non-default request-log filter as filtered-empty, + including a narrowed timeframe on a brand-new fleet. That is correct: the + operator is looking at a filter, not first-run. +- Hash scroll depends on the firewall section mounting after expand. Tests + cover the heading becoming visible; scroll is best-effort `scrollIntoView`. + +## Migration Plan + +No data migration. Existing `/firewall` bookmarks keep working with a richer +target. Operators on `/settings` see no change. diff --git a/openspec/changes/dashboard-first-run-empty-states/proposal.md b/openspec/changes/dashboard-first-run-empty-states/proposal.md new file mode 100644 index 0000000000..4e437bd1e5 --- /dev/null +++ b/openspec/changes/dashboard-first-run-empty-states/proposal.md @@ -0,0 +1,35 @@ +## Why + +First-run dashboard surfaces tell operators to "Adjust filters" or show a +zero-line Reports chart when the fleet is empty. The legacy `/firewall` +redirect lands on Settings with Advanced collapsed, so the firewall section +stays hidden. + +## What Changes + +- Accounts, APIs, and request-log empty copy distinguish first-run (no rows) + from filtered-empty (rows exist but none match). +- Dashboard empty-account cards/list include a CTA to `/accounts`. +- Reports line charts show a no-data empty state when the report payload has + no daily rows, instead of a continuous zero-filled chart. +- `/firewall` redirects to `/settings?advanced=1#firewall`, which expands + Advanced and scrolls to the firewall section. Plain `/settings` stays + collapsed by default. + +## Capabilities + +### New Capabilities + +- None + +### Modified Capabilities + +- `frontend-architecture`: first-run vs filtered empty states, dashboard + empty-account CTA, reports no-data charts, and `/firewall` Advanced deeplink. + +## Impact + +Dashboard SPA only: empty-state copy, optional EmptyState action, Reports +chart empty rendering, Settings Advanced open-from-query/hash, `/firewall` +redirect target, and i18n (`en`/`ko`/`zh-CN`). No API, database, proxy, or +nav-budget changes. diff --git a/openspec/changes/dashboard-first-run-empty-states/specs/frontend-architecture/spec.md b/openspec/changes/dashboard-first-run-empty-states/specs/frontend-architecture/spec.md new file mode 100644 index 0000000000..e7ba75f8e1 --- /dev/null +++ b/openspec/changes/dashboard-first-run-empty-states/specs/frontend-architecture/spec.md @@ -0,0 +1,108 @@ +MUST distinguish first-run empty states from filter-empty states on dashboard operator lists. + +## ADDED Requirements + +### Requirement: First-run empty lists describe setup, not filter mismatch + +Accounts, APIs, and dashboard request-log empty states SHALL distinguish a +first-run empty source list from a filtered-empty result. When the source +list has no items and no narrowing filter is applied, the empty copy SHALL +describe setup (no accounts yet, no API keys yet, no requests yet) and SHALL +NOT tell the operator to adjust filters. When the source list has items, or +request-log filters differ from their defaults, and the visible result is +empty, the empty copy SHALL describe a filter mismatch. + +#### Scenario: Accounts first-run empty copy + +- **GIVEN** the Accounts page has no accounts +- **WHEN** the account list renders +- **THEN** the empty title describes that there are no accounts yet +- **AND** the empty description does not tell the operator to adjust filters + +#### Scenario: Accounts filtered empty copy + +- **GIVEN** the Accounts page has at least one account +- **AND** the current search or status filter matches none of them +- **WHEN** the account list renders +- **THEN** the empty title describes that no accounts match +- **AND** the empty description tells the operator to adjust filters + +#### Scenario: APIs first-run empty copy + +- **GIVEN** the APIs page has no API keys +- **WHEN** the API key list renders +- **THEN** the empty title describes that there are no API keys yet +- **AND** the empty description does not tell the operator to adjust filters + +#### Scenario: Request logs first-run empty copy + +- **GIVEN** the request-log listing has no rows +- **AND** request-log filters are at their defaults +- **WHEN** the recent-requests table renders +- **THEN** the empty title is `No requests yet` +- **AND** the empty description does not say that request logs match the current filters + +#### Scenario: Request logs filtered empty copy + +- **GIVEN** the request-log listing has no rows +- **AND** at least one request-log filter differs from its default +- **WHEN** the recent-requests table renders +- **THEN** the empty copy describes that no request logs match the current filters + +### Requirement: Dashboard empty accounts include a CTA to Accounts + +The dashboard empty-account cards and list SHALL include a control that +navigates to `/accounts` when the overview has no accounts. + +#### Scenario: Empty account cards link to Accounts + +- **GIVEN** the dashboard overview has no accounts +- **WHEN** the account cards empty state renders +- **THEN** the empty state includes a link to `/accounts` + +#### Scenario: Empty account list links to Accounts + +- **GIVEN** the dashboard overview has no accounts +- **WHEN** the account list empty state renders +- **THEN** the empty state includes a link to `/accounts` + +### Requirement: Reports line charts show no-data when daily rows are absent + +When a Reports line chart receives an empty daily-row array, it SHALL render +a no-data empty state and SHALL NOT render a continuous zero-filled series +for the selected date range. When the daily-row array has at least one row, +the chart MAY still fill missing days with zeros. + +#### Scenario: Empty daily payload hides the zero-line chart + +- **GIVEN** `GET /api/reports` returns no daily rows +- **WHEN** a visible Reports line chart renders +- **THEN** the chart card shows a no-data empty state +- **AND** it does not render an area or line series of zero values + +#### Scenario: Partial daily payload still fills missing days + +- **GIVEN** a Reports line chart receives daily rows for some days in the selected range +- **WHEN** the chart renders +- **THEN** missing days in that range are still filled with zero values + +### Requirement: Legacy firewall route expands Advanced and targets the firewall section + +The `/firewall` route SHALL redirect to `/settings?advanced=1#firewall`. +Opening Settings with `advanced=1` or hash `#firewall` SHALL expand the +Advanced settings group on first render so the firewall section mounts. +The firewall section SHALL expose `id="firewall"`. Opening `/settings` +without that query or hash SHALL keep Advanced collapsed by default. + +#### Scenario: Legacy /firewall deeplink shows the firewall section + +- **WHEN** an operator opens `/firewall` +- **THEN** the SPA navigates to `/settings?advanced=1#firewall` +- **AND** the Advanced settings group is expanded +- **AND** the firewall section heading is visible without a further expand click + +#### Scenario: Plain Settings stays collapsed + +- **WHEN** an operator opens `/settings` without `advanced=1` and without `#firewall` +- **THEN** the Advanced settings group remains collapsed +- **AND** the firewall section is not mounted diff --git a/openspec/changes/dashboard-first-run-empty-states/tasks.md b/openspec/changes/dashboard-first-run-empty-states/tasks.md new file mode 100644 index 0000000000..05170d718e --- /dev/null +++ b/openspec/changes/dashboard-first-run-empty-states/tasks.md @@ -0,0 +1,19 @@ +## 1. Empty-state copy and CTA + +- [x] 1.1 Add optional `action` slot to `EmptyState` +- [x] 1.2 Split Accounts/APIs first-run vs filtered-empty copy +- [x] 1.3 Split request-log first-run vs filtered-empty copy +- [x] 1.4 Add dashboard empty-account CTA to `/accounts` on cards and list +- [x] 1.5 Add `en` / `ko` / `zh-CN` keys for the new copy + +## 2. Reports no-data and firewall deeplink + +- [x] 2.1 Show no-data empty state on Reports line charts when `daily` is empty +- [x] 2.2 Redirect `/firewall` to `/settings?advanced=1#firewall` +- [x] 2.3 Expand Advanced from `advanced=1` or `#firewall` and set firewall `id` + +## 3. Validation + +- [x] 3.1 Unit/integration tests for empty copy, CTA, no-data charts, and `/firewall` +- [x] 3.2 `openspec validate --specs` for this change +- [x] 3.3 Browser-verify first-run empty surfaces and the firewall deeplink diff --git a/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/design.md b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/design.md new file mode 100644 index 0000000000..ccb42df761 --- /dev/null +++ b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/design.md @@ -0,0 +1,128 @@ +# Design: attempt-scoped retry-circuit recording + +## Context + +All HTTP bridge upstream sends pass through +`_send_http_bridge_request_text_with_archive_id`, but retry-circuit failures can +be reported by four paths: partial stale cleanup, direct stale retirement, the +upstream reader failure funnel, and downstream stream-idle handling. These +paths may run concurrently and may await recovery or settlement before they +record the failure. + +The durable retry-circuit upsert intentionally merges separate writes as +separate observations. It cannot infer that two writes came from the same +physical send, and adding a durable attempt key would expand the schema and the +rolling-upgrade contract unnecessarily. + +## Decisions + +### Keep identity process-local and object-scoped + +Each upstream send creates a new attempt object stored on its request state. +The object carries a diagnostic ordinal plus `disarmed`, `response_observed`, +and `retry_circuit_failure_recorded` state. It also carries a settlement signal, +but deliberately does not cache a historical failure count. Observers capture +the object itself, not merely the request's current ordinal. + +An older observer therefore retains the identity of the send it classified even +if a retry replaces the request state's current attempt. The old object remains +alive only while an observer references it, so no unbounded generation set is +needed. + +### Preserve the existing failure eligibility contract + +Creating an attempt does not itself record a failure. A send exception or +cancellation disarms it using the same cleanup boundary that clears +`response_create_sent_at`. A matched `response.*` event marks it observed before +the reader performs another await. An observer that has not already recorded +the attempt must not record it after either condition wins. + +If the failure was already recorded, later duplicate observers wait for its +durable merge to settle and then read the live circuit state without another +increment. This means a later independent attempt is reflected in the returned +count, while a successful response that cleared the circuit is reported as +zero. It preserves the reader's existing threshold-dependent durable-anchor +handling without allowing a cached count from an older send to reopen or poison +state that has since changed. + +### Distinguish absent attribution from ambiguous attribution + +Failure funnels pass an explicit selection result rather than overloading +`None`. `absent` means the legacy path has no attempt object and may use the +unscoped recorder. `eligible`, `recorded`, and `settled` retain the exact object +identities, including multiple candidates. `ineligible` means an attempt was +present but lifecycle evidence makes it unsafe to charge. + +A single candidate is handled with the normal attempt-scoped recorder. Multiple +eligible or settled candidates are deliberately suppressed rather than falling +back to an unscoped strike, because an unscoped increment cannot identify which +physical send it represents and can double-count a later observer. Multiple +already-recorded candidates wait for settlement and return the live circuit +count without incrementing. + +### Claim under the existing retry-circuit lock + +The attempt marker and `consecutive_failures` increment are changed in the same +critical section guarded by `_http_bridge_retry_circuit_lock`. Durable I/O stays +outside that lock. Duplicate calls may both perform the existing durable load, +but only the first claim persists a failure. + +No new lock is introduced. Failure paths release `pending_lock` before entering +the recorder, and no retry-circuit path acquires `pending_lock` or +`lifecycle_lock` while holding the retry-circuit lock. + +### Capture before ownership and recovery awaits + +The response-create gate classifies stale owners and snapshots their attempts +while holding `pending_lock`. Shared cleanup also snapshots before it waits to +acquire that lock, so callers that do not provide a locked snapshot still retain +the pre-wait identity. The downstream timeout and reader watchdog likewise +capture before calling retry, reconnect, receive cancellation, or settlement +helpers. Reading the request state's current attempt afterward could attribute +an old timeout to a newer retry or suppress the old failure incorrectly. + +### Mark lifecycle observation before deferred delivery + +Some reasoning prelude events are intentionally deferred and therefore do not +increment the ordinary response-event counter. They still prove that upstream +accepted and began answering the physical `response.create`. The matched +attempt is marked observed immediately, before deferred-delivery branching or +any later await, while the existing event-count and downstream-visibility +semantics remain unchanged. + +### Keep replica behavior unchanged + +The active owner alone holds the upstream WebSocket and its request state, so +duplicate local observers share one attempt object. Owner forwarding does not +create another upstream send on the forwarding replica. A replay after owner +handoff is a new send and is intentionally a new strike. Existing durable +conflict merging continues to combine genuinely independent replica failures. + +## Failure Modes + +- If durable lookup or persistence fails, the first claim remains in local + circuit state as it does today; a duplicate observer must not retry the write + because the durable upsert would interpret it as a second failure. +- If a response event wins before the first claim, the attempt is not counted. + If a failure claim wins first, a later response cannot turn a duplicate + observer into another strike. +- If a successful terminal response clears the circuit before a delayed + duplicate observer resumes, the retained attempt marker prevents the old + observer from recreating the cleared failure and the live count returned to + it is zero. +- If a cleanup snapshot contains multiple eligible sends, it records none at + that ambiguous boundary. Later observers that retain an exact send identity + can still claim each genuine failure independently. +- If response accounting is deferred for a reasoning prelude, the attempt's + observed marker still wins against an eventless timeout without making the + deferred event visible or incrementing its ordinary event count. + +## Example + +Attempt A is sent and remains eventless. The downstream stream watchdog and the +reader watchdog both capture A. The downstream task claims A first, records +failure count 1, and persists once. The reader later sees that A is already +recorded, waits for settlement, reads the live count, and does not persist. If +recovery sends attempt B and B also fails before that reader resumes, the reader +returns the current count 2 without adding a third strike. If a successful +response clears the state instead, it returns 0. diff --git a/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/proposal.md b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/proposal.md new file mode 100644 index 0000000000..9f4dc9cd0c --- /dev/null +++ b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/proposal.md @@ -0,0 +1,71 @@ +# Deduplicate HTTP bridge retry-circuit failures by send attempt + +## Summary + +An eventless HTTP bridge `response.create` can be observed by both the upstream +reader watchdog and the downstream stream-idle watchdog. Each observer currently +persists an independent retry-circuit failure even though both are reporting the +same upstream send. With the default threshold of two, one silent send can +therefore open the circuit and make a later request fail locally with HTTP 503. + +Track the individual process-local upstream send attempt and let all failure +observers claim that attempt through the existing retry-circuit lock. The first +eligible observer records and persists the failure; later observers of the same +attempt reuse the resulting count without incrementing or persisting again. + +## Why + +The retry circuit is intended to protect a hard-affinity key after repeated +failures. A single upstream send observed through two local timeout paths is one +failure, not two. Durable conflict merging deliberately treats independent +persistence calls as independent failures, so deduplication must happen before +the durable write. + +A time-window or session-key dedupe would hide legitimate retries. A single +"last generation" marker is also insufficient because an observer for an older +send may resume after a newer send has started. A stable object per send keeps +old and new attempts distinct without adding a durable identifier or an +unbounded process-level set. + +## What Changes + +- Create a process-local attempt object immediately before every HTTP bridge + upstream `response.create` send. +- Disarm that attempt when the send fails or is cancelled, and mark it observed + when a matching upstream response lifecycle event wins the race. +- Capture the attempt at the moment a watchdog classifies a timeout, before any + pending-ownership, recovery, reconnect, or cleanup await can install a newer + attempt. +- Pass an explicit absent/eligible/recorded/settled/ineligible selection through + all retry-circuit failure funnels so ambiguous attribution cannot become an + unscoped strike. +- Atomically claim the attempt and increment the circuit under the existing + retry-circuit lock; only the first claim performs durable persistence. +- Let duplicate observers wait for settlement and then read the live circuit + count instead of caching a historical count on the attempt. +- Mark matched response lifecycle events on the attempt even when reasoning + prelude delivery and ordinary event accounting are deferred. +- Emit low-cardinality observability when a duplicate observer is suppressed. + +## Impact + +- One eventless send contributes at most one consecutive retry-circuit failure. +- A separately dispatched retry or replay remains a distinct eligible failure + and can open the circuit as the second strike. +- A delayed observer sees later independent failures or a successful clear in + the current circuit state without adding another strike. +- Ambiguous multi-pending cleanup fails safe by undercounting at that boundary, + never by creating an unattributed failure that could double-count a send. +- Existing circuit thresholds, cooldowns, error envelopes, account-health + handling, continuity guards, and durable conflict merging remain unchanged. +- No schema migration, runtime setting, or operator action is required. + +## Non-Goals + +- Determining or eliminating the upstream cause of an eventless send. +- Changing the eventless timeout, stream-idle timeout, retry threshold, or + cooldown durations. +- Adding cross-replica send-attempt identifiers. Only the active bridge owner + owns the upstream socket; a send after owner handoff is a new physical attempt. +- Changing replay eligibility, account selection, or continuity fail-closed + behavior. diff --git a/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/specs/responses-api-compat/spec.md b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..956988ce85 --- /dev/null +++ b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/specs/responses-api-compat/spec.md @@ -0,0 +1,95 @@ +# responses-api-compat Delta + +## ADDED Requirements + +### Requirement: HTTP bridge retry circuits count each upstream send attempt at most once + +For an HTTP Responses bridge request, multiple local failure observers that +classify the same upstream `response.create` send attempt MUST contribute at +most one consecutive retry-circuit failure and at most one durable failure +persistence operation. A separately dispatched retry or replay MUST be treated +as a new send attempt and MAY contribute the next eligible failure under the +existing retry-circuit policy. + +The proxy MUST capture the attempt being classified before awaiting recovery, +reconnection, settlement, or pending-request ownership that can dispatch a +newer attempt. When stale ownership is classified while holding the pending +lock, the classified request set and its attempt selection MUST come from that +same snapshot. A send attempt that is disarmed by send-failure or cancellation +cleanup, or that observes a matching upstream response lifecycle event before +its first failure claim, MUST NOT add a retry-circuit failure. A matched +response lifecycle event MUST mark its attempt observed even when downstream +delivery or ordinary response-event accounting is intentionally deferred. + +The proxy MUST distinguish a failure path with no attempt identity from one +whose attempt identity is present but ineligible or ambiguous. Only the former +MAY preserve legacy unscoped recording. An ineligible attempt or multiple +eligible candidates MUST NOT fall back to an unscoped failure. Duplicate +observers MUST wait for the first claim's settlement and then use the current +circuit count; they MUST NOT expose a cached historical count after a later +failure or successful clear. Deduplication MUST NOT change existing failure +classes, thresholds, cooldowns, continuity guards, or cross-replica conflict +merging. + +#### Scenario: reader and downstream watchdogs observe one eventless send + +- **GIVEN** one hard-affinity HTTP bridge `response.create` send remains eventless +- **AND** the upstream reader watchdog and downstream stream-idle watchdog both classify that send +- **WHEN** both observers report the retry-circuit failure +- **THEN** the circuit's consecutive failure count increases by exactly one +- **AND** the failure is durably persisted exactly once +- **AND** the default two-failure circuit does not open from that send alone + +#### Scenario: a separately dispatched retry is a second failure + +- **GIVEN** one send attempt has already contributed one retry-circuit failure +- **WHEN** a later retry or replay dispatches a new `response.create` and that attempt also fails eligibility checks +- **THEN** the new attempt contributes a second failure +- **AND** the existing threshold and cooldown behavior may open the circuit + +#### Scenario: a delayed old observer cannot count a newer attempt + +- **GIVEN** an observer captured attempt A before recovery dispatched attempt B +- **AND** attempt A has already contributed its failure +- **WHEN** the delayed observer resumes after attempt B is current +- **THEN** it does not increment or persist another failure for attempt A +- **AND** it does not mark attempt B as recorded +- **AND** it observes the current circuit count, including attempt B's independent failure + +#### Scenario: an upstream response wins the timeout race + +- **GIVEN** a watchdog is evaluating an eventless send attempt +- **WHEN** a matching upstream response lifecycle event is observed before the attempt's first failure claim +- **THEN** that attempt does not contribute a retry-circuit failure + +#### Scenario: a deferred reasoning prelude wins the timeout race + +- **GIVEN** a matched reasoning lifecycle event is held for deferred downstream delivery +- **AND** ordinary response-event accounting remains zero for that prelude +- **WHEN** an eventless failure observer evaluates the same send attempt +- **THEN** the attempt is already marked as response-observed +- **AND** it does not contribute or persist a retry-circuit failure +- **AND** deferred-delivery and downstream-visibility behavior remain unchanged + +#### Scenario: multiple pending attempts are ambiguous at a shared failure boundary + +- **GIVEN** a shared cleanup boundary contains multiple distinct eligible send attempts +- **WHEN** the boundary cannot attribute its failure to exactly one physical send +- **THEN** it does not fall back to an unscoped retry-circuit failure +- **AND** it does not mark any candidate attempt as recorded +- **AND** a later observer with an exact attempt identity can still record each genuine failure independently + +#### Scenario: pending-lock wait cannot replace the classified attempt + +- **GIVEN** stale cleanup captures attempt A for a request before acquiring pending ownership +- **AND** recovery installs attempt B while cleanup is waiting for the pending lock +- **WHEN** cleanup later records the classified failure +- **THEN** it retains attempt A's identity +- **AND** it does not mark attempt B as recorded + +#### Scenario: a cleared circuit is not recreated by a delayed duplicate + +- **GIVEN** a send attempt contributed a failure and a later successful terminal response cleared the circuit +- **WHEN** another observer of the old send attempt resumes +- **THEN** the old observer does not recreate or persist the cleared failure +- **AND** it receives the current circuit count of zero diff --git a/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/tasks.md b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/tasks.md new file mode 100644 index 0000000000..7d842dcc64 --- /dev/null +++ b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/tasks.md @@ -0,0 +1,28 @@ +## 1. Specification + +- [x] 1.1 Add the attempt-scoped retry-circuit requirement and race scenarios. +- [x] 1.2 Validate the change in strict mode. + +## 2. Implementation + +- [x] 2.1 Add the process-local HTTP bridge send-attempt state and lifecycle transitions. +- [x] 2.2 Capture and thread the classified attempt through all retry-circuit failure paths. +- [x] 2.3 Claim the attempt atomically with the circuit increment and suppress duplicate persistence. +- [x] 2.4 Add low-cardinality duplicate-suppression observability without new settings or schema. +- [x] 2.5 Represent absent, ineligible, and ambiguous attempt attribution explicitly; never use ambiguous `None` as an unscoped fallback. +- [x] 2.6 Read live circuit state after duplicate settlement and mark deferred response lifecycle events observed without changing delivery accounting. + +## 3. Coverage + +- [x] 3.1 Add a full HTTP bridge regression where reader and downstream watchdogs observe one send. +- [x] 3.2 Prove a new send is a new strike and a delayed observer of an old send is not. +- [x] 3.3 Cover response-wins, send-failure/cancellation, successful reset, and multiple-pending races. +- [x] 3.4 Preserve clean-close, continuity, owner-handoff, and durable conflict-merge coverage. +- [x] 3.5 Cover pending-lock attempt replacement, ambiguous selection suppression, deferred reasoning observation, and live-count changes after later failures or clear. + +## 4. Verification + +- [x] 4.1 Run focused HTTP bridge and durable retry-circuit tests. +- [x] 4.2 Run Ruff, formatting checks, and the full unit suite. +- [x] 4.3 Validate the OpenSpec change strictly and validate all main specs. +- [x] 4.4 Review the final diff for lock ordering, persistence cardinality, scope creep, and rollback compatibility. diff --git a/openspec/changes/disable-upstream-websocket-compression/proposal.md b/openspec/changes/disable-upstream-websocket-compression/proposal.md new file mode 100644 index 0000000000..c21121cdf2 --- /dev/null +++ b/openspec/changes/disable-upstream-websocket-compression/proposal.md @@ -0,0 +1,63 @@ +# Proposal: disable-upstream-websocket-compression + +## Why + +The direct-egress upstream websocket transport (`websockets.asyncio.client.connect` in +`app/core/clients/proxy_websocket.py`) passes no `compression` kwarg, so the websockets +library default (`compression="deflate"`) silently offers and negotiates `permessage-deflate` +with the upstream endpoint. No commit, spec, or comment ever chose this: the two sibling +upstream transports already run uncompressed — the routed aiohttp path uses aiohttp's +default `compress=0` (off), and the raw-handshake transport in `app/core/clients/proxy.py` +sets `compress=False`/`compress=0` explicitly. Per-frame zlib decode of high-rate upstream +event streams shows up as a measurable CPU leaf (~2.6% of profiled CPU) on the +single-weak-core proxy host, where CPU — not LAN/WAN bandwidth — is the scarce resource. + +## What Changes + +- Pass `compression=None` at the single direct-egress `websocket_connect(...)` callsite, + so upstream direct-egress websockets (Responses websocket and realtime live sideband, + which share the callsite) no longer offer `permessage-deflate` in the handshake. +- The client-facing socket is untouched: the existing normative requirement that the + server MUST continue to negotiate `permessage-deflate` on the client-facing websocket + (responses-api-compat, downstream ingress budget requirement) is unchanged, and uvicorn's + `ws_per_message_deflate` default stays enabled. +- The routed aiohttp path and raw-handshake transport are untouched (already uncompressed). + +## Owner-visible caveats + +- **Codex CLI fingerprint divergence**: codex-lb impersonates the Codex CLI persona on + the upstream handshake, and the native Codex CLI (tungstenite with + `DeflateConfig::default()`) DOES offer `permessage-deflate`. Dropping the + `Sec-WebSocket-Extensions` offer makes the direct path's handshake differ from the + native client. Mitigating evidence: the routed aiohttp path and the raw-handshake path + already send no extension offer today and are accepted in production, so extension + parity is not currently maintained anywhere. +- **Realtime live sideband shares the callsite**: the change applies to both + `_RESPONSES_WEBSOCKET_POLICY` and `_LIVE_SIDEBAND_WEBSOCKET_POLICY` sockets. Live + sideband frames (base64 audio) compress poorly anyway; if per-policy compression is + ever wanted, the kwarg can be lifted into `_UpstreamWebSocketPolicy`. +- **WAN ingress bandwidth** from the upstream increases (JSON event streams compress + ~4-8x); this is a cost trade only, not a correctness change. +- **Falsification test**: if a post-deploy profile still shows the `permessage_deflate` + decode leaf, the cost was downstream uvicorn decode (spec-protected, must keep) and + this change is a CPU no-op; revert is one line. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `responses-api-compat`: adds a requirement that direct-egress upstream websockets do + not offer `permessage-deflate`. The client-facing `permessage-deflate` MUST is + unchanged. + +## Impact + +- `app/core/clients/proxy_websocket.py`: one kwarg (`compression=None`) on the shared + direct-egress `websocket_connect` call; persona headers, subprotocols, ping-timeout + watchdog, max_size, and proxy resolution are unchanged. +- `tests/unit/test_proxy_websocket_client.py`: kwargs assertion pins + `compression is None` on the direct transport contract. diff --git a/openspec/changes/disable-upstream-websocket-compression/specs/responses-api-compat/spec.md b/openspec/changes/disable-upstream-websocket-compression/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..63af95bfd7 --- /dev/null +++ b/openspec/changes/disable-upstream-websocket-compression/specs/responses-api-compat/spec.md @@ -0,0 +1,22 @@ +## ADDED Requirements + +### Requirement: Direct-egress upstream websockets do not offer permessage-deflate +When codex-lb opens a direct-egress upstream websocket (the Responses websocket or the +realtime live sideband over the `websockets` transport, used when no upstream proxy route +applies), it MUST NOT offer the `permessage-deflate` extension in the upstream handshake, +matching the routed and raw-handshake upstream transports, which already run uncompressed. +This requirement applies only to the proxy-to-upstream link: the server MUST continue to +negotiate `permessage-deflate` on the client-facing websocket, as required by the +downstream websocket ingress requirement. + +#### Scenario: Direct upstream handshake omits the compression extension offer + +- **WHEN** codex-lb connects an upstream websocket via the direct-egress `websockets` transport +- **THEN** the handshake does not offer `permessage-deflate` (the transport is invoked with compression disabled) +- **AND** persona headers, subprotocols, open-timeout, ping-timeout, message-size cap, and proxy resolution are unchanged + +#### Scenario: Client-facing compression negotiation is unchanged + +- **WHEN** a client connects to a Responses websocket route offering `permessage-deflate` +- **THEN** the server still negotiates `permessage-deflate` on the client-facing socket +- **AND** the downstream ingress budget continues to apply to the decompressed message size diff --git a/openspec/changes/disable-upstream-websocket-compression/tasks.md b/openspec/changes/disable-upstream-websocket-compression/tasks.md new file mode 100644 index 0000000000..1db495f828 --- /dev/null +++ b/openspec/changes/disable-upstream-websocket-compression/tasks.md @@ -0,0 +1,13 @@ +## 1. Implementation + +- [x] 1.1 Pass `compression=None` at the direct-egress `websocket_connect` callsite in + `app/core/clients/proxy_websocket.py` so upstream direct-egress sockets stop + offering `permessage-deflate`. Leave the routed aiohttp path, raw-handshake + transport, and downstream uvicorn `ws_per_message_deflate` untouched. + +## 2. Validation + +- [x] 2.1 Extend the direct-transport kwargs assertions in + `tests/unit/test_proxy_websocket_client.py` with `compression is None`. +- [x] 2.2 Run the proxy websocket client unit suite, lint, and strict OpenSpec + validation. diff --git a/openspec/changes/document-compact-trigger-proxy-contract/proposal.md b/openspec/changes/document-compact-trigger-proxy-contract/proposal.md new file mode 100644 index 0000000000..665e9a6e2f --- /dev/null +++ b/openspec/changes/document-compact-trigger-proxy-contract/proposal.md @@ -0,0 +1,34 @@ +## Why + +PR 1749 changes two coupled parts of the Codex compact contract: trigger +canonicalization and the upstream transport used to obtain the compact result. +The upstream Codex Responses flow accepts a terminal `compaction_trigger` on +`POST /backend-api/codex/responses`; the legacy `/codex/responses/compact` +route can return 404. The proxy therefore needs an explicit transport contract, +including the retained `/v1` compatibility behavior, rather than relying on an +implementation detail that contradicts the existing context notes. + +## What Changes + +- Document that `POST /backend-api/codex/responses` terminal compaction + triggers produce exactly one terminal `compaction` item on the internal + compact wire. +- Document that malformed top-level trigger placement is rejected locally + before upstream compact handling. +- Document that Codex compact transport uses streamed `POST + /backend-api/codex/responses` with `stream=true` and `store=false`, and + reconstructs the compact response from the terminal SSE lifecycle. +- Document that legacy message-shaped compact output is converted to a + `compaction` item while only valid opaque `cmp_` IDs are preserved; malformed + IDs are omitted rather than rewritten. +- Document that the standalone Codex `/backend-api/codex/responses/compact` + route remains a compatibility endpoint, while `/v1/responses/compact` + preserves duplicate-trigger normalization for existing OpenAI-compatible + clients. + +## Impact + +- `responses-api-compat` change record only +- The existing compact transport guidance that requires direct + `/codex/responses/compact` without a surrogate is superseded for the Codex + Responses bridge by this explicit streamed `/codex/responses` contract. diff --git a/openspec/changes/document-compact-trigger-proxy-contract/specs/responses-api-compat/spec.md b/openspec/changes/document-compact-trigger-proxy-contract/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..a2ed56a334 --- /dev/null +++ b/openspec/changes/document-compact-trigger-proxy-contract/specs/responses-api-compat/spec.md @@ -0,0 +1,90 @@ +## MODIFIED Requirements + +### Requirement: Codex compaction triggers are bridged into compact output + +When `POST /backend-api/codex/responses` receives a request whose top-level `input` array contains exactly one `{"type":"compaction_trigger"}` item as its final element, the proxy SHALL remove that trigger before calling upstream compaction handling and SHALL emit a raw SSE stream that contains exactly one compaction output item. The internal compact request built for that flow MUST contain exactly one terminal `compaction_trigger` item on the compact wire, and the proxy MUST reject duplicate or non-terminal top-level `compaction_trigger` placement locally with HTTP 400 `invalid_request_error` before any upstream compact handling. + +The stream MUST emit `response.created`, `response.output_item.added`, `response.output_item.done`, and `response.completed` in that order with monotonically increasing sequence numbers. The added event MUST expose the selected compaction item as in progress. The done event and terminal completed response MUST carry the same terminal `compaction` item. When the selected encrypted upstream compaction item carries a valid `cmp_` ID or status, the synthetic stream MUST preserve those values with its `encrypted_content`; it MUST NOT generate or rewrite a replacement item ID. A malformed, empty, or non-`cmp_` ID MUST be omitted while the opaque encrypted content remains unchanged. + +Codex compact flows SHALL send the upstream compact request to `POST /backend-api/codex/responses` with `stream=true` and `store=false`, accept the upstream SSE response, and reconstruct one normalized compact response item from the terminal response lifecycle; they MUST NOT require the legacy `/backend-api/codex/responses/compact` upstream route to be available. + +For Codex-affinity standalone compact requests, `POST /backend-api/codex/responses/compact` SHALL remain available as a compatibility endpoint with its subscription-backed compact routing contract, and SHALL normalize an upstream remote-compaction-v2 response that includes historical message output plus a compaction summary into the single compact output item required by Codex clients. A valid upstream `cmp_` compaction item `id` and any non-empty `status` MUST be preserved in that normalized output item. An empty, non-string, or non-`cmp_` ID MUST be omitted rather than rewritten; encrypted content MUST remain unchanged. + +OpenAI-style `/v1/responses/compact` is otherwise unchanged by this requirement; when it receives duplicate top-level `compaction_trigger` items, codex-lb preserves the existing compatibility behavior and the forwarded compact input contains one terminal trigger. + +#### Scenario: terminal trigger emits a complete compact lifecycle + +- **WHEN** a `POST /backend-api/codex/responses` request ends with exactly one top-level `compaction_trigger` +- **THEN** the proxy strips the trigger and invokes compact handling +- **AND** it emits created, added, done, and completed events in that order +- **AND** their sequence numbers increase monotonically from zero +- **AND** the done event and completed response contain the same single terminal compaction item + +#### Scenario: terminal trigger becomes one compact-wire trigger + +- **WHEN** a `POST /backend-api/codex/responses` request ends with exactly one + top-level `compaction_trigger` +- **THEN** the proxy strips that trigger before compact-input preparation +- **AND** the internal compact request contains exactly one terminal + `compaction_trigger` item on its `input` array + +#### Scenario: encrypted compaction item identity survives trigger streaming + +- **WHEN** compaction handling for a terminal trigger returns encrypted content with a non-empty upstream `cmp_*` ID and terminal status +- **THEN** the added event exposes that ID with in-progress status +- **AND** the done event and completed response preserve the exact upstream ID, terminal status, and encrypted content +- **AND** the proxy does not synthesize a replacement item ID + +#### Scenario: malformed trigger placement is rejected + +- **WHEN** a `POST /backend-api/codex/responses` or + `POST /backend-api/codex/responses/compact` request contains duplicate or + non-terminal top-level `compaction_trigger` items +- **THEN** the proxy returns HTTP 400 with `invalid_request_error` +- **AND** it does not attempt upstream compact handling + +#### Scenario: Codex compact transport uses the Responses stream + +- **WHEN** a valid terminal compaction trigger is submitted through a Codex + compact flow +- **THEN** the proxy sends the compact request to + `POST /backend-api/codex/responses` with `stream=true` and `store=false` +- **AND** it accepts the upstream SSE response and reconstructs one normalized + compact response item from the terminal response lifecycle +- **AND** it does not require the legacy `/backend-api/codex/responses/compact` + upstream route to be available + +#### Scenario: Legacy message-shaped compact output does not get a rewritten item ID + +- **WHEN** the upstream compact response exposes the encrypted compact payload + as a legacy `message` item with a non-empty ID that does not begin with `cmp_` +- **THEN** the proxy converts that item to `type="compaction"` and omits the + malformed ID +- **AND** the proxy preserves the encrypted content unchanged +- **AND** an existing ID that begins with `cmp_` is preserved byte-for-byte +- **AND** the proxy does not synthesize a `cmp_msg_...` ID +- **AND** ordinary message items outside the compact-output conversion remain + unchanged + +#### Scenario: Standalone Codex compact remains a compatibility endpoint + +- **WHEN** a client calls `POST /backend-api/codex/responses/compact` +- **THEN** codex-lb preserves the endpoint and its subscription-backed compact + routing contract +- **AND** malformed duplicate or non-terminal top-level triggers are rejected + locally before any upstream compact attempt + +#### Scenario: Codex-affinity standalone compact normalizes remote v2 output + +- **WHEN** a Codex-affinity `POST /backend-api/codex/responses/compact` request receives upstream output that contains historical message items and one compaction summary item +- **THEN** the JSON response body contains exactly one `output` item for that compaction summary +- **AND** the normalized item preserves the compaction summary's valid `cmp_`-prefixed upstream ID and status +- **AND** it does not expose historical message items as standalone compact output + +#### Scenario: OpenAI-compatible compact normalizes duplicate triggers + +- **WHEN** a client calls `POST /v1/responses/compact` with duplicate + top-level `compaction_trigger` items +- **THEN** codex-lb preserves the existing compatibility behavior and returns + HTTP 200 when the compact operation succeeds +- **AND** the forwarded compact input contains one terminal trigger diff --git a/openspec/changes/document-compact-trigger-proxy-contract/tasks.md b/openspec/changes/document-compact-trigger-proxy-contract/tasks.md new file mode 100644 index 0000000000..65205563d9 --- /dev/null +++ b/openspec/changes/document-compact-trigger-proxy-contract/tasks.md @@ -0,0 +1,14 @@ +## 1. Document the contract + +- [x] 1.1 Add a focused `responses-api-compat` delta for the compact-trigger + proxy-routing contract. +- [x] 1.2 Cover exactly one terminal compact-wire `compaction_trigger` and + local malformed-placement rejection. +- [x] 1.3 Record the streamed `/backend-api/codex/responses` compact transport, + the standalone Codex compatibility endpoint, and the `/v1` normalization + asymmetry. +- [x] 1.4 Record the legacy message-shaped compact ID filtering contract. + +## 2. Validate the change + +- [x] 2.1 Run `openspec validate document-compact-trigger-proxy-contract --strict`. diff --git a/openspec/changes/expand-postgres-shm-and-pool-headroom/proposal.md b/openspec/changes/expand-postgres-shm-and-pool-headroom/proposal.md new file mode 100644 index 0000000000..efc23843ad --- /dev/null +++ b/openspec/changes/expand-postgres-shm-and-pool-headroom/proposal.md @@ -0,0 +1,48 @@ +# Expand PostgreSQL /dev/shm and default pool headroom + +## Why + +Two independent capacity ceilings surfaced on a production single-replica +PostgreSQL deployment: + +- The Compose `postgres` service runs with Docker's default 64MB `/dev/shm`. + PostgreSQL parallel workers (`work_mem=32MB`, + `max_parallel_workers_per_gather=2`) exchange spill files through dynamic + shared memory under `/dev/shm`, so parallel hash joins abort with + `could not resize shared memory segment ... No space left on device`, + which asyncpg surfaces as `DiskFullError` on the request path. +- The default SQLAlchemy pool (`database_pool_size=15`, + `database_max_overflow=10`, fixed 30s checkout timeout) exhausts under + slow-query pile-ups: once 25 request-path checkouts are held, every further + request waits 30 seconds and fails with + `QueuePool limit of size 15 overflow 10 reached, connection timed out`. + +## What Changes + +- The Compose `postgres` service sets `shm_size: 1gb`. +- Default `database_pool_size` rises 15 → 25 and `database_max_overflow` + 10 → 15, keeping the per-replica two-engine cap at + `(25 + 15) * 2 = 80` application connections — inside PostgreSQL's default + `max_connections=100` with at least 20 raw server slots reserved (same + reserve rule the Helm capacity guidance already mandates). +- Helm deployments are unaffected: the chart always injects its own + `CODEX_LB_DATABASE_POOL_SIZE` / `CODEX_LB_DATABASE_MAX_OVERFLOW` values, + and the bundled Bitnami PostgreSQL sub-chart already mounts a + memory-backed `/dev/shm` (`shmVolume.enabled=true` by default). + +## Capabilities + +### Modified Capabilities + +- `deployment-installation`: the Compose Postgres profile provisions a + `/dev/shm` large enough for parallel query. +- `database-backends`: default pool sizing preserves the raw-slot reserve on + PostgreSQL's default `max_connections`. + +## Impact + +- SQLite deployments: none (pool sizing applies to pooled backends only). +- Helm deployments: none (chart values override both settings). +- Compose/manual PostgreSQL deployments: applying `shm_size` requires the + postgres container to be recreated (seconds of downtime); per-replica + worst-case application connections rise from 50 to 80. diff --git a/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/database-backends/spec.md b/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/database-backends/spec.md new file mode 100644 index 0000000000..a5231ef1e9 --- /dev/null +++ b/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/database-backends/spec.md @@ -0,0 +1,27 @@ +## ADDED Requirements + +### Requirement: Default pool sizing preserves raw-slot reserve on default max_connections + +The default values of `database_pool_size` and `database_max_overflow` MUST +keep one replica's aggregate application connection capacity — +`(database_pool_size + database_max_overflow) * 2 pooled engines * 1 +supported worker` — at or below 80, so a single replica on PostgreSQL's +default `max_connections=100` retains at least 20 raw server slots for +PostgreSQL-reserved connections, the migration path's two-connection peak, +administration, and transient non-application clients. + +#### Scenario: Default single replica fits default max_connections + +- **WHEN** one replica runs with the default `database_pool_size` and + `database_max_overflow` +- **THEN** both pooled engines together cap at no more than 80 PostgreSQL + connections +- **AND** at least 20 raw server slots remain on a default + `max_connections=100` server + +#### Scenario: Operators can still tune the pool + +- **WHEN** `CODEX_LB_DATABASE_POOL_SIZE` or `CODEX_LB_DATABASE_MAX_OVERFLOW` + is set in the environment +- **THEN** the configured values override the defaults for both pooled + engines diff --git a/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/deployment-installation/spec.md b/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/deployment-installation/spec.md new file mode 100644 index 0000000000..05e880b54a --- /dev/null +++ b/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/deployment-installation/spec.md @@ -0,0 +1,22 @@ +## ADDED Requirements + +### Requirement: Compose Postgres service sizes /dev/shm for parallel query + +The Docker Compose `postgres` service MUST set an explicit `shm_size` of at +least 1GB. Docker's default 64MB `/dev/shm` causes PostgreSQL parallel +workers to fail with `could not resize shared memory segment ... No space +left on device` once a parallel hash join spills past the segment. + +#### Scenario: Compose postgres service pins shm_size + +- **WHEN** `docker-compose.yml` is inspected +- **THEN** the `postgres` service declares `shm_size` of at least 1GB + +#### Scenario: Parallel hash join spills past 64MB + +- **GIVEN** the Compose `postgres` service is running with the declared + `shm_size` +- **WHEN** a parallel hash join spills more than 64MB of build tuples into + dynamic shared memory +- **THEN** the query does not fail with `could not resize shared memory + segment` diff --git a/openspec/changes/expand-postgres-shm-and-pool-headroom/tasks.md b/openspec/changes/expand-postgres-shm-and-pool-headroom/tasks.md new file mode 100644 index 0000000000..558d9c804e --- /dev/null +++ b/openspec/changes/expand-postgres-shm-and-pool-headroom/tasks.md @@ -0,0 +1,17 @@ +## 1. Implementation + +- [x] 1.1 Add `shm_size: 1gb` to the Compose `postgres` service. +- [x] 1.2 Raise default `database_pool_size` to 25 and + `database_max_overflow` to 15, documenting the 80-connection / + 20-raw-slot budget at the setting definition. +- [x] 1.3 Regenerate `docs/reference/settings.md`. + +## 2. Regression coverage + +- [x] 2.1 Policy-test that the Compose `postgres` service pins `shm_size`. +- [x] 2.2 Update the settings default assertion to the new pool size. + +## 3. Validation + +- [x] 3.1 Run the compose, db-session, settings, and Helm artifact suites. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/openspec/changes/extend-rowless-rebase-to-durable-markers/design.md b/openspec/changes/extend-rowless-rebase-to-durable-markers/design.md deleted file mode 100644 index b37c273515..0000000000 --- a/openspec/changes/extend-rowless-rebase-to-durable-markers/design.md +++ /dev/null @@ -1,147 +0,0 @@ -# Design: durable-marker administrator semantic rebase - -## Authority and precedence - -The existing automatic durable-marker proof remains the first recovery path, -but it does not erase an earlier at-most-once decision. Before claiming the -marker, the service resolves any rowless authority for the same task/anchor or -exact request contract. Automatic proof may supersede only CAPTURED or -not-yet-dispatched APPROVED authority bound to the exact current marker. A -pre-marker or different-marker UNKNOWN/CONSUMED tombstone remains fail-closed. -The conflict lookup is bound to API-key scope plus the rejected anchor and is -therefore independent of optional incoming routing headers such as `thread-id`; -the stable root-task identity headers remain mandatory for creating or dispatching -administrator authority. -Only when automatic proof fails may the service capture an administrator -authority. Capture locks the origin `http_bridge_sessions` row and verifies its -exact API-key scope, account, recovery-required account, latest response hash, -marker anchor hash and empty marker-attempt claim. The new authority stores the -origin durable session ID without a cascading foreign key so its consumed -no-replay tombstone survives later bridge cleanup. - -Normal Codex root-task requests also carry a fresh per-turn -`x-codex-turn-state`. That value is routing affinity, not durable task identity. -Marker-backed capture and dispatch therefore accept it only after durable lookup -has resolved the alias to the exact hard session-header row that owns the active -marker. The stable session ID, prompt-cache key and thread ID must still be equal, -or, when an intermediary omits `thread-id`, the official Codex -`x-client-request-id` must supply the same root-task identity. When both headers -are present they must agree. A metadata session/thread ID, when present, is -corroborating evidence only and must match; it cannot create authority by -itself. Direct-header and body-nested turn-metadata carriers are parsed -independently so merge precedence cannot hide a conflict. The compatibility -header retains a 16 KiB bound. The body carrier has a separate bounded budget -because the official client deliberately removes its potentially large tool -namespace inventory from the direct header while retaining that inventory in -the request body; both carriers still pass the same closed-schema checks. -Malformed metadata, a non-turn request kind, or any explicit parent/subagent -signal in headers or client metadata rejects the fallback. A -child request remains ineligible because its client request/thread ID differs -from the shared root session and prompt-cache key. Conflicting session aliases -remain rejected. Missing-row recovery and child-thread recovery retain their -stricter no-turn-state identity gate. - -Codex 0.149 Responses-Lite serializes ordinary function/custom declarations -inside an `additional_tools` developer item using a `namespace` envelope and -may include a client-executed `tool_search` declaration. These are stateless -wire schema, not account resources. The replay classifier accepts only the -closed official shapes: a non-empty namespace containing function declarations -with required description/strict/parameters fields or custom declarations with -required description and non-null grammar-format fields, and the exact client-executed tool-search -object schema generated by 0.149. A deferred flag may be omitted or literal -`true`; malformed or false flags, unknown fields, nested namespaces, -server-executed search, hosted tools and account-scoped references remain -rejected. - -The same client version adds `session_id`, `thread_id` and `turn_id` to -`client_metadata`, and a normal root turn may also project `root_turn_id`. -These fields become account-neutral evidence only when -the canonical nested turn metadata contains the same complete identity and the -session/thread values equal the already verified session/prompt/task identity. -Body and direct-header carriers are parsed independently for both explicit -`thread-id` and fallback requests and must agree after removing only the -body-only tool namespace inventory; the direct carrier may never contain that -inventory. Flat root-turn, installation and window projections, when present, -must equal the canonical nested values. The nested object must use the -closed 0.149 turn shape, describe a turn, and carry no parent/subagent lineage, -unknown field or explicit file/container/vector-store state. Calls without an -expected verified identity continue to reject those identity keys. This is a -schema-compatibility correction, not a general same-account exception: all -three existing account-neutral capture, repository and approval gates remain. -The 0.149.0-alpha.4.1 Desktop also supplies the product-owned -`workspace_kind` string through `responsesapi_client_metadata`; Codex flattens -that value into both nested turn-metadata carriers. Recovery accepts only that -explicitly known extra as a nonblank UTF-8 string of at most 128 bytes. It -remains part of the shared body/direct projection fingerprint, so omission or -drift between carriers fails closed, while every other unknown flattened extra -continues to be rejected. - -The authority is an explicit `operator_acknowledged_semantic_rebase`. It does -not assert that the pending call was never executed. The complete client -checkpoint and trusted operator acknowledgement select the semantic state from -which the task will continue. - -Identity rejection observability records only presence/equality and recovery -predicate booleans. Optional capture-shape diagnosis is bounded to at most 512 -input items so repeated invalid requests cannot create unbounded projection work. - -## One marker generation, one winner - -Automatic exact-proof recovery writes the raw request-text digest into -`recovery_required_attempt_fingerprint`. Administrator recovery writes: - -``` -SHA256(canonical_json({ - domain: "qk_http_bridge_rowless_marker_attempt_v1", - authority_id, - generation, - wire_request_fingerprint -})) -``` - -Both paths lock and update the same durable marker row. The domains cannot be -mistaken for equal-wire idempotency. Administrator preflight commits the marker -claim and `APPROVED -> UNKNOWN` before account selection or WebSocket connect. -The later replacement-session journal remains UNKNOWN until terminal settlement. - -## Failure behavior - -A local setup failure before replacement binding restores APPROVED and clears -the administrator marker claim in one transaction. After journal creation, a -rollback requires an exact journal delete plus task-owner proof that the -initial send helper was never invoked; this also covers cancellation while the -send marker itself is committing. After the helper is invoked, the only -rollback authority is the typed transport result proving that every attempted -socket closed before its send primitive. Reconnect/setup failure before the -replacement send primitive uses that same physical proof. Cancellation is -deferred until the matching proven-unsent state has been durably restored. -The proof is a dedicated per-request state set only by the exact typed -closed-before-send result. A socket-only reconnect or a nonzero generic replay -counter is never physical non-delivery evidence. - -Any zero-event disconnect, generic transport failure, timeout or exception -after a send primitive may have delivered the request. It leaves the authority, -marker claim and journal UNKNOWN permanently. No generic replay path may use -the administrator authority. - -## Terminal and rollback floor - -For administrator recovery, `response.completed` locks authority, origin marker -and UNKNOWN journal, then atomically publishes the new response anchor, full -client checkpoint, response alias, REPLAYED journal and CONSUMED authority while -clearing all marker fields. Automatic recovery uses an equivalent transaction -over the exact owner/marker/journal generation, without an administrator -authority state transition. A persistence failure in either path is converted -to a terminal error before downstream success is delivered and leaves the old -anchor, marker claim and UNKNOWN journal intact. - -Startup schema readiness requires -`http_bridge_rowless_recovery_authorities.origin_marker_session_id`, not merely -the table name. Migration downgrade refuses while any marker-bound authority -exists, including CAPTURED. The trusted status endpoint reports marker-bound -state counts and `rowless_marker_recovery_v2`; an older v1-only image is not a -valid rollback target after the first marker-bound capture. - -Production recovery is serialized per affected task. The exact successor -source, immutable multi-platform digest, GitOps revision, workload image ID and -new rollback capability are read back before the first approval. diff --git a/openspec/changes/extend-rowless-rebase-to-durable-markers/proposal.md b/openspec/changes/extend-rowless-rebase-to-durable-markers/proposal.md deleted file mode 100644 index 497cdc0815..0000000000 --- a/openspec/changes/extend-rowless-rebase-to-durable-markers/proposal.md +++ /dev/null @@ -1,55 +0,0 @@ -# Proposal: extend rowless semantic rebase to durable recovery markers - -## Problem - -When upstream rejects a completed saved response anchor, the bridge persists a -durable recovery marker. Exact owner-bound full-resend proof can recover some -requests automatically, but a request whose retained tool-call evidence does -not exactly settle the durable pending manifest must remain fail-closed. The -current service then repeats a stable no-progress error indefinitely because -the existing administrator-approved semantic-rebase flow only accepts a -missing durable checkpoint. - -The marker proves the rejected anchor, owning account, stored checkpoint and -pending manifest. It does not prove whether an unmatched pending tool call had -an irreversible effect. Therefore this case needs an explicit operator -semantic rebase of the same durable generation, not weaker call-ID matching or -another automatic replay. - -## Change - -- Allow an active durable recovery marker to create one content-free rowless - authority only after automatic exact proof has failed and only for an exact, - self-contained, account-neutral root-task request with a settled retained - request ledger. -- Bind the authority to the exact durable session row, API-key scope, account, - rejected-anchor hash, task identity, request contract, projected payload and - final serialized wire. -- Keep capture local: it performs no new account selection, WebSocket connect - or upstream send and returns the existing administrator-authorization action. -- Accept the normal per-turn Codex turn-state affinity only after durable lookup - resolves it to the exact hard session-header row and active marker; preserve - the stricter identity gate for missing-row and child-thread recovery. -- Treat the official Codex 0.149 Responses-Lite namespace/tool-search envelope - as account neutral only when its schema is closed and stateless. Accept the - new session/thread/turn metadata keys only when session and thread values - exactly match the already verified root-task identity. -- Make administrator and automatic recovery attempts compete for the same - marker generation. The administrator claim uses a domain-separated digest - over authority ID, generation and wire fingerprint so legacy equal-wire - idempotency cannot admit both paths. -- Clear the marker attempt claim only after a physically proven unsent rollback - or the atomic terminal checkpoint. Ambiguous delivery remains UNKNOWN and is - never replayed. -- Expose marker-bound authority counts and the minimum rollback capability - `rowless_marker_recovery_v2`; fail startup when the required schema column is - absent and refuse schema downgrade while marker-bound authority exists. - -## Non-goals - -- Treating the marker as proof that an unmatched pending side effect did not occur. -- Relaxing pending call ID/type matching or PR #19's ambiguous-receive no-replay rule. -- Clearing an old anchor, marker or journal as an operational shortcut. -- Recovering multiple affected tasks concurrently. -- Treating arbitrary namespace tools, metadata drift, conversation/prompt - references, or file/container/vector-store state as account neutral. diff --git a/openspec/changes/extend-rowless-rebase-to-durable-markers/specs/responses-api-compat/spec.md b/openspec/changes/extend-rowless-rebase-to-durable-markers/specs/responses-api-compat/spec.md deleted file mode 100644 index 72a122ebcc..0000000000 --- a/openspec/changes/extend-rowless-rebase-to-durable-markers/specs/responses-api-compat/spec.md +++ /dev/null @@ -1,222 +0,0 @@ -# Delta: Responses API compatibility - -## ADDED Requirements - -### Requirement: Durable recovery markers support an explicit administrator semantic rebase - -When an active durable recovery marker cannot be recovered by exact owner-bound -automatic proof, the service MUST keep automatic safety predicates unchanged -and MAY capture one administrator-authorized semantic-rebase authority for the -same durable marker generation. - -#### Scenario: Automatic exact proof remains preferred - -- **GIVEN** an active durable recovery marker and a complete resend that exactly - settles its stored prefix and pending manifest -- **AND** no earlier rowless UNKNOWN or CONSUMED tombstone fences the same task turn -- **WHEN** the request is evaluated -- **THEN** the existing automatic proof path claims the marker generation -- **AND** no administrator authority is required or dispatched. - -#### Scenario: Automatic proof preserves earlier rowless no-replay tombstones - -- **GIVEN** an active durable recovery marker and an exact automatic full-resend proof -- **AND** a pre-marker or different-marker rowless UNKNOWN or CONSUMED authority - already fences the same task turn -- **WHEN** the automatic request is evaluated -- **THEN** it terminates locally before account selection, WebSocket connect or send -- **AND** the lookup remains enforced when `thread-id` or other dispatch-only - routing identity is missing but API scope and rejected anchor still identify the fence -- **AND** the earlier no-replay authority is not superseded by the current marker claim -- **AND** only CAPTURED or not-yet-dispatched APPROVED authority bound to the exact - current marker may be superseded by the automatic proof. - -#### Scenario: Mismatched pending evidence captures without upstream work - -- **GIVEN** the saved anchor was rejected before any response event -- **AND** the current complete request is self-contained and account neutral -- **BUT** its call/output IDs do not exactly settle the durable pending manifest -- **WHEN** automatic proof fails -- **THEN** the service persists one authority bound to the exact durable session, - API scope, account, anchor generation, task identity and request contract -- **AND** returns the stable authorization-required action -- **AND** performs no additional account selection, WebSocket connect or upstream send. - -#### Scenario: A resolved root-task turn-state alias can use the durable marker - -- **GIVEN** a normal Codex root-task resend carries a fresh per-turn - `x-codex-turn-state` affinity value -- **AND** durable lookup resolves that alias to the exact hard session-header, - account and active recovery marker -- **AND** the stable session ID and prompt-cache key equal the root task identity - supplied by `thread-id` or, when that header is omitted, by - `x-client-request-id` -- **AND** when both identity headers are present they agree, without a - conflicting session alias -- **AND** any metadata thread identity agrees and no header/body parent or - subagent signal is present -- **WHEN** automatic pending-manifest proof fails but the complete request is - otherwise eligible for marker-backed administrator recovery -- **THEN** the service captures or consumes the one marker-bound authority - instead of rejecting the request solely because turn-state affinity is present -- **AND** no-row recovery, child-thread recovery, unresolved aliases and task-ID - drift remain fail closed. - -#### Scenario: Client request identity cannot promote a child task - -- **GIVEN** a child Codex task shares its root session ID and prompt-cache key -- **AND** `thread-id` is omitted but `x-client-request-id` identifies the child -- **WHEN** the child submits a marker-backed or missing-row recovery request -- **THEN** the request remains ineligible because its client request identity - differs from the root session and prompt-cache key -- **AND** no authority, account selection, WebSocket connect or send occurs. - -#### Scenario: Metadata is corroboration, not replacement authority - -- **GIVEN** `thread-id` is omitted and the client request ID otherwise matches - the root session and prompt-cache key -- **WHEN** client metadata names a different thread or contains a parent/subagent - signal -- **THEN** marker-backed administrator recovery remains locally fail closed -- **AND** direct-header and body-nested turn metadata are checked independently - so conflicts, malformed values, oversized values and non-turn request kinds - cannot be hidden by metadata merge precedence -- **AND** metadata alone can never create a task authority. - -#### Scenario: Official Responses-Lite 0.149 schema remains stateless - -- **GIVEN** a complete root-task resend uses the official Responses-Lite - `additional_tools` developer prefix -- **AND** its tools contain only a closed namespace of validated - function/custom declarations plus an optional client-executed tool-search - declaration -- **AND** client metadata session and thread IDs exactly equal the already - verified session, prompt-cache and task identity while turn ID is nonblank -- **AND** canonical nested turn metadata contains the same complete identity, - uses the closed Codex 0.149 turn shape and agrees across body and direct-header - carriers -- **AND** an optional product-owned `workspace_kind` is a nonblank string of at - most 128 bytes and has the same value in both carriers -- **AND** a present Responses-Lite reasoning context is exactly `all_turns` -- **AND** any flat root-turn, installation or window projection equals the - canonical nested value -- **AND** a bounded body-only tool namespace inventory may exceed the 16 KiB - compatibility-header limit while a matching direct carrier omits that inventory -- **WHEN** account-neutral replay eligibility is evaluated -- **THEN** those schema and identity fields do not cause a false rejection -- **AND** no-row and marker-backed recovery retain the same self-contained, - settled-ledger and account-neutral authority gates. - -#### Scenario: Responses-Lite schema drift remains fail closed - -- **GIVEN** a request contains an empty or nested namespace, a namespace child - missing an official required field, an unknown tool field, a non-client or - malformed tool-search schema, a null or non-grammar namespace custom format, - a deferred flag other than omitted or literal - `true`, missing or drifting metadata identity, conflicting body/direct - metadata carriers, a direct carrier containing body-only tool inventory, - drifting flat root-turn/installation/window projection, parent/subagent - lineage, an empty, oversized or drifting `workspace_kind`, an unknown nested - metadata field, a null, blank, case-variant, `last_turn`, structured or - otherwise non-canonical reasoning context, - a conversation or prompt reference, or file/container/vector-store state -- **WHEN** recovery eligibility is evaluated -- **THEN** the request remains non-neutral and cannot create or consume a - semantic-rebase authority -- **AND** no account selection, WebSocket connect or upstream send occurs. - -#### Scenario: Invalid recovery metadata stops before routing - -- **GIVEN** a stale-anchor request supplies a verified root session, prompt-cache - key and thread identity -- **AND** at least one direct-header or body turn-metadata carrier is present -- **BUT** a carrier is malformed, oversized for its carrier kind or conflicts - with the other carrier -- **WHEN** no durable recovery authority has already resolved that anchor -- **THEN** the service returns a stable local invalid-metadata response -- **AND** creates no authority and performs no account selection, WebSocket - connect or upstream send -- **AND** a legacy request with no metadata carrier is not rejected by this gate. - -#### Scenario: Rowless projection removes only semantics-free Codex transport artifacts - -- **GIVEN** a complete client checkpoint whose direct call ledger is fully settled -- **AND** a tool output contains one exact empty `input_text` transport tail -- **AND** a canonical response-owned agent delivery contains one readable `input_text` - followed by one opaque `encrypted_content` transport sibling -- **WHEN** the marker-backed rowless authority is captured -- **THEN** the original item count and full-input fingerprint bind the unchanged client request -- **AND** the separately fingerprinted rowless projection removes only those exact - semantics-free parts while retaining function namespaces and all non-empty output -- **AND** reordered parts, extra fields, missing readable delivery, or any non-canonical - variant remains fail closed before account selection or upstream send. - -#### Scenario: Administrator and automatic claims are mutually exclusive - -- **GIVEN** one approved administrator authority for an active marker -- **WHEN** administrator and automatic recovery attempts race with even the same - raw wire fingerprint -- **THEN** both lock the same marker generation -- **AND** the administrator claim uses its domain-separated authority/generation digest -- **AND** exactly one path wins before account selection or connect -- **AND** the loser cannot create a second journal or upstream effect. - -#### Scenario: Proven-unsent failures restore the marker generation - -- **GIVEN** administrator preflight has claimed the marker generation -- **WHEN** local setup fails before replacement binding, the request owner is - cancelled before invoking its initial send helper, a typed first socket close - is followed by cancellation before the fresh send helper, or every attempted - socket is physically proven closed before its send primitive -- **THEN** one transaction restores APPROVED and clears the administrator marker claim -- **AND** deletes any exact UNKNOWN journal and replacement binding when present -- **AND** retains the old anchor, account and recovery-required marker. - -#### Scenario: Socket-only reconnect is not a proven-unsent send - -- **GIVEN** an administrator request reconnects a closed socket without sending -- **AND** the later initial send primitive may deliver the request -- **WHEN** that send is cancelled before any response event is observed -- **THEN** the generic reconnect counter MUST NOT authorize rollback -- **AND** authority, journal and marker claim remain UNKNOWN -- **AND** a later retry cannot produce a second upstream effect. - -#### Scenario: Ambiguous delivery remains permanently fenced - -- **GIVEN** the send primitive may have delivered the administrator request -- **WHEN** the stream fails without a physical unsent proof -- **THEN** authority and journal remain UNKNOWN -- **AND** the marker attempt claim and old anchor remain -- **AND** no automatic, reconnect or administrator replay is permitted. - -#### Scenario: Terminal completion atomically establishes the new checkpoint - -- **GIVEN** the one administrator semantic-rebase request reaches `response.completed` -- **WHEN** durable settlement commits -- **THEN** the new anchor, alias, complete client checkpoint, REPLAYED journal and - CONSUMED authority become visible together -- **AND** the recovery marker and marker attempt claim are cleared together -- **AND** any persistence failure leaves the old generation fail-closed. - -#### Scenario: Automatic terminal completion is also one durable transaction - -- **GIVEN** the automatic exact-proof path has claimed and dispatched one marker generation -- **WHEN** it reaches `response.completed` with a complete, supported pending - tool-call manifest -- **THEN** the new anchor, alias, complete client checkpoint and REPLAYED journal - become visible in the same transaction that clears every marker field -- **AND** an absent, malformed or unsupported terminal tool-call manifest is - returned as a persistence error without clearing the marker or settling the journal -- **AND** a transaction failure is returned as a terminal persistence error before - downstream success is delivered -- **AND** the old anchor, marker claim and UNKNOWN journal remain intact. - -#### Scenario: Schema and rollback capability preserve replay fences - -- **GIVEN** the rowless authority table exists -- **WHEN** its origin-marker column is absent -- **THEN** durable bridge startup readiness fails -- **AND** any marker-bound authority, including CAPTURED, requires - `rowless_marker_recovery_v2` -- **AND** migration downgrade and v1-only image rollback are rejected while such - authority exists. diff --git a/openspec/changes/extend-rowless-rebase-to-durable-markers/tasks.md b/openspec/changes/extend-rowless-rebase-to-durable-markers/tasks.md deleted file mode 100644 index 4b41b34a6d..0000000000 --- a/openspec/changes/extend-rowless-rebase-to-durable-markers/tasks.md +++ /dev/null @@ -1,22 +0,0 @@ -# Tasks - -- [x] Persist and migrate an optional non-cascading origin marker session binding. -- [x] Capture an exact marker-backed authority locally after automatic proof fails. -- [x] Domain-separate and atomically arbitrate administrator versus automatic marker claims. -- [x] Preserve same-account, exact-contract, actual-wire and at-most-once dispatch fences. -- [x] Atomically publish the replacement checkpoint and clear the marker at terminal completion. -- [x] Restore proven-unsent setup/send failures while retaining ambiguous UNKNOWN outcomes. -- [x] Add required-column readiness and marker-v2 rollback capability reporting. -- [x] Preserve legacy UNKNOWN/CONSUMED tombstones when automatic proof wins a newer marker. -- [x] Atomically settle automatic marker recovery before delivering terminal success. -- [x] Defer cancellation through the physically-proven-unsent two-socket rollback. -- [x] Canonicalize only exact empty-output and opaque agent transport artifacts in the rowless projection. -- [x] Resolve normal per-turn turn-state affinity to the exact durable marker before capture or dispatch. -- [x] Accept the official client request ID as the root-task identity only when `thread-id` is omitted and preserve child-task isolation. -- [x] Accept only the closed official Responses-Lite 0.149 namespace/tool-search schema and identity-bound client metadata. -- [x] Accept the bounded official Desktop `workspace_kind` turn metadata while preserving body/direct drift rejection. -- [x] Complete focused, migration, architecture, Ruff, format, type and strict OpenSpec gates. -- [ ] Obtain exact-final-diff Fable5 audit and independent approval on the successor PR. -- [ ] Build, mirror and verify the exact multi-platform image through normal GitOps. -- [ ] Recover affected tasks serially and prove new user, assistant and tool call/output progress. -- [ ] Complete the sustained clean production observation window. diff --git a/openspec/changes/fail-closed-draining-live-lease-claim/context.md b/openspec/changes/fail-closed-draining-live-lease-claim/context.md new file mode 100644 index 0000000000..245562dc38 --- /dev/null +++ b/openspec/changes/fail-closed-draining-live-lease-claim/context.md @@ -0,0 +1,18 @@ +Turn-state owner-forward already refuses takeover while a DRAINING owner +still holds a live lease. Durable `claim_session` used a weaker predicate: +`DRAINING` or `CLOSED` counted as `state_allows_takeover`, so +`allow_takeover=False` was ignored. `_http_bridge_allow_durable_takeover` +had the same hole and fed local session create. + +The live-owner check already exists: `_durable_bridge_lookup_active_owner` +returns None for closed, missing owner, or expired lease. Claim and local +create now use that rule. Forced recovery after a missing ring endpoint +must not override a live DRAINING owner. The locked claim row is the +source of that decision, so a stale ACTIVE lookup cannot authorize a +steal after the owner has started draining. CLOSED, expired, and +ownerless DRAINING rows stay recoverable after the draining replica +finishes or dies. + +Example: instance A marks its session DRAINING during preStop with 60s left +on the lease. Instance B claims the same session key with +`allow_takeover=False`. Owner stays A and state stays DRAINING. diff --git a/openspec/changes/fail-closed-draining-live-lease-claim/proposal.md b/openspec/changes/fail-closed-draining-live-lease-claim/proposal.md new file mode 100644 index 0000000000..768a7d929e --- /dev/null +++ b/openspec/changes/fail-closed-draining-live-lease-claim/proposal.md @@ -0,0 +1,26 @@ +# Why + +Shutdown marks durable HTTP-bridge rows `DRAINING` before releasing the lease +so the owner can finish an in-flight turn. A foreign +`claim_live_session(..., allow_takeover=False)` still steals that row because +`DRAINING` is treated as takeover-eligible even while the lease is live. +Turn-state owner-forward already fails closed. Local create and durable claim +do not, so two replicas can own one session during rolling drain. + +# What Changes + +- Treat a live DRAINING lease like a live ACTIVE lease for foreign claims. +- Align `_http_bridge_allow_durable_takeover` with the turn-state helper. +- Keep expired, released, and CLOSED rows takeover-eligible. + +# Capabilities + +### Modified Capabilities + +- `responses-api-compat`: durable claim and local session create must fail + closed on a live DRAINING lease. + +# Impact + +Turn-state forward-failure 503 stays as it is. Expired or ownerless DRAINING +rows can still be taken over after the draining owner releases or lapses. diff --git a/openspec/changes/fail-closed-draining-live-lease-claim/specs/responses-api-compat/spec.md b/openspec/changes/fail-closed-draining-live-lease-claim/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..292e2f49d4 --- /dev/null +++ b/openspec/changes/fail-closed-draining-live-lease-claim/specs/responses-api-compat/spec.md @@ -0,0 +1,39 @@ +## ADDED Requirements + +### Requirement: Live DRAINING durable leases reject foreign claims + +When a durable HTTP-bridge session is `DRAINING` and another instance still holds an unexpired lease, a foreign `claim_live_session` MUST leave the current owner and lease unchanged even when `allow_takeover` is true. Local session create MUST use the same live-owner predicate as turn-state takeover and MUST NOT treat `DRAINING` alone, or a forced recovery after a missing ring endpoint, as permission to steal a live `DRAINING` lease. The locked claim row, not a stale pre-claim lookup, MUST be the source of the `DRAINING` decision. Expired, released, or `CLOSED` rows MUST remain takeover-eligible. + +#### Scenario: Foreign claim refuses a live DRAINING lease + +- **GIVEN** instance A owns a durable session whose state is `DRAINING` +- **AND** A's lease is still unexpired +- **WHEN** instance B claims the same key with `allow_takeover` false +- **THEN** the row owner remains A +- **AND** the row stays `DRAINING` +- **AND** A's lease expiry is unchanged + +#### Scenario: Forced claim still refuses after an ACTIVE lookup becomes live DRAINING + +- **GIVEN** instance A owns a durable session whose lookup snapshot is still `ACTIVE` +- **AND** instance B would force takeover because A's endpoint is missing +- **AND** A marks the row `DRAINING` with a live lease before B's claim lock +- **WHEN** B claims the same key with `allow_takeover` true +- **THEN** the row owner remains A +- **AND** the row stays `DRAINING` + +#### Scenario: Missing owner endpoint does not force-steal a live DRAINING lease + +- **GIVEN** instance A owns a durable session whose state is `DRAINING` +- **AND** A's lease is still unexpired +- **AND** the ring cannot resolve A's endpoint +- **WHEN** instance B creates a local HTTP-bridge session for the same key +- **THEN** the durable claim is issued with `allow_takeover` false +- **AND** A's owner and lease remain unchanged + +#### Scenario: Expired DRAINING row remains takeover-eligible + +- **GIVEN** a `DRAINING` durable session whose lease is expired or whose owner is released +- **WHEN** another instance claims the same key +- **THEN** that instance becomes the owner +- **AND** the row becomes `ACTIVE` diff --git a/openspec/changes/fail-closed-draining-live-lease-claim/tasks.md b/openspec/changes/fail-closed-draining-live-lease-claim/tasks.md new file mode 100644 index 0000000000..eeb4e55cd6 --- /dev/null +++ b/openspec/changes/fail-closed-draining-live-lease-claim/tasks.md @@ -0,0 +1,27 @@ +## 1. Implementation + +- [x] 1.1 Refuse foreign `claim_session` when the row is DRAINING and the + lease is still live. +- [x] 1.2 Align `_http_bridge_allow_durable_takeover` with the live-owner + turn-state helper. +- [x] 1.3 Mask forced local recovery so a missing ring endpoint cannot + steal a live DRAINING lease. +- [x] 1.4 Refuse live DRAINING on the locked claim row even when + `allow_takeover` is true. + +## 2. Regression coverage + +- [x] 2.1 Assert `claim_live_session(allow_takeover=False)` after + `mark_instance_draining` keeps the original owner. +- [x] 2.2 Assert `_http_bridge_allow_durable_takeover` is false for a live + DRAINING lookup and true for expired or released DRAINING. +- [x] 2.3 Assert get-or-create claims with `allow_takeover` false when the + durable lookup is live DRAINING and the owner endpoint is missing. +- [x] 2.4 Assert get-or-create does not steal when an ACTIVE lookup becomes + live DRAINING before the locked claim. + +## 3. Validation + +- [x] 3.1 Run the new claim and helper tests plus the existing turn-state + fail-closed tests. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/openspec/changes/fence-successor-bridge-claims/proposal.md b/openspec/changes/fence-successor-bridge-claims/proposal.md new file mode 100644 index 0000000000..28610369db --- /dev/null +++ b/openspec/changes/fence-successor-bridge-claims/proposal.md @@ -0,0 +1,24 @@ +## Why + +Root cause of the #1695 CI flake — `POST /v1/responses` intermittently returned 409 `bridge_instance_mismatch` on a **single-instance** deployment, four times across four unrelated PRs in three days. + +When an upstream WebSocket closes cleanly, the retiring bridge session's teardown releases its durable row while the next request is already creating a successor session and claiming the same row. Two defects let that race corrupt the claim: + +1. **The fence did not distinguish predecessor from successor.** A same-owner reclaim kept the owner epoch, so the retiring session's fenced `release_session` (carrying that same epoch) still matched after the successor's claim and closed the row out from under it. Historically the epoch was kept because reused sessions re-claimed; today a reused session renews instead, so every claim comes from a successor in-memory session and there is no caller that needs epoch stability across claims. +2. **The claim's write was not authoritative.** The update mutated ORM attributes, and SQLAlchemy omits fields whose values match the transaction's read. On SQLite (`with_for_update` is a no-op) a release could commit between the claim's SELECT and its write; the claim then wrote only lease/timestamp fields, the release's `owner=None, state=CLOSED` survived, and the post-commit refresh handed the claimant a closed, ownerless row — surfaced to the client as the 409. + +## What Changes + +- Every claim of an existing durable row advances the owner epoch, including same-owner reclaims, so the predecessor's outstanding fenced release/renewals no-op after the successor's claim. +- The claim's update is an explicit `UPDATE` statement that sets every ownership field unconditionally, so a write interleaved between the claim's read and its commit cannot survive into the claim's result. +- Foreign-claim rejection semantics (live DRAINING, fail-closed lookups) are unchanged. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: durable bridge claims fence out the predecessor session and are authoritative over interleaved writes. diff --git a/openspec/changes/fence-successor-bridge-claims/specs/responses-api-compat/spec.md b/openspec/changes/fence-successor-bridge-claims/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..1f9022fb61 --- /dev/null +++ b/openspec/changes/fence-successor-bridge-claims/specs/responses-api-compat/spec.md @@ -0,0 +1,87 @@ +## ADDED Requirements + +### Requirement: Durable bridge claims fence out the retiring predecessor + +A successful `claim_live_session` over an existing durable row MUST advance the owner epoch, including when the claiming instance already owns the row, so fenced updates issued by the predecessor local session — its release and any outstanding renewals — no-op after the claim instead of racing the successor. The claim's write MUST be authoritative: it MUST set every ownership field (owner, process epoch, owner epoch, lease, state, account) unconditionally, so a concurrent write committing between the claim's read and its commit cannot survive into the claim's result. A claim that returns successfully MUST reflect the claimant as the live owner. A session creator that has already lost its inflight registry slot MUST NOT claim the durable row at all: claiming would advance the epoch past the session that won and fence that winner's own renewals out of a row it legitimately owns. A creator that nonetheless fails to register — because another session already holds the registry slot for its key, or because a replacement creator holds the in-flight slot and has not published its session yet — MUST hand the epoch it claimed to that registered session when both point at the same row and selected the same account. When the registered session selected a DIFFERENT account it no longer shares the row — the claim rewrote the row's account binding and cleared its continuity aliases — so the creator MUST release the row rather than preserve it, letting that session be fenced promptly instead of dispatching against a row bound elsewhere, so the winner's renewals keep matching, and MUST NOT release the durable row. Independently of that handoff, a renewal fenced by an epoch advance from THIS process — matching the instance ID, the owner process epoch, and the row's account binding — MUST adopt the newer epoch when the renewing session still holds the registry slot for its key — that is a superseded creator, not an ownership loss — while a session whose slot a different local session holds MUST still be evicted with the existing retryable instance-mismatch contract: it claimed last, so its epoch is current and its fenced release would otherwise close the row out from under the registered session. Concurrent claims over the same row MUST serialize on the epoch: the write MUST land only if the epoch still matches the claim's read, and a losing claim MUST retry against fresh state (within a bounded budget), so two claimants can never hold colliding fences. A claim that loses to a concurrent writer MUST revalidate its takeover permission against the fresh read rather than reusing the caller's pre-claim decision, so a loser cannot steal the winner's now-live lease; a live foreign owner then fails closed and the real owner is reported. A caller retrying a claim MUST NOT restore takeover permission against a live foreign owner either, so the fail-closed outcome survives the retry rather than being undone by a fresh claim. The snapshot a claim returns MUST be the state that claim itself wrote — not a post-commit re-read, which a later claim's commit could have already overwritten with its own epoch. + +#### Scenario: A successor claim fences the predecessor's release + +- **GIVEN** a retiring bridge session and a successor session claiming the same durable row on the same instance +- **WHEN** the successor's claim commits before the predecessor's release lands +- **THEN** the release is fenced out by the advanced epoch and the row stays ACTIVE and owned by the instance + +#### Scenario: A release committing mid-claim does not corrupt the claim + +- **GIVEN** the predecessor's release commits between the successor claim's read and its write +- **WHEN** the claim commits +- **THEN** the claim's result reflects the claimant as the live owner with the advanced epoch +- **AND** the request proceeds instead of failing with `bridge_instance_mismatch` + +#### Scenario: Racing successor claims cannot share an epoch + +- **GIVEN** two successor claims that both read the same owner epoch before either writes +- **WHEN** both commit +- **THEN** they land on distinct epochs, with the loser retrying against fresh state + +#### Scenario: A losing claimant does not steal the winner's lease + +- **GIVEN** two replicas recovering the same released row, both permitted to take over +- **WHEN** one wins and the other re-reads the winner's now-live lease +- **THEN** the loser fails closed and reports the winner as owner instead of claiming the row +- **AND** the caller does not retry the claim with takeover permission against that live owner + +#### Scenario: A rejected creator leaves the registered winner's row alone + +- **GIVEN** an inflight waiter was evicted and a replacement session won the registry slot +- **WHEN** the stale creator finishes creating its session +- **THEN** it does not claim the durable row, so the winner's epoch is untouched +- **AND** it closes its own session without releasing the durable row, leaving the winner's row live +- **AND** if it had already claimed (eviction landing during the claim), the winner adopts that epoch so its renewals keep matching + +#### Scenario: The registered session adopts a same-instance epoch advance + +- **GIVEN** a session still holding the registry slot for its key whose durable row was advanced by this instance +- **WHEN** its lease renewal is fenced by that newer epoch +- **THEN** it adopts the epoch and keeps renewing instead of being evicted + +#### Scenario: A newer process incarnation still fences the predecessor + +- **GIVEN** two process incarnations sharing a configured instance ID across a graceful restart +- **WHEN** the successor claims and the predecessor's session renews +- **THEN** the predecessor is evicted rather than adopting the successor's epoch + +#### Scenario: An advance that rebound the row to another account is not adopted + +- **GIVEN** a registered session whose durable row was advanced and rebound to a different account +- **WHEN** its renewal is fenced by that advance +- **THEN** it is evicted rather than adopting the epoch, so it never dispatches on the other account's row + +#### Scenario: A session that lost its slot is still evicted + +- **GIVEN** a session whose registry slot is now held by a different local session +- **WHEN** its renewal is fenced +- **THEN** it is evicted and the retryable instance-mismatch error is raised + +#### Scenario: A replacement that has not published yet is still the winner + +- **GIVEN** a replacement creator holds the in-flight slot and has claimed but not yet registered its session +- **WHEN** the stale creator fails and settles +- **THEN** it does not release the durable row, which the replacement is about to publish against + +#### Scenario: A row rebound away from the winner is released + +- **GIVEN** a registered winner on one account and a stale creator whose claim rebound the row to another +- **WHEN** the stale creator settles +- **THEN** it releases the row instead of preserving it, so the winner is fenced promptly rather than dispatching against a row bound elsewhere + +#### Scenario: A sole creator still releases its row + +- **GIVEN** a failed creator with no registered session and no replacement in flight +- **WHEN** it settles +- **THEN** it releases the durable row rather than leaking it + +#### Scenario: Foreign-claim rejection is unchanged + +- **GIVEN** a durable row owned by another instance with a live lease +- **WHEN** a claim without takeover permission runs +- **THEN** the owner and lease remain unchanged, as before diff --git a/openspec/changes/fence-successor-bridge-claims/tasks.md b/openspec/changes/fence-successor-bridge-claims/tasks.md new file mode 100644 index 0000000000..7239fc83c5 --- /dev/null +++ b/openspec/changes/fence-successor-bridge-claims/tasks.md @@ -0,0 +1,29 @@ +## 1. Fix + +- [x] 1.1 `claim_session` advances the owner epoch on every claim of an existing row, including same-owner reclaims +- [x] 1.2 The claim's update path writes all ownership fields through an explicit `UPDATE` instead of ORM attribute mutation +- [x] 1.2b The insert path builds its own snapshot too, so a concurrent advance cannot hand two claimants the same fence +- [x] 1.3 The update is a compare-and-set on the epoch read, so racing claims serialize instead of sharing a fence; the loser retries against fresh state + +- [x] 1.4 Contended retries drop takeover permission, at the repository and at the service's claim retry +- [x] 1.5 A creator that has lost its inflight slot aborts before claiming; one that fails to register anyway closes its session without releasing the durable row + +## 2. Tests + +- [x] 2.1 Same-owner reclaim advances the epoch, and a predecessor release fenced on the old epoch no-ops (row stays ACTIVE and owned) +- [x] 2.2 Deterministic interleave reproduction: a release committing between the claim's SELECT and its write does not survive into the claim's result (fails on the pre-fix code) +- [x] 2.3 Racing successor claims land on distinct epochs (deterministic competitor injection) +- [x] 2.3b A CAS loser does not steal a foreign winner's live lease (fails without revalidation) +- [x] 2.3c The service's claim retry stops at a live foreign owner instead of restoring takeover +- [x] 2.3d A rejected creator does not release the registered winner's durable row +- [x] 2.3e A creator superseded mid-claim hands its epoch to the registered winner; unrelated rows are untouched +- [x] 2.3f A fenced renewal adopts a same-instance advance when registered, and still evicts when a different session holds the slot +- [x] 2.3g A replacement holding only the in-flight slot is protected; a sole creator still releases +- [x] 2.3h A newer process incarnation sharing the instance ID still fences the predecessor out +- [x] 2.3i Neither adoption nor handover crosses an account change; the session is evicted instead +- [x] 2.4 Route-level regression through POST /v1/responses: captive predecessor release lands late and is fenced out +- [x] 2.5 Existing claim/takeover suites pass unchanged (DRAINING rejection, account-change fencing, process-epoch semantics) + +## 3. Spec + +- [x] 3.1 Add the successor-fencing and authoritative-write requirement to `responses-api-compat` diff --git a/openspec/changes/file-pin-does-not-rewrite-thread-locality/.openspec.yaml b/openspec/changes/file-pin-does-not-rewrite-thread-locality/.openspec.yaml new file mode 100644 index 0000000000..0c73c8f54e --- /dev/null +++ b/openspec/changes/file-pin-does-not-rewrite-thread-locality/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-15 diff --git a/openspec/changes/file-pin-does-not-rewrite-thread-locality/context.md b/openspec/changes/file-pin-does-not-rewrite-thread-locality/context.md new file mode 100644 index 0000000000..65772f6eba --- /dev/null +++ b/openspec/changes/file-pin-does-not-rewrite-thread-locality/context.md @@ -0,0 +1,14 @@ +## Purpose + +Stop a live file pin from rewriting current-Codex thread locality. + +## Decision + +The required owner bypasses the thread PROMPT_CACHE row the same way +it already bypasses the process-session soft row. + +## Example + +Upload pins `file_xyz` to account A. Thread `t1` is already mapped to +account B. A Responses turn that references `file_xyz` goes to A; the +`t1` row remains B. The next unpinned `t1` turn still uses B. diff --git a/openspec/changes/file-pin-does-not-rewrite-thread-locality/design.md b/openspec/changes/file-pin-does-not-rewrite-thread-locality/design.md new file mode 100644 index 0000000000..08dff38f7f --- /dev/null +++ b/openspec/changes/file-pin-does-not-rewrite-thread-locality/design.md @@ -0,0 +1,42 @@ +## Context + +`#1521` made file pins durable hard ownership and bypassed the +process-session soft row. `#1703` then made `thread_header` / +PROMPT_CACHE the current Codex soft mapping. The bypass was not +updated, so a required file owner still enters sticky persist and +upserts thread B→A. + +## Goals / Non-Goals + +**Goals:** + +- File-pinned routing stays on the pin account. +- An existing thread PROMPT_CACHE row is not rewritten. +- Process-session seed remains insert-if-absent. + +**Non-Goals:** + +- Changing 1011 file-pin reconnect. +- Weakening file-pin fail-closed or hard-owner conflict checks. +- Dashboard or settings changes. + +## Decisions + +- Null the writable sticky key for both `session_header` and + `thread_header` in `preferred_owner_sticky_inputs`. Selection then + takes the unbound required-owner path. +- Keep `legacy_sticky_key` so a conflicting raw process-session owner + still fail-closes. +- Leave `sticky_seed_key` to the caller so a missing process + preference can still initialize without writing the thread row. + +**Alternative considered:** persist the thread row onto the file +owner so later unpinned turns stay there. Rejected: the file pin is +hard only for this turn; thread locality is a separate soft mapping. + +## Risks / Trade-offs + +- [Risk] A later unpinned turn on the same thread stays on the + pre-file account and cannot see the upload. → Mitigation: that is + the existing unpinned-file compatibility path; the pin still binds + any turn that references the file. diff --git a/openspec/changes/file-pin-does-not-rewrite-thread-locality/proposal.md b/openspec/changes/file-pin-does-not-rewrite-thread-locality/proposal.md new file mode 100644 index 0000000000..80167449d5 --- /dev/null +++ b/openspec/changes/file-pin-does-not-rewrite-thread-locality/proposal.md @@ -0,0 +1,36 @@ +## Why + +A live `input_file.file_id` pin is hard ownership. After thread-scoped +affinity, current Codex locality is the `thread_header` PROMPT_CACHE +row, but `preferred_owner_sticky_inputs` only bypasses +`session_header`. A file-pinned Responses turn therefore rewrites the +thread mapping to the upload account, so later unpinned turns follow +the file owner. + +## What Changes + +- Treat `thread_header` as the current-Codex soft row that a resolved + file/response/bridge owner must bypass. +- Keep consulting the raw process-session compatibility row for hard + conflicts. +- Keep process-session seed insert-if-absent. Do not write or rebind + the thread row on the required-owner path. +- Keep explicit `turn_state` as hard ownership. + +## Capabilities + +### New Capabilities + +- None. + +### Modified Capabilities + +- `sticky-session-operations`: A resolved file-pin owner MUST be + selected without consulting or rewriting the thread-scoped soft + mapping. + +## Impact + +- `app/modules/proxy/affinity.py` preferred-owner sticky inputs. +- Focused selection tests. +- No API, schema, setting, dashboard, or wire-format change. diff --git a/openspec/changes/file-pin-does-not-rewrite-thread-locality/specs/sticky-session-operations/spec.md b/openspec/changes/file-pin-does-not-rewrite-thread-locality/specs/sticky-session-operations/spec.md new file mode 100644 index 0000000000..3ec60a18b6 --- /dev/null +++ b/openspec/changes/file-pin-does-not-rewrite-thread-locality/specs/sticky-session-operations/spec.md @@ -0,0 +1,21 @@ +## ADDED Requirements + +### Requirement: File-pin required owner does not rewrite thread locality + +A resolved live `input_file.file_id` pin MUST be selected as the required owner without consulting or rewriting the current-Codex thread-scoped soft mapping. The process-session compatibility row MAY still be consulted as independent hard ownership. If that raw row conflicts with the pin account, the request MUST fail closed. A missing process-session preference MAY still initialize insert-if-absent. + +#### Scenario: File-pinned request owner overrides thread locality + +- **GIVEN** a request carries a `thread-id` whose bounded mapping points to account A +- **AND** its `input_file.file_id` is durably pinned to account B +- **WHEN** the request is routed +- **THEN** account B is treated as the required owner +- **AND** the thread mapping is neither consulted as an owner nor rewritten + +#### Scenario: File pin still conflicts with a raw process-session owner + +- **GIVEN** a raw process-session `codex_session` row points to account A +- **AND** a live file pin points to account B +- **WHEN** the request is routed +- **THEN** the service fails with `continuity_owner_conflict` before upstream dispatch +- **AND** neither the raw row nor the thread row is rewritten diff --git a/openspec/changes/file-pin-does-not-rewrite-thread-locality/tasks.md b/openspec/changes/file-pin-does-not-rewrite-thread-locality/tasks.md new file mode 100644 index 0000000000..c016d94535 --- /dev/null +++ b/openspec/changes/file-pin-does-not-rewrite-thread-locality/tasks.md @@ -0,0 +1,19 @@ +## 1. Implementation + +- [x] 1.1 Bypass the writable `thread_header` sticky key in + `preferred_owner_sticky_inputs` the same way as `session_header`. + +## 2. Regression coverage + +- [x] 2.1 Assert preferred-owner selection nulls the thread sticky key + and keeps the process seed / raw legacy key. +- [x] 2.2 Assert an existing thread row is not upserted when a file + pin is the required owner. +- [x] 2.3 Cover the same file-pin plus existing-thread case through + `/backend-api/codex/responses`, including the later unpinned + thread turn and process-seed sibling. + +## 3. Validation + +- [x] 3.1 Run the focused selection tests. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/openspec/changes/fix-compact-previous-response-quota-failover/proposal.md b/openspec/changes/fix-compact-previous-response-quota-failover/proposal.md new file mode 100644 index 0000000000..9194aafe58 --- /dev/null +++ b/openspec/changes/fix-compact-previous-response-quota-failover/proposal.md @@ -0,0 +1,47 @@ +# Fix Compact Previous-Response Quota Failover + +## Why + +When a long conversation's previous-response owner account is quota-excluded and the next +client action is a compaction, the compact request wedges the session. The compact path pins +selection to the resolved owner (`fallback_on_preferred_account_unavailable` is false whenever +a pin exists) and raises the selection failure straight to the client (`429 +usage_limit_reached` / 503, or the owner's in-request 429). Because the pin re-resolves the +same exhausted owner on every retry, the client cannot compact — and cannot shrink its history +to continue — until the owner's quota window resets. + +Normal turns already escape exactly this state through account-neutral fresh replay (strip the +stale `previous_response_id` anchor, verify the payload is a self-contained account-neutral +full resend, exclude the dead owner, reselect). This change gives the compact surface the same +selection-time recovery, rescoped per the maintainer review on PR #1490: activation only for +`previous_response_id`-only pins over self-contained histories, reusing the existing +account-neutral fresh-replay gates, gated on quota-caused owner exclusion, with no +continuity-rebind/CAS/fencing machinery. + +## What Changes + +- When a compact request is pinned **only** by `previous_response_id` (no turn-state owner, no + input-file owner, no session identity on the request, no session-ownership affinity) and + account selection cannot return the pinned owner, the proxy attempts account-neutral + fresh-replay recovery instead of failing: it verifies the anchor-free upstream compact + payload against the shared account-neutral fresh-replay rules plus the retained-prior-output + transcript shape, and on success removes `previous_response_id`, strips downstream + session/turn affinity aliases from upstream-bound headers, excludes the unavailable owner, + and reselects among the remaining eligible accounts. +- Recovery activates only for quota-caused owner loss: the owner's persisted status is + `RATE_LIMITED`/`QUOTA_EXCEEDED` at selection time, or the owner was excluded mid-request by + a pre-visible quota/rate-limit failover. Post-selection authentication, refresh, transport, + and transient exclusions keep their existing owner-bound surfaces. +- Every request outside the gate keeps today's fail-closed behavior, now also recorded on the + existing `continuity_fail_closed` observability counter (surface `compact`, reason + `owner_account_unavailable`). + +## Impact + +- Affected specs: `responses-api-compat` (one added requirement). +- Affected code: `app/modules/proxy/_service/compact.py` only (recovery branch in the account + selection loop, a payload-verification helper reusing `app/modules/proxy/replay_safety.py` + and `app/modules/proxy/continuity.py`, and an owner-status quota check). +- No new settings, endpoints, schemas, durable-bridge methods, or dashboard surfaces. + Reservation settlement is unchanged: recovery introduces no new terminal raise; existing + settle-before-raise sites still cover every exit. diff --git a/openspec/changes/fix-compact-previous-response-quota-failover/specs/responses-api-compat/spec.md b/openspec/changes/fix-compact-previous-response-quota-failover/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..df4798fcc6 --- /dev/null +++ b/openspec/changes/fix-compact-previous-response-quota-failover/specs/responses-api-compat/spec.md @@ -0,0 +1,173 @@ +## ADDED Requirements + +### Requirement: Compact requests recover from quota-caused previous-response owner loss + +When a compact request is pinned to a previous-response owner account, that pin is the only +continuity pin (no client-supplied turn-state owner and no input-file owner), and account +selection cannot return the pinned owner, the proxy MUST attempt account-neutral fresh-replay +recovery before surfacing the failure, provided every activation gate below holds. Outside the +gates the proxy MUST keep today's fail-closed failure for that request, MUST NOT send any part +of the payload to another account, and MUST record the fail-closed outcome on the continuity +fail-closed observability counter for the compact surface. + +Recovery MUST activate only for quota-caused owner loss. At selection time, before the owner +was ever used for the request, the owner's persisted account status MUST be rate-limited or +quota-exhausted; an owner unselectable for any other reason (re-authentication required, +deactivated, paused, local capacity caps on an active account, or a failed status lookup) +stays owner-bound. An owner the selector skipped for routing policy — API-key assignment +scope, single-account routing, or a prior in-request exclusion — MUST stay owner-bound +regardless of its persisted quota status, because policy, not quota, caused that selection +loss. Mid-request, only a pre-visible quota or rate-limit failure of the pinned owner that +permits failover makes recovery eligible; post-selection authentication, refresh, transport, +timeout, and transient exclusions of the pinned owner keep their existing owner-bound +handling. + +Recovery MUST NOT activate when the request carries a session identity (a turn-state or +session header) that can bind live or durable HTTP-bridge continuity, or when the resolved +affinity is session ownership (a Codex-session affinity key, a raw legacy session row, or a +conversation handle requiring an unambiguous owner): this recovery deliberately carries no +continuity-rebind machinery, so anything that would need rebinding stays owner-bound. +Prompt-cache and sticky-thread locality keys are advisory cache locality that ordinary sticky +selection already falls back from and do not block recovery. + +Local verification MUST run against the exact serialized upstream-bound compact payload +without `previous_response_id`, after every transformation the compact serializer applies. +Transport-stage mutations that follow that serialization are outside the client payload being +proven and MUST be limited to proxy-injected, account-agnostic controls that are applied +identically to the owner send and the replay send (the Responses-Lite +`reasoning.context` control and inline image fetching); they carry no client or account +state, so the shared account-neutral rules — which the HTTP bridge replay paths likewise +apply to pre-transport serializations — retain their meaning. It +MUST require that serialized `input` to be a list of more than one item that is item-for-item +identical to the validated request `input`, so that no request whose wire history is dropped +or trimmed — including single-item collapse and oversized-input trim markers — is ever +replayed on another account, which could not resolve the omitted owner-resident context. It +MUST validate that same serialized payload against the shared account-neutral fresh-replay +rules: self-contained tool call/output pairing, no server-assigned item ids, no encrypted or +compaction state, no nonblank conversation or prompt handles, no account-scoped +file/container/vector handles, no hosted/MCP call state, and only recognized account-neutral +fields and shapes. Because a self-contained payload may still be a delta that relies on the +owner to hold the earlier conversation, the serialized `input` MUST additionally parse as a +transcript whose final segment retains completed assistant output followed only by fresh +client input, using the shared retained-prior-output rule anchored at the last assistant +message. Histories those gates cannot prove — including delta-shaped inputs without retained +assistant output and transcripts without fresh follow-up input — stay owner-bound. + +This transcript-shape rule is the evidence ceiling of this scope: completeness relative to the +anchored conversation is not provable from the payload alone, and the durable prefix metadata +that could prove it is deliberately not consulted here (per the maintainer rescope of the +original change, which excluded durable-bridge plumbing from this recovery). A delta resend +that itself carries a completed assistant exchange ahead of the fresh input is therefore +indistinguishable from a full resend and MAY be recovered as the client's authoritative local +history — the same trust the shared account-neutral fresh-replay rules already grant a normal +turn that abandons an unavailable owner. Clients that resend partial histories under a +previous-response anchor accept summarization over that partial history when the owner is +quota-lost; the alternative surface is the current hard failure until the owner's quota +window resets. + +For an eligible recovery, the proxy MUST remove `previous_response_id` from the upstream +compact payload, strip downstream session/turn affinity aliases from the upstream-bound +headers, exclude the unavailable owner account from the remaining attempts, and reselect among +the remaining eligible accounts with fallback enabled. + +#### Scenario: Quota-excluded owner at selection time fails over with a verified full resend + +- **GIVEN** account A owns the previous response referenced by a compact request and account B is eligible +- **AND** account A's persisted status is rate-limited or quota-exhausted +- **AND** the compact payload carries an account-neutral full-resend `input` that retains prior assistant output ahead of the new client input +- **AND** the request carries no session identity and no session-ownership affinity +- **WHEN** pinned account selection cannot return account A +- **THEN** the proxy sends the compact upstream exactly once on account B without `previous_response_id` +- **AND** the compact response is returned successfully + +#### Scenario: Owner exhausts quota during the compact request + +- **GIVEN** the pinned previous-response owner is selected for a compact request +- **AND** the upstream compact fails with a pre-visible quota or rate-limit error that permits failover +- **WHEN** reselection cannot return the now-excluded owner +- **THEN** the proxy applies the same account-neutral fresh-replay recovery on another eligible account +- **AND** the owner's quota failure is not surfaced to the client when the recovery succeeds + +#### Scenario: Post-selection authentication failure on the pinned owner stays owner-bound + +- **GIVEN** the pinned previous-response owner is selected for a compact request with an account-neutral full-resend `input` +- **AND** the upstream compact fails with `401` again after the forced token refresh, which excludes the owner from the remaining attempts +- **WHEN** reselection cannot return the now-excluded owner +- **THEN** the proxy surfaces the owner's authentication failure +- **AND** account-neutral fresh-replay recovery does not activate and no part of the payload is sent to another account + +#### Scenario: Policy-skipped owner stays owner-bound despite quota status + +- **GIVEN** a previous-response-pinned compact request whose owner account is excluded by API-key assignment scope or single-account routing +- **AND** that owner's persisted status is coincidentally rate-limited or quota-exhausted +- **WHEN** pinned account selection skips the owner +- **THEN** the request fails with the existing selection error +- **AND** account-neutral fresh-replay recovery does not activate + +#### Scenario: Responses-Lite full resend behind a canonical tool bundle is recoverable + +- **GIVEN** a quota-excluded previous-response-pinned compact request whose `input` opens with a canonical `additional_tools` bundle and its immediately following developer instruction +- **AND** the remaining transcript retains prior assistant output ahead of fresh client input and passes the account-neutral fresh-replay rules +- **WHEN** pinned account selection cannot return the owner +- **THEN** the shared canonical-Lite prefix handling recognizes the developer instruction +- **AND** the recovery replays the anchor-free payload on another eligible account + +#### Scenario: Non-quota owner loss at selection time stays owner-bound + +- **GIVEN** a previous-response-pinned compact request whose owner account is paused, deactivated, or requires re-authentication +- **WHEN** pinned account selection cannot return the owner +- **THEN** the request fails with the existing selection error +- **AND** account-neutral fresh-replay recovery does not activate +- **AND** the continuity fail-closed counter records the compact-surface outcome + +#### Scenario: Non-neutral compact payload stays fail-closed + +- **GIVEN** a pinned compact request whose `input` retains encrypted compaction state, server-assigned item ids, or account-scoped file handles +- **WHEN** the quota-excluded pinned owner cannot be selected +- **THEN** the request fails with the existing selection or upstream error +- **AND** no part of the payload is sent to another account +- **AND** the continuity fail-closed counter records the compact-surface outcome + +#### Scenario: Delta-shaped history without retained output stays fail-closed + +- **GIVEN** a pinned compact request whose multi-item `input` carries no retained assistant output ahead of fresh client input +- **WHEN** the quota-excluded pinned owner cannot be selected +- **THEN** the request fails with the existing selection or upstream error +- **AND** the proxy keeps the anchor and sends no part of the payload to another account + +#### Scenario: History the wire serializer shortens stays fail-closed + +- **GIVEN** a pinned compact request whose `input` loses history when serialized for upstream, either collapsing to a single item or being trimmed to a head, trim marker, and tail +- **WHEN** the quota-excluded pinned owner cannot be selected +- **THEN** the request fails with the existing selection or upstream error +- **AND** the proxy does not replay the shortened history on another account + +#### Scenario: Session-identified compact stays owner-bound + +- **GIVEN** a pinned compact request that carries a session or turn-state identity able to bind live or durable HTTP-bridge continuity +- **WHEN** the quota-excluded pinned owner cannot be selected +- **THEN** the request fails with the existing selection error +- **AND** account-neutral fresh-replay recovery does not activate + +#### Scenario: Turn-state-pinned and file-pinned compacts remain owner-bound + +- **GIVEN** a compact request pinned by a client-supplied turn-state owner or an input-file owner +- **WHEN** that owner account cannot be selected +- **THEN** the request fails closed with the existing continuity or selection error +- **AND** account-neutral fresh-replay recovery does not activate + +#### Scenario: Additional owner pins record the fail-closed outcome + +- **GIVEN** a compact request whose `previous_response_id` owner is also resolved by a turn-state or input-file pin naming the same account +- **WHEN** the pinned owner cannot be selected +- **THEN** the request fails closed with the existing selection error +- **AND** account-neutral fresh-replay recovery does not activate +- **AND** the continuity fail-closed counter records the compact-surface outcome + +#### Scenario: Unresolvable previous-response owner remains fail-closed + +- **GIVEN** a compact request whose `previous_response_id` owner cannot be resolved from any record +- **AND** more than one account is eligible +- **WHEN** the request is evaluated before account selection +- **THEN** the request fails with `previous_response_owner_unavailable` +- **AND** the proxy does not treat the missing owner as a selector result or replay on another account diff --git a/openspec/changes/fix-compact-previous-response-quota-failover/tasks.md b/openspec/changes/fix-compact-previous-response-quota-failover/tasks.md new file mode 100644 index 0000000000..1d584e8223 --- /dev/null +++ b/openspec/changes/fix-compact-previous-response-quota-failover/tasks.md @@ -0,0 +1,32 @@ +# Tasks + +- [x] 1. Add a compact account-neutral replay verification helper in + `app/modules/proxy/_service/compact.py` that returns the anchor-free + `ResponsesCompactRequest` only when the request carries `previous_response_id`, a + list-shaped `input` with more than one item, an upstream-bound serialization that is + item-for-item identical to the validated request `input`, passes + `responses_payload_is_account_neutral_fresh_replay`, and retains prior assistant output + ahead of new client input via `responses_input_suffix_retains_prior_output`. +- [x] 2. In the `compact_responses` account-selection loop, when selection returns no account + and the request is pinned only by the previous-response owner, activate recovery for a + verified payload when the owner loss is quota-caused (persisted + `RATE_LIMITED`/`QUOTA_EXCEEDED` status at selection time, or a pre-visible quota/rate-limit + in-request failover of the owner): exclude the owner, drop the pin, strip session/turn + affinity aliases from upstream-bound headers via + `without_http_bridge_session_affinity_headers`, and reselect with fallback enabled. +- [x] 3. Keep every other case fail-closed and record `continuity_fail_closed` (surface + `compact`, reason `owner_account_unavailable`) when the pinned selection failure is + surfaced: additional turn-state/input-file owner pins on the same owner, session identity + on the request, session-ownership affinity, non-quota owner loss, and unverifiable + histories. +- [x] 4. Add unit tests for the verification helper (eligible full resend; missing anchor; + single-item and string inputs; server-assigned ids; encrypted compaction state; delta + histories without retained output; transcripts without fresh follow-up input; wire-trimmed + oversized histories). +- [x] 5. Add integration regression tests at `POST /backend-api/codex/responses/compact`: + selection-time quota loss recovers on the other account without `previous_response_id`; + mid-request owner 429 recovers the same way; repeated post-refresh 401 stays owner-bound; + delta histories, account-scoped histories, session-identified requests, and paused owners + stay fail-closed with nothing sent to another account. +- [x] 6. Run `uv run ruff check`, `uv run ruff format --check`, `uv run ty check`, and the + unit + compact integration suites; validate the change with strict OpenSpec validation. diff --git a/openspec/changes/fix-disconnect-cleanup-leak/.openspec.yaml b/openspec/changes/fix-disconnect-cleanup-leak/.openspec.yaml new file mode 100644 index 0000000000..84cfc12459 --- /dev/null +++ b/openspec/changes/fix-disconnect-cleanup-leak/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-06 diff --git a/openspec/changes/fix-disconnect-cleanup-leak/design.md b/openspec/changes/fix-disconnect-cleanup-leak/design.md new file mode 100644 index 0000000000..970214b6d6 --- /dev/null +++ b/openspec/changes/fix-disconnect-cleanup-leak/design.md @@ -0,0 +1,5 @@ +# Design + +- Run required teardown in an owned asyncio task inside an anyio shield and tolerate repeated cancellation delivery until that task completes. +- Use the same cancellation-deferring primitive for source-chat upstream closure, API-key reservation release, and request-log persistence. +- Track whether a Responses terminal event was observed; only classify cancellation as `client_disconnected` when no terminal event has been observed. diff --git a/openspec/changes/fix-disconnect-cleanup-leak/proposal.md b/openspec/changes/fix-disconnect-cleanup-leak/proposal.md new file mode 100644 index 0000000000..d7bad1a580 --- /dev/null +++ b/openspec/changes/fix-disconnect-cleanup-leak/proposal.md @@ -0,0 +1,5 @@ +# Fix disconnect cleanup and terminal settlement + +Streaming and source-chat disconnect cleanup must complete even while Starlette/anyio is cancelling the request task. A completed terminal Responses event must remain authoritative after a later downstream disconnect. + +This change hardens database/session teardown, source-chat reservation and request-log cleanup, and Responses stream settlement classification. diff --git a/openspec/changes/fix-disconnect-cleanup-leak/specs/api-keys/spec.md b/openspec/changes/fix-disconnect-cleanup-leak/specs/api-keys/spec.md new file mode 100644 index 0000000000..8ee6fc5221 --- /dev/null +++ b/openspec/changes/fix-disconnect-cleanup-leak/specs/api-keys/spec.md @@ -0,0 +1,10 @@ +## ADDED Requirements + +### Requirement: Disconnect cleanup settles source-chat reservations + +When a source-chat request is cancelled or its streaming body is closed, the proxy MUST close the upstream iterator, release its API-key reservation, and write or explicitly abort the source request-log row despite repeated cancellation delivery. + +#### Scenario: Client disconnects during source stream + +- **WHEN** the downstream client disconnects before source-stream completion +- **THEN** the reservation is released and the source request is logged as an aborted/error request. diff --git a/openspec/changes/fix-disconnect-cleanup-leak/specs/responses-api-compat/spec.md b/openspec/changes/fix-disconnect-cleanup-leak/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..0c0aba73f5 --- /dev/null +++ b/openspec/changes/fix-disconnect-cleanup-leak/specs/responses-api-compat/spec.md @@ -0,0 +1,11 @@ +## ADDED Requirements + +### Requirement: Terminal stream settlement is immutable after delivery + +When a Responses stream has observed and delivered a terminal event (`response.completed`, `response.failed`, `response.incomplete`, or `error`), a later downstream cancellation MUST NOT rewrite the terminal status, error, usage, or account-health settlement. + +#### Scenario: Disconnect after terminal event + +- **WHEN** the downstream closes after receiving a terminal event +- **THEN** the request log and settlement retain the terminal event's outcome +- **AND** the proxy does not record `client_disconnected` for that stream. diff --git a/openspec/changes/fix-disconnect-cleanup-leak/tasks.md b/openspec/changes/fix-disconnect-cleanup-leak/tasks.md new file mode 100644 index 0000000000..e473d9fe28 --- /dev/null +++ b/openspec/changes/fix-disconnect-cleanup-leak/tasks.md @@ -0,0 +1,5 @@ +- [x] Harden pooled session rollback/close against repeated cancellation delivery. +- [x] Complete source-chat stream close, reservation release, and request-log persistence on disconnect. +- [x] Release non-stream source-chat reservations on cancellation and unexpected upstream failures. +- [x] Preserve terminal Responses settlement after downstream disconnect. +- [x] Run regression and named proxy/integration suites. diff --git a/openspec/changes/fix-inflight-future-abandoned/.openspec.yaml b/openspec/changes/fix-inflight-future-abandoned/.openspec.yaml new file mode 100644 index 0000000000..84cfc12459 --- /dev/null +++ b/openspec/changes/fix-inflight-future-abandoned/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-06 diff --git a/openspec/changes/fix-inflight-future-abandoned/design.md b/openspec/changes/fix-inflight-future-abandoned/design.md new file mode 100644 index 0000000000..77f0f2602e --- /dev/null +++ b/openspec/changes/fix-inflight-future-abandoned/design.md @@ -0,0 +1,27 @@ +## Context + +The bridge lookup loop first resolves a reusable previous-response session and records it in `session_to_return_after_close`. The generic create arm is selected independently by `inflight_future is None`, so it can register a new pending future for the already-resolved key before the function returns the reused session. The existing cleanup/janitor intentionally removes only completed futures and is covered by a unit test. + +## Goals / Non-Goals + +**Goals:** + +- Make the reuse decision terminal for session creation in that loop. +- Leave no unresolved future registered for a key whose existing session is returned. +- Preserve all create, waiter, handoff, timeout, and janitor behavior for paths that do not reuse a previous-response session. + +**Non-Goals:** + +- Do not change janitor eligibility or restart-blocking semantics for genuinely live creation futures. +- Do not redesign durable ownership, session closing, or response routing. + +## Decisions + +Guard the generic `inflight_future is None` creation arm with `session_to_return_after_close is None`. This is the smallest local invariant: once reuse has selected a session, the loop may still close detached sessions, then returns the selected session without publishing a creation future. An early `continue` or future resolution would add lifecycle behavior without benefit and could interfere with the existing create-chain arms. + +The regression uses the real `_get_or_create_http_bridge_session` previous-response lookup path and asserts both registry state and a second successful reuse. The existing janitor test remains unchanged as a negative control. + +## Risks / Trade-offs + +- [Risk] A future branch might set `session_to_return_after_close` for a case that still needs creation. → Mitigation: the symbol has one assignment, in the validated live-session reuse arm; all other arms leave it `None` and retain the original create condition. +- [Risk] A future remains from an earlier concurrent creator. → Mitigation: reuse already requires the canonical previous-key inflight lookup to be empty; waiter behavior remains guarded by `inflight_future is not None`. diff --git a/openspec/changes/fix-inflight-future-abandoned/proposal.md b/openspec/changes/fix-inflight-future-abandoned/proposal.md new file mode 100644 index 0000000000..80e65c5e21 --- /dev/null +++ b/openspec/changes/fix-inflight-future-abandoned/proposal.md @@ -0,0 +1,25 @@ +## Why + +The HTTP bridge's previous-response reuse path can return an already-live session after first publishing a new, unresolved session-creation future for that same anchor. That orphaned future permanently marks the bridge as restart-blocking and makes later requests fail with a continuity 502, so the create chain must not run when reuse has already selected a session. + +## What Changes + +- Prevent the generic HTTP bridge session-creation arm from publishing an inflight future when the previous-response path has selected an existing session for return. +- Preserve the existing done-future janitor contract and all other session-creation and handoff arms. +- Keep regression coverage for both registry cleanup and a successful second request on the same previous-response anchor. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: HTTP bridge previous-response reuse must not leave an unresolved session-creation future registered for the reused anchor. + +## Impact + +- Affected code: `app/modules/proxy/_service/http_bridge/mixin.py` session lookup/create chain. +- Affected tests: focused HTTP bridge bughunt regression and existing unit/integration bridge suites. +- No API schema, persistence, or janitor behavior changes. diff --git a/openspec/changes/fix-inflight-future-abandoned/specs/responses-api-compat/spec.md b/openspec/changes/fix-inflight-future-abandoned/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..76ed8af65e --- /dev/null +++ b/openspec/changes/fix-inflight-future-abandoned/specs/responses-api-compat/spec.md @@ -0,0 +1,51 @@ +## MODIFIED Requirements + +### Requirement: Continuity-dependent Responses follow-ups fail closed with retryable errors +When a Responses follow-up depends on previously established continuity state, the service MUST return a retryable continuity error if that continuity cannot be reconstructed safely. The service MUST NOT expose raw `previous_response_not_found` for bridge-local metadata loss or similar internal continuity gaps. When forwarding a turn-state-anchored follow-up to its bridge owner fails with `bridge_owner_unreachable` and a fresh durable lookup shows the owner no longer holds an active lease (released, expired, or the row is missing or CLOSED), the service MUST recover the follow-up locally through durable takeover instead of returning the retryable error. The fresh durable lookup MUST use the same resolution semantics as request routing, including the latest-turn-state fallback, so a row originally resolved without a registered alias remains takeover-eligible. When the durable lease is still actively held by another instance — including DRAINING rows whose lease has not been released or expired — the service MUST keep failing closed with the retryable error. + +#### Scenario: HTTP bridge loses local continuity metadata for a follow-up request +- **WHEN** an HTTP `/v1/responses` or `/backend-api/codex/responses` follow-up request depends on `previous_response_id` or a hard continuity turn-state +- **AND** the bridge cannot reconstruct the matching live continuity state from local or durable metadata +- **THEN** the service returns a retryable OpenAI-format error +- **AND** the error code is not `previous_response_not_found` + +#### Scenario: in-flight bridge follower loses continuity while waiting on the same canonical session +- **WHEN** a follow-up request waits on an in-flight HTTP bridge session for the same hard continuity key +- **AND** the bridge still cannot reconstruct safe continuity state once the leader finishes +- **THEN** the service returns a retryable OpenAI-format error +- **AND** the error code is not `previous_response_not_found` + +#### Scenario: multiplexed follow-ups fail closed only for the matching continuity anchor +- **WHEN** a websocket or HTTP bridge session has multiple pending follow-up requests with different `previous_response_id` anchors +- **AND** continuity loss is detected for exactly one of those anchors +- **THEN** the service applies the retryable fail-closed continuity error only to the matching follow-up request +- **AND** it does not expose raw `previous_response_not_found` +- **AND** unrelated pending requests continue on their own response lifecycle + +#### Scenario: multiplexed follow-ups sharing one anchor fail closed together without leaking raw continuity errors +- **WHEN** a websocket or HTTP bridge session has multiple pending follow-up requests that share the same `previous_response_id` anchor +- **AND** upstream emits an anonymous continuity loss event such as `previous_response_not_found` for that shared anchor +- **THEN** the service rewrites each affected follow-up into a retryable continuity error +- **AND** no affected follow-up exposes raw `previous_response_not_found` +- **AND** the run remains usable for subsequent requests after the rewritten failures + +#### Scenario: single pre-created follow-up still fails closed when continuity loss omits explicit response id in message +- **WHEN** a websocket follow-up request is pending with `previous_response_id` and has not received a stable upstream `response.id` yet +- **AND** upstream emits `previous_response_not_found` with `param=previous_response_id` +- **AND** the upstream error message omits the literal previous response identifier +- **THEN** the service still maps that continuity loss to the pending follow-up +- **AND** it rewrites the downstream terminal event to a retryable continuity error +- **AND** it does not surface raw `previous_response_not_found` to the client + +#### Scenario: turn-state follow-up recovers locally after the owner released its lease +- **WHEN** a turn-state-anchored follow-up without `previous_response_id` is forwarded to its bridge owner during the post-shutdown ring grace window +- **AND** the forward fails with `bridge_owner_unreachable` +- **AND** a fresh durable lookup using the request-routing resolution semantics (registered alias or latest-turn-state fallback) shows the lease is released or expired +- **THEN** the service retries the follow-up locally through durable takeover instead of returning the retryable 503 +- **AND** the takeover retry carries the fresh durable lookup as its continuity anchor even when the turn-state alias registration was lost +- **AND** a fresh durable lookup showing a live lease held by another instance — even for a DRAINING row — still fails closed with the retryable `bridge_owner_unreachable` error + +#### Scenario: previous-response reuse does not register an abandoned creation future +- **WHEN** an HTTP bridge request resolves a live compatible session through `previous_response_id` +- **THEN** the bridge returns that existing session without registering an unresolved inflight session-creation future for its canonical key +- **AND** a subsequent request on the same previous-response anchor can reuse the session successfully diff --git a/openspec/changes/fix-inflight-future-abandoned/tasks.md b/openspec/changes/fix-inflight-future-abandoned/tasks.md new file mode 100644 index 0000000000..9bf866d275 --- /dev/null +++ b/openspec/changes/fix-inflight-future-abandoned/tasks.md @@ -0,0 +1,10 @@ +## 1. Implementation + +- [x] 1.1 Guard the generic HTTP bridge session-creation arm so a previous-response reuse selection cannot publish an inflight future. +- [x] 1.2 Keep the existing janitor and all non-reuse create-chain arms unchanged. + +## 2. Verification + +- [x] 2.1 Run the F1 bughunt regression and confirm it fails on the baseline and passes after the fix, including the second-request reuse assertion. +- [x] 2.2 Run the HTTP bridge unit and integration suites, including the existing live-inflight janitor test. +- [x] 2.3 Validate OpenSpec artifacts and inspect the final diff/status before committing. diff --git a/openspec/changes/fix-refresh-singleflight-successor-guard/proposal.md b/openspec/changes/fix-refresh-singleflight-successor-guard/proposal.md new file mode 100644 index 0000000000..1fd011930d --- /dev/null +++ b/openspec/changes/fix-refresh-singleflight-successor-guard/proposal.md @@ -0,0 +1,14 @@ +# Fix refresh singleflight successor settlement + +The refresh singleflight negative cache must not publish a failed attempt's +error after a successor refresh has replaced that attempt for the same key. +This keeps callers arriving during the successor refresh joined to the live +operation instead of serving stale failure state. + +## Scope + +- Guard negative-cache writes and clears with the same current-task check that + guards inflight removal. +- Add the successor-race regression coverage already exercised by the F8 + bughunt probe. +- Do not change downstream account-status failure handling. diff --git a/openspec/changes/fix-refresh-singleflight-successor-guard/specs/usage-refresh-policy/spec.md b/openspec/changes/fix-refresh-singleflight-successor-guard/specs/usage-refresh-policy/spec.md new file mode 100644 index 0000000000..9589909b7d --- /dev/null +++ b/openspec/changes/fix-refresh-singleflight-successor-guard/specs/usage-refresh-policy/spec.md @@ -0,0 +1,25 @@ +# usage-refresh-policy Delta + +## ADDED Requirements + +### Requirement: Refresh singleflight settlement cannot poison a successor + +When a refresh task completes, it MUST mutate the inflight entry and +refresh-failure cache only if it is still the current inflight task for that +singleflight key. A completion from an older attempt MUST NOT publish or clear +negative-cache state belonging to a successor refresh. + +#### Scenario: Failed attempt is followed by a live successor + +- **GIVEN** a refresh task fails for a key +- **AND** a successor task for the same key is installed before the failed + task's completion settlement runs +- **WHEN** another caller arrives while the successor is still in flight +- **THEN** the caller joins the successor task +- **AND** the failed attempt's error is not served from the negative cache + +#### Scenario: Existing failure settlement has no successor + +- **GIVEN** a refresh task fails and remains the current inflight task +- **WHEN** its completion settlement runs +- **THEN** the configured negative-cache cooldown behavior is preserved diff --git a/openspec/changes/fix-refresh-singleflight-successor-guard/tasks.md b/openspec/changes/fix-refresh-singleflight-successor-guard/tasks.md new file mode 100644 index 0000000000..a1e5412413 --- /dev/null +++ b/openspec/changes/fix-refresh-singleflight-successor-guard/tasks.md @@ -0,0 +1,3 @@ +- [x] Guard refresh singleflight settlement cache mutations by task ownership. +- [x] Verify the independent-caller successor-race regression. +- [x] Run the account refresh unit suites and OpenSpec validation. diff --git a/openspec/changes/fix-reports-full-cost-thousands-separators/proposal.md b/openspec/changes/fix-reports-full-cost-thousands-separators/proposal.md new file mode 100644 index 0000000000..aacb6766a6 --- /dev/null +++ b/openspec/changes/fix-reports-full-cost-thousands-separators/proposal.md @@ -0,0 +1,23 @@ +## Why + +The `/reports` page has a shared USD formatter that renders full currency values with grouping separators, but several non-compact Cost surfaces bypass it with manual `toFixed(2)` string interpolation. Operators therefore see `$1400.00` instead of `$1,400.00` in the affected full-value surfaces. + +## What Changes + +- Use the existing shared USD formatter for non-compact Reports Cost values in the summary card, average-cost subtitle, Daily Breakdown table, Cost by Day chart axis, and chart tooltip. +- Preserve intentionally compact Cost labels such as `$1.4K` in constrained distribution-chart surfaces. +- Preserve full decimal precision in Daily Breakdown CSV export; its machine-readable output is not a display surface. + +## Capabilities + +### New Capabilities + +### Modified Capabilities + +- `frontend-architecture`: `/reports` full-value USD display surfaces use grouped currency formatting while compact visualizations retain compact notation. + +## Impact + +- Frontend: Reports summary cards, Daily Breakdown table, and Cost by Day chart tooltip. +- Tests: focused Reports component regression coverage for grouped full-value currency rendering. +- Specs: `frontend-architecture` delta for Reports currency presentation. diff --git a/openspec/changes/fix-reports-full-cost-thousands-separators/specs/frontend-architecture/spec.md b/openspec/changes/fix-reports-full-cost-thousands-separators/specs/frontend-architecture/spec.md new file mode 100644 index 0000000000..ff86a7acbf --- /dev/null +++ b/openspec/changes/fix-reports-full-cost-thousands-separators/specs/frontend-architecture/spec.md @@ -0,0 +1,20 @@ +## ADDED Requirements + +### Requirement: Reports full-value USD displays use grouped currency formatting + +The dashboard SHALL render non-compact USD Cost values on `/reports` through the shared currency formatter so values at or above one thousand include locale-appropriate grouping separators and exactly two fractional digits. This requirement applies to the Total Cost summary value, its average-cost-per-day subtitle, Daily Breakdown Cost cells, and Cost by Day axis and tooltip values. + +#### Scenario: Summary and daily Cost values exceed one thousand USD + +- **WHEN** an authenticated operator views `/reports` data whose full-value Cost amount is `1400` +- **THEN** the Total Cost summary value renders `$1,400.00` +- **AND** the average-cost-per-day subtitle, when its amount is `1400`, renders `$1,400.00` +- **AND** a Daily Breakdown Cost cell whose amount is `1400` renders `$1,400.00` +- **AND** a Cost by Day axis tick whose amount is `1400` renders `$1,400.00` +- **AND** a Cost by Day tooltip whose amount is `1400` renders `$1,400.00` + +#### Scenario: Intentionally compact Cost visualization remains compact + +- **WHEN** an authenticated operator views a constrained Reports distribution visualization whose Cost label uses compact notation +- **THEN** that visualization may continue to render a compact label such as `$1.4K` +- **AND** the full-value Cost displays remain grouped currency values diff --git a/openspec/changes/fix-reports-full-cost-thousands-separators/tasks.md b/openspec/changes/fix-reports-full-cost-thousands-separators/tasks.md new file mode 100644 index 0000000000..e8e2a11773 --- /dev/null +++ b/openspec/changes/fix-reports-full-cost-thousands-separators/tasks.md @@ -0,0 +1,16 @@ +## 1. Specification + +- [x] 1.1 Add a `frontend-architecture` delta defining grouped rendering for full-value Reports USD surfaces while retaining intentional compact notation. + +## 2. Implementation + +- [x] 2.1 Route full-value Cost summary and average-per-day displays through the existing shared USD formatter. +- [x] 2.2 Route the Daily Breakdown Cost cell and Cost by Day tooltip through the existing shared USD formatter. +- [x] 2.3 Leave compact distribution-chart Cost labels and CSV numeric export unchanged. + +## 3. Verification + +- [x] 3.1 Add focused regression tests for four-digit full-value Cost rendering. +- [x] 3.2 Run focused Reports tests, frontend typecheck, and lint. +- [ ] 3.3 Run strict OpenSpec validation if the repository tool is available. (Not run: validator is unavailable in this checkout.) +- [x] 3.4 Run an independent GPT-5.6 SOL review on the final diff and address blocking findings. diff --git a/openspec/changes/extend-rowless-rebase-to-durable-markers/.openspec.yaml b/openspec/changes/fix-usage-refresh-shared-future-waiters/.openspec.yaml similarity index 100% rename from openspec/changes/extend-rowless-rebase-to-durable-markers/.openspec.yaml rename to openspec/changes/fix-usage-refresh-shared-future-waiters/.openspec.yaml diff --git a/openspec/changes/fix-usage-refresh-shared-future-waiters/design.md b/openspec/changes/fix-usage-refresh-shared-future-waiters/design.md new file mode 100644 index 0000000000..312b234671 --- /dev/null +++ b/openspec/changes/fix-usage-refresh-shared-future-waiters/design.md @@ -0,0 +1,69 @@ +## Context + +`_UsageRefreshSingleflight` owns one task per account and may expose that task +to many request and scheduler waiters. Its two shared waits still use +`asyncio.shield`, unlike the bridge and token-refresh sites converted by +[`harden-shared-future-admission-waits`](../harden-shared-future-admission-waits/). +The existing helper already provides the required result, exception, and +cancellation semantics; see the +[`proxy-admission-control`](specs/proxy-admission-control/spec.md) delta and +the existing +[`proxy-runtime-observability`](../../specs/proxy-runtime-observability/) +signals. + +## Goals / Non-Goals + +**Goals:** + +- Make both joining and non-joining usage-refresh waits constant-cost under + cancellation storms. +- Preserve singleflight task ownership and successor ordering. +- Prove the usage-refresh surface, not only the generic helper, maintains one + fan-out callback. + +**Non-Goals:** + +- Change refresh selection, persistence, exception swallowing, or shutdown. +- Rewrite the helper or convert request-owned cleanup shields. +- Integrate or rebase the unrelated session-ownership work in PR #1887. + +## Decisions + +### Reuse the established shared-future helper at both wait sites + +Both the `join_existing=True` return path and the `join_existing=False` +predecessor wait can accumulate many waiters on one task, so both call +`wait_on_shared_future`. Reusing the established helper preserves waiter +cancellation isolation while keeping one fan-out callback. Keeping +`asyncio.shield` at either site would retain the incident mechanism; a second +usage-specific helper would duplicate the existing contract. + +### Preserve the current control flow + +The non-joining path continues swallowing predecessor failures and retries the +loop, while caller cancellation continues propagating. The final wait +continues propagating the selected task's result or exception. This limits the +fix to waiter mechanics and avoids changing refresh policy. + +### Test callback structure through the usage singleflight + +The regression test attaches many `run` callers, inspects the in-flight task's +callback count, cancels most callers, and verifies the count remains bounded +while a survivor receives the factory result. This test fails with the old +shield implementation because each waiter attaches callbacks to the shared +task. + +## Risks / Trade-offs + +- **Risk:** The private callback-list assertion depends on CPython asyncio + internals. **Mitigation:** Match the existing helper and bridge regression + pattern, and guard only the structural property involved in the production + incident. +- **Risk:** PR #1887 also edits the same singleflight. **Mitigation:** Keep this + patch focused on current `main` and call out the conflict so that PR must + carry this conversion forward. + +## Migration Plan + +Deploy through the normal image release process after merge. Rollback is a +code rollback; there are no data, schema, or configuration migrations. diff --git a/openspec/changes/fix-usage-refresh-shared-future-waiters/proposal.md b/openspec/changes/fix-usage-refresh-shared-future-waiters/proposal.md new file mode 100644 index 0000000000..a1f9e54992 --- /dev/null +++ b/openspec/changes/fix-usage-refresh-shared-future-waiters/proposal.md @@ -0,0 +1,40 @@ +## Why + +The usage-refresh singleflight was omitted when shared, many-waiter futures +were hardened after the 2026-08-20 event-loop livelock. After roughly 91 hours +of production uptime, cancelled usage-refresh waiters again drove the event +loop into the same `asyncio.shield` callback-removal failure mode, so this +remaining shared wait site must use the established fan-out helper. + +## What Changes + +- Route both usage-refresh singleflight wait paths through + `wait_on_shared_future`, preserving result, cancellation, exception, and + `join_existing=False` sequencing semantics. +- Keep the shared refresh factory task running when an individual waiter is + cancelled or times out, with one bounded fan-out callback on that task. +- Add usage-refresh surface regression coverage for concurrent joins, + cancellation isolation, callback fan-out, and successor sequencing. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `proxy-admission-control`: extend the established shared-future admission + contract to usage-refresh singleflight waiters. + +## Impact + +- `app/modules/usage/updater.py` and `tests/unit/test_usage_updater.py`. +- No API, schema, configuration, dependency, or deployment changes. +- This is a focused follow-on to + [`harden-shared-future-admission-waits`](../harden-shared-future-admission-waits/) + and relies on the existing + [`proxy-admission-control`](../../specs/proxy-admission-control/) wait + mechanism and + [`proxy-runtime-observability`](../../specs/proxy-runtime-observability/) + event-loop lag signals. diff --git a/openspec/changes/fix-usage-refresh-shared-future-waiters/specs/proxy-admission-control/spec.md b/openspec/changes/fix-usage-refresh-shared-future-waiters/specs/proxy-admission-control/spec.md new file mode 100644 index 0000000000..3630c0996d --- /dev/null +++ b/openspec/changes/fix-usage-refresh-shared-future-waiters/specs/proxy-admission-control/spec.md @@ -0,0 +1,45 @@ +## MODIFIED Requirements + +### Requirement: Admission waits on shared futures scale O(1) per waiter + +When multiple requests wait on one shared future (an inflight bridge session creation, a capacity slot, a token-refresh singleflight, or a usage-refresh singleflight), the system MUST use the established shared-future fan-out wait mechanism so attaching a waiter, a waiter timing out, and a waiter being cancelled each perform O(1) work on the shared future. The shared future MUST carry a constant number of done callbacks regardless of waiter count, and the wait mechanism itself MUST NOT cancel or otherwise mutate the shared future or the work it represents when a waiter times out or is cancelled. Admission handlers MAY still settle the shared future explicitly after a waiter's timeout (the http-bridge timeout handler fails and unregisters the inflight future so piled-up waiters converge on one overload outcome); that settlement is an admission-contract decision, not a side effect of waiting. The shared future's result, exception, or cancellation MUST propagate to every waiter with the same semantics as `asyncio.wait_for(asyncio.shield(shared), timeout)`. + +#### Scenario: Waiter pile-up keeps the shared future's callback list constant + +- **WHEN** many requests wait on the same inflight bridge-session future +- **THEN** the shared future carries a constant number of done callbacks +- **AND** the callback count does not grow with the number of waiters + +#### Scenario: Mass timeout does not degrade the event loop + +- **GIVEN** waiters piled onto a shared future that has not resolved within + the admission wait timeout +- **WHEN** the waiters time out together +- **THEN** each timeout detaches in O(1) without scanning the shared future's + callback list +- **AND** the surviving admission contract (local-overload `429` with the + capacity error code) is unchanged + +#### Scenario: Client-disconnect storm leaves the owner's creation running + +- **WHEN** every waiter on an inflight session future is cancelled by client + disconnects +- **THEN** the shared future stays pending and the owner's session creation + continues +- **AND** no per-waiter callbacks remain attached to the shared future + +#### Scenario: Cancelled usage-refresh waiters leave shared refresh running + +- **GIVEN** many callers are waiting on one in-flight usage refresh +- **WHEN** all but one caller are cancelled +- **THEN** the cancelled callers detach without adding or removing per-waiter + callbacks on the shared refresh task +- **AND** the shared refresh continues to completion for the remaining caller + +#### Scenario: Non-joining usage refresh starts after its predecessor + +- **GIVEN** a usage refresh is already in flight for an account +- **WHEN** another caller requests a non-joining refresh for that account +- **THEN** it waits without cancelling or mutating the in-flight refresh +- **AND** it starts a successor refresh only after the in-flight refresh has + finished diff --git a/openspec/changes/fix-usage-refresh-shared-future-waiters/tasks.md b/openspec/changes/fix-usage-refresh-shared-future-waiters/tasks.md new file mode 100644 index 0000000000..5083e7c262 --- /dev/null +++ b/openspec/changes/fix-usage-refresh-shared-future-waiters/tasks.md @@ -0,0 +1,17 @@ +## 1. Shared-Future Wait Conversion + +- [x] 1.1 Audit remaining `asyncio.shield` calls in `app/` and confirm only usage-refresh singleflight matches the shared, many-waiter class. +- [x] 1.2 Replace both `_UsageRefreshSingleflight.run` shared-task waits with `wait_on_shared_future` while preserving cancellation and exception semantics. + +## 2. Regression Coverage + +- [x] 2.1 Test that concurrent usage-refresh waiters receive the same result and cancelling all but one does not cancel the factory task. +- [x] 2.2 Test that cancelled usage-refresh waiters detach with one bounded fan-out callback on the in-flight task. +- [x] 2.3 Test that `join_existing=False` waits for the predecessor before starting a successor through the shared-future helper. +- [x] 2.4 Temporarily restore the shield implementation, run the callback fan-out regression test, and record the failing sabotage result. + +## 3. Verification + +- [x] 3.1 Run the targeted usage updater and shared-future waiter unit tests. +- [x] 3.2 Run the repository lint source of truth and strict OpenSpec validation. +- [x] 3.3 Record commands and exit codes in `/tmp/codex-lb-1896-verification.md`. diff --git a/openspec/changes/fix-websocket-response-create-lease-cancellation/design.md b/openspec/changes/fix-websocket-response-create-lease-cancellation/design.md new file mode 100644 index 0000000000..796035a099 --- /dev/null +++ b/openspec/changes/fix-websocket-response-create-lease-cancellation/design.md @@ -0,0 +1,7 @@ +# Design + +`_release_websocket_response_create_gate` keeps its existing state-clearing and +gate-release ordering, but awaits the captured account lease release through +`asyncio.shield`. The release operation therefore continues after cancellation +of the surrounding WebSocket task, returning the account slot without changing +the existing response-create gate semantics. diff --git a/openspec/changes/fix-websocket-response-create-lease-cancellation/proposal.md b/openspec/changes/fix-websocket-response-create-lease-cancellation/proposal.md new file mode 100644 index 0000000000..1427bfa9ef --- /dev/null +++ b/openspec/changes/fix-websocket-response-create-lease-cancellation/proposal.md @@ -0,0 +1,13 @@ +# Change: Make WebSocket response-create lease cleanup cancellation-safe + +## Why + +WebSocket terminal cleanup clears the request state's account response-create +lease before awaiting its asynchronous release. Cancellation at that await can +leave the account slot counted until stale-lease reclamation. + +## What Changes + +- Shield the account response-create lease release in WebSocket gate cleanup. +- Add regression coverage for cancellation under load-balancer runtime-lock + contention and retain coverage for genuine stale-lease reclamation. diff --git a/openspec/changes/fix-websocket-response-create-lease-cancellation/specs/proxy-admission-control/spec.md b/openspec/changes/fix-websocket-response-create-lease-cancellation/specs/proxy-admission-control/spec.md new file mode 100644 index 0000000000..344b5c9470 --- /dev/null +++ b/openspec/changes/fix-websocket-response-create-lease-cancellation/specs/proxy-admission-control/spec.md @@ -0,0 +1,16 @@ +## ADDED Requirements + +### Requirement: WebSocket response-create lease cleanup is cancellation-safe + +When WebSocket terminal cleanup has captured an account response-create lease, it MUST complete the asynchronous lease release even if the surrounding task is cancelled while waiting for the load-balancer runtime lock. Cleanup MUST retain the existing response-create gate release semantics. + +#### Scenario: Cancellation under lease-release contention returns the account slot + +- **GIVEN** a WebSocket request owns an account response-create lease and its + response-create gate +- **AND** the load-balancer runtime lock is held by another task +- **WHEN** terminal cleanup is cancelled while releasing the account lease +- **THEN** the account response-create slot MUST be returned after the lock is + freed +- **AND** the request state does not retain the released lease +- **AND** the response-create gate cleanup semantics remain unchanged diff --git a/openspec/changes/fix-websocket-response-create-lease-cancellation/tasks.md b/openspec/changes/fix-websocket-response-create-lease-cancellation/tasks.md new file mode 100644 index 0000000000..84d999b223 --- /dev/null +++ b/openspec/changes/fix-websocket-response-create-lease-cancellation/tasks.md @@ -0,0 +1,6 @@ +# Tasks + +- [x] Make WebSocket response-create lease release cancellation-safe. +- [x] Add cancellation and stale-reclaim regression coverage. +- [x] Run targeted WebSocket, HTTP bridge, and load-balancer lease tests. +- [x] Validate the OpenSpec documents. diff --git a/openspec/changes/fix-windows-asset-mime-types/proposal.md b/openspec/changes/fix-windows-asset-mime-types/proposal.md new file mode 100644 index 0000000000..f9fc5e146a --- /dev/null +++ b/openspec/changes/fix-windows-asset-mime-types/proposal.md @@ -0,0 +1,18 @@ +## Why + +On Windows, Python's `mimetypes` merges file-type mappings from the `HKCR` registry, where third-party software commonly remaps web extensions (`.js` → `text/plain`). Starlette's `FileResponse` resolves `media_type` through `mimetypes.guess_type`, and browsers enforce strict MIME checking for ES module scripts, so on such machines every `/assets/*.js` response ships as `text/plain` and the dashboard renders as a blank page (issue #1698). macOS/Linux use the built-in table and never hit this. + +## What Changes + +- Pin the MIME type of every extension the built dashboard serves (`.js`, `.mjs`, `.css`, `.svg`, `.json`, `.woff`, `.woff2`, `.html`) via `mimetypes.add_type` at application import, which overrides the merged registry table on all platforms. +- No behavior change on platforms whose default table is already correct — the pinned values equal the stdlib defaults. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `frontend-architecture`: Dashboard delivery additionally guarantees correct web MIME types independent of the host OS's `mimetypes` registry state. diff --git a/openspec/changes/fix-windows-asset-mime-types/specs/frontend-architecture/spec.md b/openspec/changes/fix-windows-asset-mime-types/specs/frontend-architecture/spec.md new file mode 100644 index 0000000000..ef73075a74 --- /dev/null +++ b/openspec/changes/fix-windows-asset-mime-types/specs/frontend-architecture/spec.md @@ -0,0 +1,39 @@ +## MODIFIED Requirements + +### Requirement: Dashboard serving is compressed, cache-correct, and chart-lazy + +Dashboard API and static-asset responses MUST be served gzip-compressed when the client accepts it, while proxy paths MUST NOT pass through a compressing wrapper. Content-hashed assets under `/assets/` MUST be served with immutable year-long `Cache-Control`; `index.html` MUST remain `no-cache`. Chart vendor code MUST NOT load before first paint: it MUST live in an async-only chunk that is neither statically imported by the entry chunk nor modulepreloaded. Static assets MUST be served with their correct web MIME types (`.js`/`.mjs` as `text/javascript`, `.css` as `text/css`, `.svg` as `image/svg+xml`, `.json` as `application/json`, `.woff`/`.woff2` as `font/woff`/`font/woff2`, `.html` as `text/html`) regardless of the host operating system's `mimetypes` registry state, so strict browser MIME checking never rejects dashboard module scripts. + +#### Scenario: Assets are compressed and immutable + +- **WHEN** a browser requests a hashed asset under `/assets/` with `Accept-Encoding: gzip` +- **THEN** the response is gzip-encoded +- **AND** carries `Cache-Control: public, max-age=31536000, immutable` + +#### Scenario: index.html stays fresh across deploys + +- **WHEN** the SPA shell is requested +- **THEN** the response carries `Cache-Control: no-cache` + +#### Scenario: Proxy streaming paths are never compressed by the dashboard wrapper + +- **WHEN** a request targets a proxy path (`/backend-api/*`, `/v1/*`) +- **THEN** the dashboard gzip middleware passes it through untouched + +#### Scenario: Ranged asset requests bypass compression + +- **WHEN** an asset request carries a `Range` header +- **THEN** the response is served uncompressed with a valid 206 `Content-Range` over unencoded bytes + +#### Scenario: Chart vendor code loads lazily + +- **WHEN** the built dashboard entry page loads +- **THEN** the recharts chunk is not statically imported by the entry chunk and not modulepreloaded +- **AND** charts render correctly once their async chunk loads + +#### Scenario: Module scripts survive a poisoned OS MIME registry + +- **GIVEN** the host operating system maps `.js` to `text/plain` in its `mimetypes` sources (e.g. Windows `HKCR` registry entries) +- **WHEN** a browser requests a hashed `.js` asset under `/assets/` +- **THEN** the response `Content-Type` is `text/javascript` +- **AND** the dashboard SPA boots instead of failing strict module MIME checking diff --git a/openspec/changes/fix-windows-asset-mime-types/tasks.md b/openspec/changes/fix-windows-asset-mime-types/tasks.md new file mode 100644 index 0000000000..b0afb727da --- /dev/null +++ b/openspec/changes/fix-windows-asset-mime-types/tasks.md @@ -0,0 +1,12 @@ +## 1. Fix + +- [x] 1.1 Register `mimetypes.add_type` overrides for all dashboard asset extensions at `app/main.py` import, before any `FileResponse` is constructed + +## 2. Tests + +- [x] 2.1 Route-level regression: with a poisoned `.js -> text/plain` mapping re-registered over, `GET /assets/*.js` serves `text/javascript` (the externally failing product path from issue #1698) +- [x] 2.2 Unit: `_ensure_web_asset_mime_types()` restores every pinned extension after simulated registry poisoning + +## 3. Spec + +- [x] 3.1 Extend the `frontend-architecture` dashboard-delivery requirement with MIME-type correctness independent of the OS registry diff --git a/openspec/changes/goal-restart-thread-header-abandonment/.openspec.yaml b/openspec/changes/goal-restart-thread-header-abandonment/.openspec.yaml new file mode 100644 index 0000000000..0c73c8f54e --- /dev/null +++ b/openspec/changes/goal-restart-thread-header-abandonment/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-15 diff --git a/openspec/changes/goal-restart-thread-header-abandonment/context.md b/openspec/changes/goal-restart-thread-header-abandonment/context.md new file mode 100644 index 0000000000..ba6c52edab --- /dev/null +++ b/openspec/changes/goal-restart-thread-header-abandonment/context.md @@ -0,0 +1,24 @@ +## Purpose + +Close the `#1703` × `#1680` composition hole: current Codex always +sends `thread-id`, so the merged goal-restart recovery never fires. + +## Decision + +Abandonment stays a `session_header` *interpretation* of the raw +process-session key. Request locality may be `thread_header`. Explicit +`turn_state` is unchanged. + +## Failure modes + +- Incremental or file-pinned restarts must still fail closed on the + required owner. +- After retirement, a later thread-id turn must not revive the raw + row as hard ownership. + +## Example + +Process session `sid` maps to quota-exceeded account A. Codex resends +an account-neutral goal body with `session-id: sid` and +`thread-id: t1`. Selection retires `sid` for `session_header`, routes +to B, and later `t1` turns stay on B. diff --git a/openspec/changes/goal-restart-thread-header-abandonment/design.md b/openspec/changes/goal-restart-thread-header-abandonment/design.md new file mode 100644 index 0000000000..3f2988efc6 --- /dev/null +++ b/openspec/changes/goal-restart-thread-header-abandonment/design.md @@ -0,0 +1,56 @@ +## Context + +`#1679` / `#1680` added a proof-gated exception that retires an +unavailable raw `codex_session` owner for `session_header` +interpretation. `#1703` then made `thread-id` the winning locality +source for current Codex. The two compose incorrectly: the flag and +CAS both require `sticky_source == "session_header"`, which current +Codex never is. + +The raw compatibility row is the process-session key. Looking it up +with `continuity_source=thread_header` treats a `session_header` +tombstone as a live hard owner, so even a successful session-only +restart is undone by the next thread-id turn. + +## Goals / Non-Goals + +**Goals:** + +- Account-neutral goal restart with `session-id` + `thread-id` retires + the unavailable raw owner for process-session interpretation and + routes to a replacement. +- Later same-thread turns without a new hard owner stay on that + replacement. +- Explicit `turn_state` of the same text stays hard-bound. + +**Non-Goals:** + +- Changing file-pin, previous-response, conversation, or tool-state + fail-closed ownership. +- Making `thread_header` an abandonment scope on the raw row. +- Dashboard, settings, or schema changes. + +## Decisions + +- Grant `abandon_unavailable_legacy_owner` for `thread_header` only + when a process session is also present. Thread-only clients have no + process-session raw row to retire. +- Allow retirement CAS when request source is `thread_header`. The + write remains `abandonment_scope=session_header`. +- Load the raw `legacy_sticky_key` with `continuity_source=session_header`. + That lookup is process-session interpretation, not thread identity. + +**Alternative considered:** keep CAS gated on request source and only +set the flag. Rejected because the CAS would still not run. + +**Alternative considered:** abandon the raw row for every source. +Rejected because colliding explicit `turn_state` must stay hard. + +## Risks / Trade-offs + +- [Risk] A thread-header request could retire a raw row that was + written as turn-state with equal text. → Mitigation: CAS still + writes `session_header` scope only; turn-state lookup of that text + keeps the stored owner. +- [Risk] Existing tests only exercise `session_id` without `thread-id`. + → Mitigation: add the missing header combination next to those tests. diff --git a/openspec/changes/goal-restart-thread-header-abandonment/proposal.md b/openspec/changes/goal-restart-thread-header-abandonment/proposal.md new file mode 100644 index 0000000000..c4a953aae3 --- /dev/null +++ b/openspec/changes/goal-restart-thread-header-abandonment/proposal.md @@ -0,0 +1,43 @@ +## Why + +Current Codex sends both a shared process `session-id` and a distinct +`thread-id` on a self-contained goal restart. Affinity classifies that +request as `thread_header`, so the one-shot +`abandon_unavailable_legacy_owner` flag never sets and retirement CAS +never runs. The restart stays fail-closed on the unavailable legacy +owner even though the payload is account-neutral. + +## What Changes + +- Grant goal-restart abandonment when a thread-scoped request still + carries a process session, not only when locality source is + `session_header`. +- Let retirement CAS retire the raw process-session row for + `session_header` interpretation from that thread-scoped request. +- Consult the raw process-session row as `session_header` + interpretation so a scoped tombstone hides it from later thread-id + turns. Explicit `turn_state` of the same text stays hard. +- Keep incremental, file-pinned, conversation-bound, and unresolved + tool-state requests fail-closed. + +## Capabilities + +### New Capabilities + +- None. + +### Modified Capabilities + +- `sticky-session-operations`: Current Codex `thread-id` on a + self-contained goal restart MUST still abandon the unavailable raw + process-session owner for `session_header` interpretation and keep + later same-thread continuity on the replacement. + +## Impact + +- `app/modules/proxy/affinity.py` restart-capability gate. +- `app/modules/proxy/_load_balancer/sticky_selection.py` retirement CAS + source check. +- `app/modules/proxy/load_balancer.py` raw-row lookup source. +- Focused affinity and sticky-selection tests. +- No API, schema, setting, dashboard, or wire-format change. diff --git a/openspec/changes/goal-restart-thread-header-abandonment/specs/sticky-session-operations/spec.md b/openspec/changes/goal-restart-thread-header-abandonment/specs/sticky-session-operations/spec.md new file mode 100644 index 0000000000..655f1d7553 --- /dev/null +++ b/openspec/changes/goal-restart-thread-header-abandonment/specs/sticky-session-operations/spec.md @@ -0,0 +1,43 @@ +## ADDED Requirements + +### Requirement: Thread-scoped current Codex restarts still abandon a raw process-session owner + +A self-contained Codex goal-continuation restart that also carries a distinct `thread-id` MUST still be eligible for the existing process-session abandonment exception. The request's thread-scoped locality source MUST NOT prevent the one-shot abandonment capability or the compare-and-set retirement of the raw process-session row. + +The retirement write MUST remain scoped to `session_header` +interpretation of that raw key. An explicit `turn_state` lookup of the +same text MUST stay hard-bound to the stored account. After a +successful retirement, later same-thread turns that have no new hard +owner MUST keep continuity on the replacement account and MUST NOT +treat the `session_header`-abandoned raw row as live hard ownership. + +Ordinary incremental, file-pinned, conversation-bound, and unresolved +tool-state requests MUST remain fail-closed on their required owner. + +#### Scenario: Goal restart with process session and thread-id abandons the unavailable raw owner + +- **GIVEN** a process-session identifier has a raw legacy `codex_session` mapping to account A +- **AND** account A is paused, rate-limited, or quota-exceeded +- **AND** account B is eligible +- **AND** the request also carries a distinct `thread-id` +- **WHEN** Codex sends the recognized goal-continuation marker with an account-neutral self-contained full resend and no other continuity dependency +- **THEN** the proxy marks the still-current raw mapping to account A abandoned only for process-session interpretation +- **AND** it routes the restarted turn to account B +- **AND** subsequent same-thread continuity remains on account B + +#### Scenario: Thread-id on a goal restart cannot erase colliding explicit turn-state ownership + +- **GIVEN** a raw legacy `codex_session` row was written as explicit turn-state ownership for account A +- **AND** a later request carries the same text as a process-session header plus a distinct `thread-id` +- **WHEN** a marked self-contained goal restart abandons that text for process-session interpretation +- **THEN** the restart may select account B +- **AND** an explicit turn-state lookup of the same text remains hard-bound to account A + +#### Scenario: Account-dependent thread-scoped restart stays fail-closed + +- **GIVEN** a process-session identifier has a raw legacy mapping to unavailable account A +- **AND** the request carries a distinct `thread-id` +- **AND** the body has a previous response, conversation, file pin, or unresolved tool state +- **WHEN** the request is selected +- **THEN** the request fails closed on account A +- **AND** the raw mapping is neither deleted nor rebound diff --git a/openspec/changes/goal-restart-thread-header-abandonment/tasks.md b/openspec/changes/goal-restart-thread-header-abandonment/tasks.md new file mode 100644 index 0000000000..788859600e --- /dev/null +++ b/openspec/changes/goal-restart-thread-header-abandonment/tasks.md @@ -0,0 +1,23 @@ +## 1. Implementation + +- [x] 1.1 Grant `abandon_unavailable_legacy_owner` for `thread_header` + when a process session is present and the payload is + account-neutral. +- [x] 1.2 Allow retirement CAS when request source is `thread_header`. + Keep the write scoped to `session_header`. +- [x] 1.3 Load the raw `legacy_sticky_key` as `session_header` + interpretation so a scoped tombstone hides it from later + thread-id turns. + +## 2. Regression coverage + +- [x] 2.1 Assert session-id + thread-id goal restart sets the + abandonment flag; turn-state and account-dependent payloads do + not. +- [x] 2.2 Assert sticky selection retires the raw owner and selects a + replacement when source is `thread_header`. + +## 3. Validation + +- [x] 3.1 Run the focused affinity and sticky-selection tests. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/openspec/changes/harden-shared-future-admission-waits/proposal.md b/openspec/changes/harden-shared-future-admission-waits/proposal.md new file mode 100644 index 0000000000..db45bc1f0e --- /dev/null +++ b/openspec/changes/harden-shared-future-admission-waits/proposal.md @@ -0,0 +1,48 @@ +# Harden shared-future admission waits and surface event-loop lag + +## Why + +On 2026-08-20 a production instance livelocked: the event loop spent ~98% of +its CPU inside `asyncio.Future.remove_done_callback` and kept grinding at full +CPU with zero client sessions attached. Admission waiters were piling onto +shared registry futures via `asyncio.wait_for(asyncio.shield(...))`, which +attaches per-waiter callbacks to the shared future and removes them with O(n) +scans; on Python 3.14 `shield` additionally leaks one callback per attempt +onto a still-pending future, so mass timeouts and client-disconnect storms +degrade to O(n²) and starve the loop. The outage surfaced only as global +slowness and health-check flapping — no signal said "the event loop itself is +starved", which stretched diagnosis by hours. + +## What Changes + +- Replace `wait_for(shield(shared))` on shared, many-waiter futures (http-bridge + inflight/capacity registries, token-refresh singleflight) with a fan-out + helper that keeps exactly one callback on the shared future and gives each + waiter a private O(1)-detach proxy future. Wait semantics (result/exception/ + cancellation propagation, timeout contract, waiter cancellation isolation) + are unchanged. +- Add an event-loop lag watchdog: a once-per-second sampler that exports + `codex_lb_event_loop_lag_seconds` / `codex_lb_event_loop_lag_warnings_total` + and emits a rate-limited warning log when scheduling lag crosses a threshold. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `proxy-admission-control`: admission waits on shared futures must not attach + per-waiter callbacks to the shared object. +- `proxy-runtime-observability`: event-loop scheduling lag is an explicit + operator signal (metrics + rate-limited warning log). + +## Impact + +- `app/core/utils/shared_future.py` (new helper), `app/modules/proxy/_service/http_bridge/mixin.py`, + `app/modules/accounts/auth_manager.py` (call-site swaps; no behavior change). +- `app/core/resilience/loop_lag_monitor.py` (new watchdog), + `app/core/metrics/prometheus.py`, `app/main.py`, + `app/core/config/settings.py` (`event_loop_lag_warn_threshold_seconds`, + default 0.5s, `0` disables; zero-config — no operator action needed). diff --git a/openspec/changes/harden-shared-future-admission-waits/specs/proxy-admission-control/spec.md b/openspec/changes/harden-shared-future-admission-waits/specs/proxy-admission-control/spec.md new file mode 100644 index 0000000000..6f77225445 --- /dev/null +++ b/openspec/changes/harden-shared-future-admission-waits/specs/proxy-admission-control/spec.md @@ -0,0 +1,41 @@ +## ADDED Requirements + +### Requirement: Admission waits on shared futures scale O(1) per waiter + +When multiple requests wait on one shared future (an inflight bridge session +creation, a capacity slot, or a token-refresh singleflight), attaching a +waiter, a waiter timing out, and a waiter being cancelled MUST each perform +O(1) work on the shared future. The shared future MUST carry a constant number +of done callbacks regardless of waiter count, and the wait mechanism itself +MUST NOT cancel or otherwise mutate the shared future or the work it +represents when a waiter times out or is cancelled. Admission handlers MAY +still settle the shared future explicitly after a waiter's timeout (the +http-bridge timeout handler fails and unregisters the inflight future so +piled-up waiters converge on one overload outcome); that settlement is an +admission-contract decision, not a side effect of waiting. The shared future's +result, exception, or cancellation MUST propagate to every waiter with the +same semantics as `asyncio.wait_for(asyncio.shield(shared), timeout)`. + +#### Scenario: Waiter pile-up keeps the shared future's callback list constant + +- **WHEN** many requests wait on the same inflight bridge-session future +- **THEN** the shared future carries a constant number of done callbacks +- **AND** the callback count does not grow with the number of waiters + +#### Scenario: Mass timeout does not degrade the event loop + +- **GIVEN** waiters piled onto a shared future that has not resolved within + the admission wait timeout +- **WHEN** the waiters time out together +- **THEN** each timeout detaches in O(1) without scanning the shared future's + callback list +- **AND** the surviving admission contract (local-overload `429` with the + capacity error code) is unchanged + +#### Scenario: Client-disconnect storm leaves the owner's creation running + +- **WHEN** every waiter on an inflight session future is cancelled by client + disconnects +- **THEN** the shared future stays pending and the owner's session creation + continues +- **AND** no per-waiter callbacks remain attached to the shared future diff --git a/openspec/changes/harden-shared-future-admission-waits/specs/proxy-runtime-observability/spec.md b/openspec/changes/harden-shared-future-admission-waits/specs/proxy-runtime-observability/spec.md new file mode 100644 index 0000000000..97752ff627 --- /dev/null +++ b/openspec/changes/harden-shared-future-admission-waits/specs/proxy-runtime-observability/spec.md @@ -0,0 +1,32 @@ +## ADDED Requirements + +### Requirement: Event-loop scheduling lag is observable + +The system MUST sample event-loop scheduling lag (timer drift of a +once-per-second sleep) while serving and export it as the +`codex_lb_event_loop_lag_seconds` gauge. Samples at or above the configured +warning threshold MUST increment `codex_lb_event_loop_lag_warnings_total` and +emit a warning log that names the observed lag, the worst lag suppressed since +the previous line, and the threshold; the warning log MUST be rate-limited so +a sustained stall cannot flood the log. The threshold MUST be configurable via +`event_loop_lag_warn_threshold_seconds` with a working default requiring no +operator action, and `0` MUST disable the watchdog. + +#### Scenario: Starved event loop produces an explicit operator signal + +- **WHEN** the event loop is starved (callback storm, synchronous work on the + loop, or CPU saturation) and scheduling lag reaches the warning threshold +- **THEN** `codex_lb_event_loop_lag_warnings_total` increments +- **AND** a rate-limited `event_loop_lag` warning names the observed lag and + threshold, distinguishing loop starvation from upstream slowness + +#### Scenario: Healthy loop stays quiet + +- **WHEN** scheduling lag stays below the warning threshold +- **THEN** the gauge is still updated for dashboards +- **AND** no warning is logged and the warning counter does not increment + +#### Scenario: Watchdog can be disabled + +- **WHEN** `event_loop_lag_warn_threshold_seconds` is set to `0` +- **THEN** the watchdog task is not started diff --git a/openspec/changes/harden-shared-future-admission-waits/tasks.md b/openspec/changes/harden-shared-future-admission-waits/tasks.md new file mode 100644 index 0000000000..e547ca13fe --- /dev/null +++ b/openspec/changes/harden-shared-future-admission-waits/tasks.md @@ -0,0 +1,18 @@ +## 1. Shared-future waiter fan-out + +- [x] 1.1 Add `wait_on_shared_future` (`app/core/utils/shared_future.py`): one fan-out callback on the shared future, per-waiter proxy futures with O(1) attach/detach, `wait_for(shield())`-equivalent semantics. +- [x] 1.2 Swap the http-bridge admission wait sites (inflight session future, capacity wait future) in `app/modules/proxy/_service/http_bridge/mixin.py` to the helper. +- [x] 1.3 Swap the token-refresh singleflight wait in `app/modules/accounts/auth_manager.py` to the helper. + +## 2. Event-loop lag watchdog + +- [x] 2.1 Add `app/core/resilience/loop_lag_monitor.py`: 1s sleep-drift sampler, gauge + counter export, rate-limited warning log. +- [x] 2.2 Register `codex_lb_event_loop_lag_seconds` and `codex_lb_event_loop_lag_warnings_total` in `app/core/metrics/prometheus.py` (both branches + `__all__`). +- [x] 2.3 Wire the monitor task into the app lifespan (`app/main.py`) behind `event_loop_lag_warn_threshold_seconds` (default 0.5, `0` disables), cancelled on shutdown. + +## 3. Verification + +- [x] 3.1 Helper semantics tests (`tests/unit/test_shared_future_waiters.py`): result/exception/cancellation propagation, timeout leaves shared pending, mass-timeout keeps callback count at 1, waiter cancellation isolation, singleflight task survival. +- [x] 3.2 Bridge-surface regression test (`tests/unit/test_proxy_http_bridge.py::test_admission_waiters_do_not_accumulate_callbacks_on_shared_inflight_future`): 50 admission waiters on one inflight future keep exactly one shared callback through a cancellation storm and mass timeout; verified to fail against the old shield pattern. +- [x] 3.3 Watchdog tests (`tests/unit/test_loop_lag_monitor.py`): starved loop warns + increments counter, healthy loop stays quiet, warning log rate-limited. +- [x] 3.4 Run affected unit suites, ruff, and strict OpenSpec validation. diff --git a/openspec/changes/hold-operation-fenced-hard-turn-cooldown/.openspec.yaml b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/.openspec.yaml new file mode 100644 index 0000000000..4af864176c --- /dev/null +++ b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-14 diff --git a/openspec/changes/hold-operation-fenced-hard-turn-cooldown/design.md b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/design.md new file mode 100644 index 0000000000..11b8db08ef --- /dev/null +++ b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/design.md @@ -0,0 +1,41 @@ +## Context + +Hard turn-state requests can omit `previous_response_id` while still carrying a +real Codex turn-state continuity anchor. Their replay identity is protected by +the durable operation ledger, but the startup cooldown guard runs before +operation registration. It therefore classifies the request as continuity-bound +without safe replay and returns 503 before the ledger can serialize recovery. + +The HTTP response already includes `Retry-After`, and an already-started SSE +failure includes an SSE `retry:` directive. Production telemetry shows Codex +Desktop retrying in milliseconds anyway, so another client hint does not address +the observed failure mode. + +## Decision + +Treat a turn-state-only hard request as eligible to wait through cooldown only +when all of the following hold: + +- recovery mode is `server_anchored_replay_once` or + `server_indefinite_recovery`; +- the durable operation ledger is enabled; +- the request has a real hard continuity anchor; +- the bridge has both a durable session id and current owner epoch; +- no response id or upstream response event has been observed; and +- request budget remains. + +The wait is clamped to the smaller of cooldown remaining and request budget. +It does not reserve a replay, mutate the operation journal, or send upstream. +When the cooldown expires, normal submission performs the existing operation +fingerprint lookup and atomic recovery claim. One-shot mode keeps its existing +maximum of one recovery dispatch; indefinite mode retains its existing explicit +opt-in semantics. + +## Explicit exclusions + +- No change to the default `fail_closed` mode. +- No transparent replay without a durable session and owner fence. +- No cross-account, file-pinned, image, soft-affinity, or eventful recovery. +- No weakening of operation fingerprint, ownership, or replay-count checks. +- No infinite retry added by this change; bounded one-shot mode is the + recommended deployment setting for this incident class. diff --git a/openspec/changes/hold-operation-fenced-hard-turn-cooldown/proposal.md b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/proposal.md new file mode 100644 index 0000000000..5c6d83bd73 --- /dev/null +++ b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/proposal.md @@ -0,0 +1,27 @@ +## Why + +When two eventless upstream attempts open the HTTP bridge retry circuit, Codex +Desktop immediately retries the same hard turn-state request. The bridge +currently returns a startup 503 before consulting the durable operation ledger. +Codex does not honor the full retry-circuit delay and can exhaust its client +retry budget during the cooldown, pausing the task even though the bridge and +VPS remain healthy. + +## What Changes + +- In an explicitly enabled server recovery mode, hold a turn-state-only hard + continuation through the active retry-circuit cooldown before submission. +- Require a live durable session id and owner epoch, zero response events, and + no response id before waiting. +- Dispatch nothing while waiting. After cooldown, use the existing durable + operation ledger and one-shot/indefinite recovery policy to arbitrate whether + the request may be created, claimed, replayed, or failed closed. +- Preserve the current immediate 503 for the default `fail_closed` mode, + in-memory fallback sessions, soft affinity, and eventful requests. +- Emit a low-cardinality bridge event when the operation-fenced wait begins. + +## Impact + +- HTTP Responses bridge startup behavior during retry-circuit cooldown. +- No database schema, public API, account routing, or default configuration + change. diff --git a/openspec/changes/hold-operation-fenced-hard-turn-cooldown/specs/responses-api-compat/spec.md b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..a487f23e1f --- /dev/null +++ b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/specs/responses-api-compat/spec.md @@ -0,0 +1,67 @@ +## ADDED Requirements + +### Requirement: Operation-fenced hard turns preserve client retry budget during cooldown + +A hard turn-state HTTP bridge request arriving during retry-circuit cooldown MUST remain pending until cooldown expires only if an explicit server recovery mode is enabled, the request has not observed a response id or response event, and the bridge has a live durable session and owner epoch. The proxy MUST NOT dispatch upstream while waiting. After the wait, the request MUST pass through the existing durable operation-ledger admission before any `response.create` is sent. + +#### Scenario: One-shot hard turn waits before durable arbitration + +- **GIVEN** `server_anchored_replay_once` is enabled +- **AND** a turn-state-only hard continuation has a live durable owner +- **AND** its retry circuit is cooling down before submission +- **WHEN** the request reaches bridge startup +- **THEN** the proxy waits for the bounded cooldown instead of returning 503 +- **AND** it sends no upstream request during the wait +- **AND** normal durable operation admission runs after cooldown + +#### Scenario: Missing durable fence remains fail closed + +- **GIVEN** a turn-state-only hard continuation has no durable session or owner + epoch +- **WHEN** its retry circuit is cooling down +- **THEN** the proxy does not wait or dispatch upstream +- **AND** it returns the existing cooldown failure with a retry hint + +#### Scenario: Operation ledger disabled remains fail closed + +- **GIVEN** ambiguous continuation recovery mode is enabled +- **AND** a turn-state-only hard continuation has a live durable session and + owner epoch +- **AND** the durable operation ledger is disabled +- **WHEN** its retry circuit is cooling down before submission +- **THEN** the proxy preserves the existing cooldown failure +- **AND** it does not wait or dispatch upstream + +#### Scenario: Default mode remains fail closed + +- **GIVEN** ambiguous continuation recovery mode is `fail_closed` +- **WHEN** any continuity-bound hard request arrives during cooldown +- **THEN** the proxy preserves the existing immediate cooldown failure +- **AND** it does not create or claim a durable recovery operation + +#### Scenario: Request budget expires while waiting + +- **GIVEN** an operation-fenced hard turn is allowed to wait through cooldown +- **AND** its request budget expires before the cooldown does +- **WHEN** the bounded wait reaches the request deadline +- **THEN** the proxy releases the request reservation and returns a terminal + timeout +- **AND** it does not submit `response.create` after the deadline + +#### Scenario: Cooldown waiter stays within the per-session queue limit + +- **GIVEN** an operation-fenced hard turn is eligible to wait through cooldown +- **AND** the bridge session is already at its configured queue limit +- **WHEN** the request reaches the cooldown wait point before submission +- **THEN** the proxy rejects the request with the existing bridge queue full + error +- **AND** it does not sleep or dispatch upstream + +#### Scenario: Durable ownership is renewed while the cooldown wait is pending + +- **GIVEN** an operation-fenced hard turn is waiting through startup cooldown +- **AND** the cooldown exceeds one durable lease refresh cadence +- **WHEN** the wait continues before submission +- **THEN** the proxy renews and revalidates the durable owner lease before the + wait completes +- **AND** it fails closed if durable ownership changes during the wait diff --git a/openspec/changes/hold-operation-fenced-hard-turn-cooldown/tasks.md b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/tasks.md new file mode 100644 index 0000000000..3f7ccce4ae --- /dev/null +++ b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/tasks.md @@ -0,0 +1,8 @@ +- [x] 1. Reproduce the production turn-state-only startup cooldown as a unit + regression that currently returns 503 before submission. +- [x] 2. Hold only explicitly enabled, zero-event, durable operation-fenced hard + turns through the bounded cooldown. +- [x] 3. Preserve fail-closed behavior when the durable session/owner proof is + absent and keep one-shot recovery bounded by the existing atomic claim. +- [x] 4. Run focused tests, relevant bridge suites, Ruff, type/architecture + checks, whitespace checks, and strict OpenSpec validation. diff --git a/openspec/changes/isolate-request-and-refresh-db-sessions/design.md b/openspec/changes/isolate-request-and-refresh-db-sessions/design.md deleted file mode 100644 index 26475e2f3e..0000000000 --- a/openspec/changes/isolate-request-and-refresh-db-sessions/design.md +++ /dev/null @@ -1,55 +0,0 @@ -## Context - -The service has two equally sized SQLAlchemy engines: one for foreground -requests and one for detached/background work. Production configured each -engine with two steady connections plus one overflow connection. Auth Guardian -also ran three refresh candidates concurrently. Each candidate retained an -outer repository session while a shielded AuthManager task attempted additional -background checkouts and upstream OAuth. At the same time, request firewall -and authentication cache misses incorrectly requested background sessions. - -## Goals / Non-Goals - -**Goals:** - -- Keep foreground request admission available when background work is busy. -- Ensure no token-refresh database checkout spans admission waits, peer-claim - waits, or external OAuth I/O. -- Preserve singleflight, refresh claims, ciphertext-guarded CAS, peer adoption, - shielded cancellation, and fail-closed firewall/auth behavior. -- Expose residual foreground checkout exhaustion as a sanitized retryable 503. - -**Non-Goals:** - -- Increasing PostgreSQL `max_connections` or adding another pool. -- Weakening refresh claims, response replay proofs, or credential identity. -- Treating a larger pool as the product fix. - -## Decisions - -- Add one reusable `get_request_session()` context over the existing main - `SessionLocal`; the FastAPI dependency delegates to the same owner. -- Keep `get_background_session()` for schedulers and detached operations. -- Use the existing `BackgroundAccountsRepository` as AuthManager's production - port. Every method opens, uses, detaches from, and closes a short session. -- Guardian closes its candidate repository before constructing or awaiting the - shielded refresh owner. -- ProxyService accepts an explicit per-operation refresh repository in - production while retaining the existing factory seam for isolated tests. -- SQLAlchemy checkout timeout is a fail-closed 503 with OpenAI/dashboard-native - envelopes and `Retry-After`; exception details and pool topology are never - returned to clients. - -## Risks / Trade-offs - -- [A detached ORM object is accessed after close] → The per-operation - repository expunges returned objects before closing; focused tests exercise - Guardian use after candidate scope exit. -- [A caller bypasses production injection] → The application composition - root explicitly injects `BackgroundAccountsRepository`; the compatibility - factory remains only for tests/custom constructors. -- [The main pool is itself exhausted] → Fail closed with a stable 503 and - `Retry-After`, preserving firewall enforcement and hiding SQLAlchemy details. -- [A pool-size increase masks the bug] → Session-lifetime tests assert the - repository scope is closed at the upstream barrier rather than relying on - capacity alone. diff --git a/openspec/changes/isolate-request-and-refresh-db-sessions/proposal.md b/openspec/changes/isolate-request-and-refresh-db-sessions/proposal.md deleted file mode 100644 index 38e36c3f62..0000000000 --- a/openspec/changes/isolate-request-and-refresh-db-sessions/proposal.md +++ /dev/null @@ -1,36 +0,0 @@ -## Why - -Production `/v1/responses` requests returned raw HTTP 500 errors even though -PostgreSQL had ample server capacity. Three Auth Guardian refresh candidates -could retain all three background-pool checkouts while waiting for nested -refresh work and upstream OAuth. Firewall and API-key cache misses also used -that background pool, so scheduler starvation reached the request front door. - -## What Changes - -- Give foreground middleware, authentication, and proxy repositories an - explicit main/request-pool session boundary. -- Make Guardian candidate reads and token-refresh persistence use short, - per-operation background sessions that close before waits or upstream I/O. -- Return a sanitized, retryable HTTP 503 response when a foreground database - checkout times out instead of exposing an internal QueuePool error as 500. -- Add behavioral coverage for cancellation cleanup, pool ownership, Guardian - refresh lifetime, and the externally visible retry contract. - -## Capabilities - -### New Capabilities - -None. - -### Modified Capabilities - -- `database-backends`: Separate foreground and detached/background checkout - ownership and forbid token refresh from holding a checkout across network I/O. - -## Impact - -Affected surfaces are database session ownership, firewall/API-key/Codex -identity cache misses, Auth Guardian refresh, ProxyService token refresh, the -pool-timeout error envelope, and focused unit/integration tests. No schema, -migration, credential, API-key identity, or replay-safety rule changes. diff --git a/openspec/changes/isolate-request-and-refresh-db-sessions/specs/database-backends/spec.md b/openspec/changes/isolate-request-and-refresh-db-sessions/specs/database-backends/spec.md deleted file mode 100644 index 2a1c9a2186..0000000000 --- a/openspec/changes/isolate-request-and-refresh-db-sessions/specs/database-backends/spec.md +++ /dev/null @@ -1,74 +0,0 @@ -## MODIFIED Requirements - -### Requirement: Database pool controls cover isolated request and background sessions - -The service SHALL size both the main request pool and the background-task pool -from `database_pool_size` and `database_max_overflow`. The background pool -SHALL always derive from those two settings; it exists to isolate detached and -scheduler checkouts from the request pool, not to be sized independently. - -Foreground middleware, authentication cache misses, identity reads, and proxy -request repositories MUST use the main request pool. Detached refresh, -scheduler, claim, and background persistence operations MUST use the background -pool. A foreground checkout timeout MUST fail closed and return a sanitized -HTTP 503 response with `Retry-After`; it MUST NOT expose SQLAlchemy pool details -or bypass firewall/authentication. - -#### Scenario: Background pool inherits main pool capacity - -- **WHEN** the application creates the background-task DB engine for a pooled backend -- **THEN** the background pool uses `database_pool_size` and `database_max_overflow` -- **AND** no separate background pool sizing setting exists - -#### Scenario: Background saturation does not consume request-pool checkouts - -- **GIVEN** detached refresh or scheduler work has consumed every available background checkout -- **WHEN** a foreground firewall or API-key cache miss needs database state -- **THEN** it acquires a main/request-pool session -- **AND** it does not wait for a background checkout -- **AND** firewall and authentication remain fail closed - -#### Scenario: Foreground checkout timeout is retryable and sanitized - -- **WHEN** a foreground database checkout exceeds its bounded pool wait -- **THEN** the HTTP response status is 503 -- **AND** the response includes `Retry-After` -- **AND** the response uses the route's native error envelope -- **AND** no SQLAlchemy exception text, connection string, or pool topology is returned - -### Requirement: Detached background tasks own short database session lifetimes - -Detached background tasks MUST own database session lifetime independently from -cancellable callers. A task intentionally decoupled from its caller (including -a singleflight refresh held alive by `asyncio.shield`) MUST NOT use a session -owned by the cancellable caller. - -Token and account refresh owners MUST perform candidate reads, refresh-claim -operations, fresh state reads, route resolution, CAS persistence, peer adoption, -and claim release through short, independently owned database operations. All -such database scopes MUST close before admission waits, peer-claim waits, or -external network I/O. Shielding and cancellation MUST NOT strand a checkout. - -#### Scenario: Guardian refresh closes candidate read before upstream work - -- **GIVEN** Auth Guardian selects a stale eligible account -- **WHEN** it starts the shielded account refresh -- **THEN** the repository used to read and validate the candidate has already closed -- **AND** refresh database operations use independent short background sessions -- **AND** the upstream OAuth wait holds no candidate-read checkout - -#### Scenario: Client cancellation does not strand refresh connections - -- **GIVEN** a proxy request joins a shielded singleflight token refresh -- **WHEN** the client task is cancelled while upstream refresh work continues -- **THEN** the refresh finishes or fails through independently owned short sessions -- **AND** refresh claims and guarded token state reach their normal terminal behavior -- **AND** every database checkout returns after the refresh task drains - -#### Scenario: Peer rotation remains authoritative - -- **GIVEN** another replica rotates the account token while this replica performs upstream OAuth -- **WHEN** this replica attempts its guarded persistence phase -- **THEN** ciphertext-guarded CAS prevents overwriting the peer token -- **AND** the peer row is adopted through a fresh short read -- **AND** the refresh claim is released through an independent short operation diff --git a/openspec/changes/isolate-request-and-refresh-db-sessions/tasks.md b/openspec/changes/isolate-request-and-refresh-db-sessions/tasks.md deleted file mode 100644 index b6e3b45275..0000000000 --- a/openspec/changes/isolate-request-and-refresh-db-sessions/tasks.md +++ /dev/null @@ -1,21 +0,0 @@ -## 1. Session ownership - -- [x] 1.1 Add the explicit main/request-pool session context and preserve robust rollback/close semantics. -- [x] 1.2 Route firewall, API-key, Codex identity, and foreground proxy repositories through the request pool. -- [x] 1.3 Bind production token refresh to per-operation background repositories. - -## 2. Refresh lifetime - -- [x] 2.1 Close Guardian candidate reads before shielded refresh work. -- [x] 2.2 Preserve singleflight, claim, CAS, peer-adoption, and cancellation semantics. - -## 3. Failure contract - -- [x] 3.1 Render database checkout exhaustion as sanitized HTTP 503 plus `Retry-After`. -- [x] 3.2 Keep firewall/auth fail closed and prevent internal pool detail leakage. - -## 4. Verification - -- [x] 4.1 Add request-pool ownership, cancellation, Guardian lifetime, and timeout-envelope tests. -- [x] 4.2 Run complete proportional unit/integration, lint, type, OpenSpec, and diff gates. -- [ ] 4.3 Deploy through normal source/image/GitOps reviews and prove multi-session production stability. diff --git a/openspec/changes/keep-aborted-invalidation-bumps-pending/proposal.md b/openspec/changes/keep-aborted-invalidation-bumps-pending/proposal.md new file mode 100644 index 0000000000..f4e85b9530 --- /dev/null +++ b/openspec/changes/keep-aborted-invalidation-bumps-pending/proposal.md @@ -0,0 +1,27 @@ +## Why + +The invalidation-bus spec already requires that "coalesced (`request_bump`) namespaces MUST remain pending and be retried on subsequent poll cycles until a bump succeeds". The implementation violated it for one case. + +`_flush_pending_bumps` clears each namespace's pending marker before awaiting its write — deliberately, so a `request_bump()` arriving mid-write re-queues instead of being coalesced into the version already being written. But it restored the marker only when `bump()` returned `False`. A write that was **cancelled** or **raised** left the namespace neither written nor pending, with nothing logged and no retry holding it. Since `_run` swallows poll exceptions and keeps cycling, a raising write silently lost its namespace during ordinary operation. + +## What Changes + +- Restore the pending marker when the bump write aborts, so the required retry actually happens. The two abort kinds are handled differently: `CancelledError` restores and re-raises (task teardown must abort the flush), while an ordinary `Exception` — abnormal, since `bump()` reports failure by returning `False` — restores, logs at warning, and continues, so a persistently raising namespace cannot starve the namespaces sorting after it. + +The restore is unconditional even when the abort's outcome is ambiguous (cancellation or a driver error arriving after the database accepted the commit): a redundant bump only re-runs peers' idempotent invalidation callbacks, while dropping an unconfirmed write leaves them stale until the fallback TTL. The bus already tolerates extra version increments — `request_bump` arriving mid-flush deliberately produces one. + +Process shutdown is deliberately out of scope: `stop()` cancels the polling task, so a bump queued at that moment has no cycle left to drain it. That is already the documented contract — "a lost bump still converges within the fallback TTL" — and guaranteeing delivery against an unresponsive database at shutdown is a separate concern with its own bounding and task-ownership design. + +## Why the ambiguous case still restores + +A cancellation or driver error can arrive after the database accepted the commit, so the restore can produce a redundant bump. That is the deliberate trade: a redundant bump only re-runs peers' idempotent invalidation callbacks, while dropping an unconfirmed write leaves them stale until the fallback TTL. The bus already tolerates extra increments — a `request_bump` arriving mid-flush produces one by design. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `query-caching`: state explicitly that an aborted (not merely failed) write keeps its namespace queued. diff --git a/openspec/changes/keep-aborted-invalidation-bumps-pending/specs/query-caching/spec.md b/openspec/changes/keep-aborted-invalidation-bumps-pending/specs/query-caching/spec.md new file mode 100644 index 0000000000..885ddae521 --- /dev/null +++ b/openspec/changes/keep-aborted-invalidation-bumps-pending/specs/query-caching/spec.md @@ -0,0 +1,57 @@ +## MODIFIED Requirements + +### Requirement: Cache invalidation bumps and polling are resilient and observable +`bump()` MUST retry transient write failures (including SQLite "database is locked") with a short backoff; on final failure it MUST log at ERROR with the namespace, increment `codex_lb_cache_invalidation_bump_failures_total{namespace}`, and MUST NOT fail the originating mutation. Coalesced (`request_bump`) namespaces MUST remain pending and be retried on subsequent poll cycles until a bump succeeds, including when the write aborts rather than merely failing: an aborted write MUST restore the pending marker regardless of whether the database had already accepted its commit. A write that raises MUST NOT prevent the remaining pending namespaces from flushing in the same cycle. A `request_bump` arriving while a flush for the same namespace is already awaiting its bump MUST be preserved and produce a later bump. When any invalidation callback for a namespace fails, the poller MUST NOT acknowledge the observed version and MUST re-run that namespace's callbacks on subsequent poll cycles until they succeed. The poller MUST escalate consecutive poll failures above debug level after a bounded count (WARNING after 3, ERROR after 10) and increment `codex_lb_cache_invalidation_poll_failures_total`. + +#### Scenario: Bump failure under database lock is observable and does not fail the mutation + +- **GIVEN** the database rejects cache-invalidation writes with a lock error for longer than the retry budget +- **WHEN** a mutation attempts a durable namespace bump +- **THEN** the mutation itself still succeeds +- **AND** an ERROR log naming the namespace is emitted and the bump-failure counter increments + +#### Scenario: Pending coalesced namespace flushes on the next successful cycle + +- **GIVEN** a coalesced `request_bump` namespace failed to flush during a poll cycle +- **WHEN** the database becomes writable again +- **THEN** the next poll cycle flushes the pending namespace and increments its version + +#### Scenario: Bump requested during an in-flight flush produces a later bump + +- **GIVEN** a coalesced flush is awaiting the bump write for a namespace +- **WHEN** another mutation commits and requests a bump for the same namespace before the flush completes +- **THEN** the namespace is re-queued and flushed again on a subsequent cycle, incrementing the version beyond the in-flight bump + +#### Scenario: Failed invalidation callback keeps the version unacknowledged and is retried + +- **GIVEN** a replica observes an `account_routing` version bump +- **AND** its routing snapshot refresh fails with a transient database error +- **WHEN** the poll cycle completes +- **THEN** the replica does not record the new version as seen +- **AND** the refresh is retried on subsequent poll cycles until it succeeds + +#### Scenario: Consecutive poll failures escalate above debug + +- **GIVEN** a replica's poller cannot read the `cache_invalidation` table +- **WHEN** three consecutive polls fail +- **THEN** a WARNING is logged and the poll-failure counter increments + +#### Scenario: An aborted bump write keeps its namespace queued + +- **GIVEN** a coalesced flush has cleared a namespace's pending marker and is awaiting its bump write +- **WHEN** that write aborts — cancelled or raised — before the database accepts its commit +- **THEN** the namespace is restored to the pending set for a later cycle, and no version is written + +#### Scenario: A raising namespace does not starve the others + +- **GIVEN** two pending namespaces where the first (in sort order) raises on every bump attempt +- **WHEN** a flush cycle runs +- **THEN** the raising namespace stays pending with no version written +- **AND** the other namespace is bumped in that same cycle + +#### Scenario: An abort after the commit was accepted still restores the namespace + +- **GIVEN** a bump write aborts — cancelled, or the driver raises — after the database accepted its commit but before completion is reported +- **WHEN** the abort is handled +- **THEN** the namespace is restored to the pending set and bumped on a later cycle +- **AND** the resulting duplicate version increment is accepted diff --git a/openspec/changes/keep-aborted-invalidation-bumps-pending/tasks.md b/openspec/changes/keep-aborted-invalidation-bumps-pending/tasks.md new file mode 100644 index 0000000000..285fbf9efc --- /dev/null +++ b/openspec/changes/keep-aborted-invalidation-bumps-pending/tasks.md @@ -0,0 +1,13 @@ +## 1. Fix + +- [x] 1.1 Restore the pending namespace in `_flush_pending_bumps` when the bump write aborts: cancellation restores and re-raises; an abnormal raise restores, logs, and continues with the remaining namespaces so a persistently raising namespace cannot starve the ones sorting after it + +## 2. Tests + +- [x] 2.1 A cancelled write restores the marker, and the marker is cleared before the write (locking in the intended coalescing) +- [x] 2.2 A raising write restores the marker and does not block later namespaces from flushing +- [x] 2.3 End-to-end: the running poller retries the aborted namespace and writes its version + +## 3. Spec + +- [x] 3.1 Make "remains pending" explicitly cover an aborted write, not only a failed one, keeping the requirement normative and the rationale in the proposal diff --git a/openspec/changes/keep-abrupt-eventless-drop-account-neutral/context.md b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/context.md new file mode 100644 index 0000000000..39d600e4fb --- /dev/null +++ b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/context.md @@ -0,0 +1,23 @@ +A websocket that dies without a close frame before any application-layer +response event is the weakest possible evidence of account ill-health. The +reader failure path nevertheless penalized it because +`close_classification` was computed only for `close_code is not None`, so +the frame-less case fell through to the default `penalize_account=True`. +Meanwhile the strictly stronger signal — a graceful 1000 close with zero +events — was already exempted, and #1718 established the same precedent for +stream idle timeouts. + +The fix adds `_is_account_neutral_transport_drop` beside +`_classify_upstream_close` (no close frame AND zero response events) and +consults it in the reader failure path. To avoid masking a genuine account +ban that manifests as repeated drops, the neutral drop is recorded into the +existing `_record_http_bridge_account_timeout_signal` accumulator: three +eventless failures within the 300-second window still apply the minimum +drain penalty. No new settings are introduced. + +Incident shape from #1754: three drops ~12 minutes apart never meet the +300-second window, so the owner stays routable and continuity-bound +follow-ups keep working; before the fix they crossed +`ERROR_BACKOFF_THRESHOLD` and produced eight +`previous_response_owner_unavailable` 502s in eight seconds while the other +pool account idled. diff --git a/openspec/changes/keep-abrupt-eventless-drop-account-neutral/proposal.md b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/proposal.md new file mode 100644 index 0000000000..0ef5273b1d --- /dev/null +++ b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/proposal.md @@ -0,0 +1,50 @@ +# Why + +An abrupt upstream websocket drop with no close frame and zero response +events is charged to the account: the HTTP bridge reader only consults +`_classify_upstream_close` when a close frame arrived, so a frame-less +transport reset always sets `penalize_account=True` while a graceful 1000 +close before any response event is exempted. That is inverted with respect +to the available evidence — the account never spoke at the application +layer. Three such drops cross the error-backoff threshold and 502 every +continuity-bound follow-up (`previous_response_owner_unavailable`) for 30 +seconds while healthy pool siblings idle (issue #1754). + +# What Changes + +- Classify an abrupt upstream websocket ending — a terminal close or receive + error with no upstream-authored close frame (the synthetic abnormal-closure + code 1006 counts as frame-less), no established account-neutral transport + classification, and no observed application-layer output — as + account-neutral in the HTTP bridge reader failure path: no `record_error` + health write for the individual drop. +- Keep the existing penalty when an upstream-authored close frame arrived + (including non-clean codes such as 1008/1011), when application-layer + output was already observed (streamed response events or a buffered + reasoning prelude), or when a non-terminal protocol-invalid frame (for + example a binary message) triggered the failure, and keep all established + account-neutral transport codes on their existing contract. +- Feed account-neutral eventless drops that settle their pending requests as + failures into the existing windowed eventless account drain signal so + repeated drops on the same account still drain it (same threshold/window + as repeated eventless upstream timeouts), keeping genuine account faults + visible. Drops recovered by the bounded pre-created replay keep their + existing behavior. +- The per-bridge retry circuit continues to record the failure at bridge + scope, unchanged. + +# Capabilities + +## Modified Capabilities + +- `responses-api-compat`: HTTP bridge abrupt eventless upstream drops must + stay account-neutral for per-drop health writes while repeated drops still + drain the account through the windowed eventless failure signal. + +# Impact + +Continuity-bound conversations survive sporadic infrastructure resets +instead of 502-storming against a self-inflicted 30-second owner backoff. +Accounts whose sockets repeatedly drop eventlessly are still drained by the +existing windowed signal. Clean-close, close-frame, and mid-stream drop +semantics are unchanged, as is the per-bridge retry circuit. diff --git a/openspec/changes/keep-abrupt-eventless-drop-account-neutral/specs/responses-api-compat/spec.md b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..9fafb71553 --- /dev/null +++ b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/specs/responses-api-compat/spec.md @@ -0,0 +1,25 @@ +## ADDED Requirements + +### Requirement: Abrupt eventless upstream websocket drops remain account-neutral + +When an HTTP bridge upstream websocket ends with a terminal transport message (a close or receive error) that carries no upstream-authored close frame, no established account-neutral transport classification (process-network, liveness-timeout, keepalive-timeout), and no application-layer output was observed for the pending requests (zero response events and no buffered reasoning prelude), the proxy MUST NOT write per-drop account error-health (`record_error`) for that unclassified `stream_incomplete` drop. The synthetic abnormal-closure code 1006, which RFC 6455 reserves and which adapters synthesize locally when the socket dies without a close frame, MUST be treated as frame-less. When such a drop settles its pending requests as failures, the proxy MUST record it into the windowed eventless account failure signal so that repeated eventless drops on the same account within the window still apply the drain penalty; drops recovered by the bounded pre-created replay keep their existing behavior, and drops already covered by an established account-neutral transport classification keep their existing contract and are not added to the signal. A failure that carries an upstream-authored close frame (including non-clean codes), occurs after application-layer output was observed, or arrives as a non-terminal protocol-invalid frame (for example a binary message) MUST keep the existing account penalty semantics. The per-bridge retry circuit MUST still record the failure at bridge scope. + +#### Scenario: Sporadic frame-less drops do not strand a continuity-bound conversation + +- **GIVEN** a conversation continuity-bound to account A via `previous_response_id` +- **AND** account A's upstream websocket drops three times with no close frame and zero response events, spread wider than the eventless failure window +- **WHEN** the client sends the next continuity-bound follow-up +- **THEN** account A's `error_count` receives no per-drop increment and stays below the error-backoff threshold +- **AND** the follow-up still routes to account A instead of failing with `previous_response_owner_unavailable` + +#### Scenario: Repeated eventless drops inside the window still drain the account + +- **GIVEN** an account whose upstream websocket drops with no close frame and zero response events on three separate bridge failures within the eventless failure window +- **WHEN** the third drop is recorded +- **THEN** the windowed eventless failure signal applies the minimum drain penalty so new turns avoid the account until its health probe succeeds + +#### Scenario: Close frames and observed-output drops keep the account penalty + +- **GIVEN** an upstream websocket ending that carries an upstream-authored close frame (for example 1008 or 1011) before any response event, or a frame-less drop after application-layer output was observed (streamed response events or a buffered reasoning prelude), or a non-terminal protocol-invalid binary frame +- **WHEN** the reader failure path settles the pending requests +- **THEN** the account penalty semantics are unchanged from before this change diff --git a/openspec/changes/keep-abrupt-eventless-drop-account-neutral/tasks.md b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/tasks.md new file mode 100644 index 0000000000..b03daf3a4f --- /dev/null +++ b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/tasks.md @@ -0,0 +1,31 @@ +## 1. Implementation + +- [x] 1.1 Add `_is_account_neutral_transport_drop` (no close frame AND zero + response events) beside `_classify_upstream_close`. +- [x] 1.2 Consult it in the HTTP bridge reader failure path so the frame-less + eventless drop no longer sets `penalize_account=True`. +- [x] 1.3 Record account-neutral drops into the windowed eventless account + drain signal (`_record_http_bridge_account_timeout_signal`) so repeated + drops still drain the account. + +## 2. Regression coverage + +- [x] 2.1 Flip the `[routed-receive-error]` pin intentionally: + `penalize_account is False` for a frame-less eventless drop. +- [x] 2.2 Assert the drop records the windowed drain signal with + `detail=eventless_transport_drop`. +- [x] 2.3 Assert a drop after streamed response events still penalizes. +- [x] 2.4 Assert a non-clean close frame (1008/1011) with zero events still + penalizes. +- [x] 2.5 Assert a non-terminal protocol-invalid binary frame still penalizes + and records no drop signal. +- [x] 2.6 Assert the synthetic abnormal-closure code 1006 counts as + frame-less and stays account-neutral. +- [x] 2.7 Assert a drop after a buffered reasoning prelude (output observed, + zero response events) still penalizes. +- [x] 2.8 Helper unit coverage for `_is_account_neutral_transport_drop`. + +## 3. Validation + +- [x] 3.1 Run the HTTP bridge unit suite and proxy utils suite. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/.openspec.yaml b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/.openspec.yaml new file mode 100644 index 0000000000..4af864176c --- /dev/null +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-14 diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/context.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/context.md new file mode 100644 index 0000000000..cb7818c2f9 --- /dev/null +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/context.md @@ -0,0 +1,29 @@ +# Keep file-pin owner on soft 1011 reconnect + +## Purpose + +Close the HTTP-bridge reconnect hole where a live `input_file.file_id` pin is +treated as skippable prompt-cache locality after upstream close `1011`. + +## Decision + +Honor `file_required_preferred_account` in reconnect owner resolution, and +pass it from submit-on-closed fresh-upstream retry. Do not persist pins +across replicas here. + +## Constraints + +File pins are hard ownership. Soft `1011` skip-same-account stays valid only +when no live file pin (and no other required owner) is present. + +## Failure mode + +If the pin account is excluded or cannot reconnect, fail closed with the +existing required-owner unavailable error. Do not fall back to another +account and forward the `file_id`. + +## Example + +Upload `file_xyz` on account A, then send `/v1/responses` with that +`input_file` on a soft prompt-cache bridge session. Upstream closes `1011` +before the next turn is accepted. Reconnect must keep account A required. diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/design.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/design.md new file mode 100644 index 0000000000..8505b71389 --- /dev/null +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/design.md @@ -0,0 +1,51 @@ +## Context + +`_reconnect_http_bridge_session` promotes `request_state.preferred_account_id` +to a required owner only when the caller sets `require_preferred_account` or +the session is account-neutral. Submit-on-closed recovery calls +`_retry_http_bridge_request_on_fresh_upstream`, which passes +`require_same_account` only for hard keys and never passes +`require_preferred_account`. After upstream `1011`, a soft `prompt_cache` +session therefore sets `skip_same_account`, excludes the file owner, and +allows fallback. The later precreated-recovery path already pins files. + +The existing file-pin requirement already says a live pin MUST override +prompt-cache locality. This change closes the reconnect hole rather than +inventing a new ownership model. + +## Goals / Non-Goals + +**Goals:** + +- Soft `1011` reconnect of a file-pinned request keeps the pin account + required, or fail-closes if that account is excluded or unavailable. +- Movable soft `1011` reconnects without a live file pin still skip the + closed account. + +**Non-Goals:** + +- Durable cross-replica pin persistence (open `#1521`). +- Changing hard-session `1011` keep-owner behavior. +- Changing compact or native WebSocket file routing. + +## Decisions + +- Honor `file_required_preferred_account` inside reconnect owner resolution + so every reconnect caller is covered, not only submit-on-closed. +- Also pass `require_preferred_account` from + `_retry_http_bridge_request_on_fresh_upstream` so that path matches the + already-correct precreated recovery call. +- If the file-required flag is set but `preferred_account_id` is missing, + use the current session account (the session was already on the pin owner). + +**Alternative considered:** only change the one call site. Rejected because +reconnect still ignores `file_required_preferred_account`, so a future +caller can reopen the hole. + +## Risks / Trade-offs + +- [Risk] A file-pinned request can no longer leave a `1011`-closed soft + session's account. → Mitigation: that is the required contract; fail closed + instead of sending the file to another account. +- [Risk] Existing unit tests assert the fresh-upstream retry call shape. + → Mitigation: update the no-file assertion and add a file-pin assertion. diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/proposal.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/proposal.md new file mode 100644 index 0000000000..9cc3266886 --- /dev/null +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/proposal.md @@ -0,0 +1,33 @@ +## Why + +A live `input_file.file_id` pin is hard ownership and must stay on the +uploading account. Soft HTTP-bridge reconnect after upstream `1011` currently +treats that owner as skippable prompt-cache locality, so submit-on-closed +recovery can send the file to another account. + +## What Changes + +- Treat `file_required_preferred_account` as a required reconnect owner, even + when the session key is soft and the close code is `1011`. +- Pass that requirement from submit-on-closed fresh-upstream retry so it + cannot drop the pin. +- Keep `1011` skip-same-account for movable soft sessions that have no live + file pin. + +## Capabilities + +### New Capabilities + +- None. + +### Modified Capabilities + +- `responses-api-compat`: HTTP-bridge reconnect after `1011` must keep a live + file-pin owner required, or fail closed. + +## Impact + +- `app/modules/proxy/_service/http_bridge/mixin.py` reconnect owner resolution. +- `app/modules/proxy/_service/http_bridge/request_submit.py` fresh-upstream retry. +- Unit coverage next to the existing hard-`1011` reconnect tests. +- No API, schema, dashboard, or settings changes. diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..e8084a6eae --- /dev/null +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md @@ -0,0 +1,53 @@ +## ADDED Requirements + +### Requirement: Soft HTTP-bridge 1011 reconnect keeps a live file-pin owner + +A still-unsubmitted HTTP-bridge reconnect MUST keep a live `input_file.file_id` +pin as a required owner after a soft session closes with `1011`. +When an HTTP-bridge session is soft (prompt-cache or request locality) and +upstream closed it with `1011`, a still-unsubmitted request that carries a +live `input_file.file_id` pin MUST keep that pin account as a required +reconnect owner. The proxy MUST NOT exclude that account solely because the +close code was `1011`, and MUST NOT fall back to another account while the +pin is live. If the required pin account is already excluded or cannot be +reconnected, the proxy MUST fail closed with the existing required-owner +unavailable error. A soft `1011` reconnect that has no live file pin and no +other required owner MAY still skip the closed account. + +#### Scenario: Soft 1011 reconnect keeps the file-pin account required + +- **GIVEN** a live in-memory pin `file_xyz -> account_a` +- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz` +- **WHEN** the proxy reconnects that session +- **THEN** account selection MUST treat `account_a` as the required owner +- **AND** it MUST NOT add `account_a` to the excluded-account set solely because of `1011` +- **AND** it MUST NOT enable preferred-account fallback to another account + +#### Scenario: Soft 1011 reconnect without a file pin may skip the closed account + +- **GIVEN** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the still-unsubmitted request has no live file pin and no other required owner +- **WHEN** the proxy reconnects that session +- **THEN** account selection MAY exclude `account_a` and choose another eligible account + +#### Scenario: Soft 1011 file-pin reconnect fails closed when the required owner cannot be selected + +- **GIVEN** a live in-memory pin `file_xyz -> account_a` +- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz` +- **AND** account selection cannot return `account_a` +- **WHEN** the proxy reconnects that session +- **THEN** the proxy MUST fail closed with the existing required-owner unavailable error +- **AND** it MUST NOT replace that envelope with a generic selection failure + +#### Scenario: Soft 1011 file-pin reconnect fails closed when the required owner cannot be connected + +- **GIVEN** a live in-memory pin `file_xyz -> account_a` +- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz` +- **AND** account selection returns `account_a` +- **AND** opening a replacement upstream for `account_a` fails +- **WHEN** the proxy reconnects that session on submit +- **THEN** the client-visible error MUST be the existing required-owner unavailable error +- **AND** it MUST NOT be replaced with a generic `upstream_unavailable` envelope diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md new file mode 100644 index 0000000000..b6cb2a89d2 --- /dev/null +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md @@ -0,0 +1,25 @@ +## 1. Implementation + +- [x] 1.1 Treat `file_required_preferred_account` as a required owner in + `_reconnect_http_bridge_session`. +- [x] 1.2 Pass `require_preferred_account` from + `_retry_http_bridge_request_on_fresh_upstream` when a live file pin is + present. + +## 2. Regression coverage + +- [x] 2.1 Assert soft `1011` reconnect with a live file pin keeps the owner + required and does not exclude it. +- [x] 2.2 Assert soft `1011` reconnect without a file pin may still skip the + closed account. +- [x] 2.3 Update the fresh-upstream retry call-shape assertion for the new + `require_preferred_account` argument. +- [x] 2.4 Assert soft `1011` file-pin reconnect fails closed with the + required-owner envelope when selection cannot return the pin account. +- [x] 2.5 Assert submit-on-closed emits the required-owner envelope when + the pin account is selected but the replacement socket cannot be opened. + +## 3. Validation + +- [x] 3.1 Run the focused HTTP-bridge reconnect unit tests. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/openspec/changes/label-sync-rate-limit-fallback/proposal.md b/openspec/changes/label-sync-rate-limit-fallback/proposal.md deleted file mode 100644 index 0beb2bd7ac..0000000000 --- a/openspec/changes/label-sync-rate-limit-fallback/proposal.md +++ /dev/null @@ -1,24 +0,0 @@ -## Why - -The Codex label sync workflow authenticates with a user PAT (`CODEX_LABEL_SYNC_TOKEN`), whose 5,000/hr REST quota is shared with every other consumer of that user's token (interactive sessions, agents, other automations). During busy review cycles the quota exhausts and every label sync run fails with `API rate limit exceeded (HTTP 403)`, painting spurious CI failures on open PRs for up to an hour — observed repeatedly on 2026-07-13 during the adaptive-windows review cycle (#1266/#1267/#1268). - -## What Changes - -- The sync script detects rate-limit exhaustion on any `gh` call and switches once to a fallback token (`GH_FALLBACK_TOKEN`), retrying the failed call; the workflow provides `github.token` as that fallback, which carries a separate per-repository Actions quota. -- When no distinct fallback is available (or it is also exhausted), behavior is unchanged: the run fails per the read/classification failure contract. - -## Capabilities - -### New Capabilities - -None. - -### Modified Capabilities - -- `github-automation`: label sync gains a runtime rate-limit token fallback on top of the existing configuration-time token preference. - -## Impact - -- Code: `.github/scripts/sync_codex_ok_labels.py`, `.github/workflows/codex-review-labels.yml` -- Tests: `tests/unit/test_sync_codex_ok_labels.py` -- Specs: `openspec/specs/github-automation/spec.md` diff --git a/openspec/changes/label-sync-rate-limit-fallback/specs/github-automation/spec.md b/openspec/changes/label-sync-rate-limit-fallback/specs/github-automation/spec.md deleted file mode 100644 index 4a5d3a38cd..0000000000 --- a/openspec/changes/label-sync-rate-limit-fallback/specs/github-automation/spec.md +++ /dev/null @@ -1,25 +0,0 @@ -## MODIFIED Requirements - -### Requirement: Codex review label sync write-token fallback - -The `Codex review labels` workflow MUST execute the label synchronization script from the trusted default branch and MUST prefer a repository-provided write token before falling back to the default `github.token`. When the active token's API quota is exhausted at runtime, the script MUST switch once to a configured fallback token and retry the failed call instead of failing the run outright. - -#### Scenario: Privileged token is configured - -- **WHEN** the workflow synchronizes Codex review labels -- **THEN** it uses `CODEX_LABEL_SYNC_TOKEN` when present -- **AND** it falls back to `RELEASE_PLEASE_TOKEN` before `github.token` -- **AND** it checks out the default branch with persisted checkout credentials disabled - -#### Scenario: Active token hits its rate limit - -- **GIVEN** the workflow provides `github.token` as `GH_FALLBACK_TOKEN` -- **WHEN** a gh call fails with `API rate limit exceeded` -- **THEN** the script switches to the fallback token once and retries the failed call -- **AND** subsequent calls in the run keep using the fallback token - -#### Scenario: No usable fallback token - -- **WHEN** a gh call fails with `API rate limit exceeded` -- **AND** no fallback token is configured, or it matches the active token, or it is also exhausted -- **THEN** the run fails as a read/classification failure per the existing contract diff --git a/openspec/changes/label-sync-rate-limit-fallback/tasks.md b/openspec/changes/label-sync-rate-limit-fallback/tasks.md deleted file mode 100644 index 6bf9b6c4e7..0000000000 --- a/openspec/changes/label-sync-rate-limit-fallback/tasks.md +++ /dev/null @@ -1,10 +0,0 @@ -## 1. Runtime token fallback - -- [x] 1.1 Detect `API rate limit exceeded` failures in the gh wrapper and switch once to `GH_FALLBACK_TOKEN`, retrying the failed call. -- [x] 1.2 Provide `github.token` as `GH_FALLBACK_TOKEN` in both label-sync workflow jobs. -- [x] 1.3 Unit coverage: fallback activates once and retries; no-op when the fallback is absent or identical; exhausted fallback still fails. - -## 2. Validation - -- [x] 2.1 Run the sync-script unit suite. -- [x] 2.2 Validate with `openspec validate label-sync-rate-limit-fallback --strict`. diff --git a/openspec/changes/normalize-single-account-warmup-summary/.openspec.yaml b/openspec/changes/normalize-single-account-warmup-summary/.openspec.yaml new file mode 100644 index 0000000000..f161d5cc47 --- /dev/null +++ b/openspec/changes/normalize-single-account-warmup-summary/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-16 diff --git a/openspec/changes/normalize-single-account-warmup-summary/design.md b/openspec/changes/normalize-single-account-warmup-summary/design.md new file mode 100644 index 0000000000..227e60a15d --- /dev/null +++ b/openspec/changes/normalize-single-account-warmup-summary/design.md @@ -0,0 +1,33 @@ +## Context + +Warmup already returns a structured result for each submitted account, but `_submit_warmup_request` conditionally re-raises `ProxyAuthError` and `ProxyRateLimitError` when the submission pool contains only one account. The production FastAPI exception handlers then return top-level 401 or 429 envelopes instead of the warmup response model. + +## Goals / Non-Goals + +**Goals:** +- Make ordinary auth and rate-limit failures use the existing failed-account representation for every pool cardinality. +- Preserve request logging, result ordering, bounded scheduling, and response schema. +- Prove the behavior through the production FastAPI route. + +**Non-Goals:** +- Change API-key authentication for calling the warmup endpoint. +- Change account selection, eligibility, concurrency, or upstream routing. +- Change invalid-mode or strict-eligibility `ValueError` handling. +- Change global auth or rate-limit exception envelopes for other endpoints. + +## Decisions + +### Decision: Normalize at the existing per-account submission boundary + +Always convert `ProxyAuthError` and `ProxyRateLimitError` inside `_submit_warmup_request`, where account identity and request-log fields are already available. This removes the cardinality-dependent re-raise without adding route-specific exception handling or changing global handlers. + +Alternative considered: catch these exceptions in `_run_v1_warmup`. This was rejected because the route no longer has the per-account result context and would duplicate service normalization. + +### Decision: Keep the existing response model unchanged + +Use the existing `WarmupFailedAccountData` mapping and error codes (`auth_error` and `rate_limit_exceeded`). No API schema or scheduling changes are required. + +## Risks / Trade-offs + +- **[Risk] A caller may have relied on the undocumented one-account 401/429 behavior** -> **Mitigation:** the cardinality-independent HTTP 200 summary is already the normative contract and existing multi-account behavior. +- **[Risk] Broad exception handling could accidentally change unrelated failures** -> **Mitigation:** remove only the conditional re-raise for the two named exception classes and cover both through FastAPI integration tests. diff --git a/openspec/changes/normalize-single-account-warmup-summary/proposal.md b/openspec/changes/normalize-single-account-warmup-summary/proposal.md new file mode 100644 index 0000000000..2efc75a488 --- /dev/null +++ b/openspec/changes/normalize-single-account-warmup-summary/proposal.md @@ -0,0 +1,23 @@ +## Why + +A warmup request with one eligible account currently returns a top-level 401 or 429 when that account fails authentication or rate limiting, while the same failure in a larger pool is represented in the documented HTTP 200 per-account summary. Pool cardinality should not change the endpoint contract or remove the account-level diagnostic. + +## What Changes + +- Normalize single-account `ProxyAuthError` and `ProxyRateLimitError` failures into the existing `failed` summary entries. +- Preserve the existing summary schema, multi-account behavior, invalid-request handling, account selection, scheduling, and global exception envelopes outside warmup. +- Add production FastAPI integration coverage for both single-account failure classes. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `proxy-warmup`: Clarify that ordinary per-account authentication and rate-limit failures return the structured HTTP 200 summary regardless of target-pool cardinality. + +## Impact + +The change is limited to the warmup service's pre-submit error normalization, its integration coverage, and the `proxy-warmup` contract. It adds no dependencies, settings, routes, or schema fields. diff --git a/openspec/changes/normalize-single-account-warmup-summary/specs/proxy-warmup/spec.md b/openspec/changes/normalize-single-account-warmup-summary/specs/proxy-warmup/spec.md new file mode 100644 index 0000000000..840d855436 --- /dev/null +++ b/openspec/changes/normalize-single-account-warmup-summary/specs/proxy-warmup/spec.md @@ -0,0 +1,26 @@ +## MODIFIED Requirements + +### Requirement: Warmup endpoint is exposed on the v1 proxy surface +The system SHALL expose `POST /v1/warmup` on the same authenticated proxy surface as other `/v1/*` routes. The endpoint SHALL accept a JSON body with `mode` and SHALL return HTTP 200 with a structured JSON summary of submitted, skipped, and failed account warmups for every valid execution. Per-account `ProxyAuthError` and `ProxyRateLimitError` failures SHALL be represented in the `failed` summary regardless of the number of target accounts. + +The system SHALL also expose `POST /v1/warmup/{mode}` on the same authenticated proxy surface. That route SHALL not require a request body and SHALL execute the same warmup behavior as the body-based route for the supplied `mode`. + +#### Scenario: Authenticated warmup request succeeds +- **WHEN** a client calls `POST /v1/warmup` with a valid API key and valid mode +- **THEN** the system returns 200 with a per-account warmup result summary + +#### Scenario: Single-account authentication failure returns summary +- **WHEN** a valid warmup request targets exactly one account and its submission raises `ProxyAuthError` +- **THEN** the system returns 200 with `total_accounts=1` and one `failed` entry with error code `auth_error` + +#### Scenario: Single-account rate-limit failure returns summary +- **WHEN** a valid warmup request targets exactly one account and its submission raises `ProxyRateLimitError` +- **THEN** the system returns 200 with `total_accounts=1` and one `failed` entry with error code `rate_limit_exceeded` + +#### Scenario: Invalid mode is rejected +- **WHEN** a client calls `POST /v1/warmup` with an unsupported mode value +- **THEN** the system returns a 400 invalid request error + +#### Scenario: Path-based warmup request succeeds without a body +- **WHEN** a client calls `POST /v1/warmup/normal` with a valid API key and no request body +- **THEN** the system returns 200 with the same per-account warmup result summary as the body-based route diff --git a/openspec/changes/normalize-single-account-warmup-summary/tasks.md b/openspec/changes/normalize-single-account-warmup-summary/tasks.md new file mode 100644 index 0000000000..8040b63230 --- /dev/null +++ b/openspec/changes/normalize-single-account-warmup-summary/tasks.md @@ -0,0 +1,15 @@ +## 1. Contract + +- [x] 1.1 Define the cardinality-independent warmup failure contract and implementation boundaries. +- [x] 1.2 Sync the clarified requirement to the main `proxy-warmup` specification. + +## 2. Regression and implementation + +- [x] 2.1 Add production FastAPI integration coverage for one-account auth and rate-limit failures and capture the failing baseline. +- [x] 2.2 Remove only the single-account conditional re-raise for `ProxyAuthError` and `ProxyRateLimitError`. + +## 3. Verification + +- [x] 3.1 Capture focused GREEN and adjacent warmup integration results. +- [x] 3.2 Run strict OpenSpec validation, affected lint/type checks, and production FastAPI surface proof. +- [x] 3.3 Review the committed diff independently and address in-scope findings. diff --git a/openspec/changes/omit-unset-chat-tools/context.md b/openspec/changes/omit-unset-chat-tools/context.md new file mode 100644 index 0000000000..2ed08eca0a --- /dev/null +++ b/openspec/changes/omit-unset-chat-tools/context.md @@ -0,0 +1,14 @@ +Responses Lite and `/v1/responses` already drop synthesized top-level +`tools` via `model_fields_set` (issue #1184). Chat conversion was left out +of that change because `_normalize_chat_tools` always constructed a list, +and source-routed chat already popped empty arrays. + +The Codex-backend chat path still goes through `to_responses_request()` +and then `to_payload()`. An omitted chat `tools` field becomes +`"tools": []` on the upstream Responses body. Models that reject an +explicit empty tools param can 400 the same way Lite did. + +Example: `POST /v1/chat/completions` with +`{"model":"gpt-5.2","messages":[{"role":"user","content":"hi"}]}`. +The mapped payload must not contain `tools`. The same request with +`"tools": []` must still forward `[]`. diff --git a/openspec/changes/omit-unset-chat-tools/proposal.md b/openspec/changes/omit-unset-chat-tools/proposal.md new file mode 100644 index 0000000000..7c9a612619 --- /dev/null +++ b/openspec/changes/omit-unset-chat-tools/proposal.md @@ -0,0 +1,27 @@ +# Why + +`/v1/chat/completions` maps onto Responses. `ChatCompletionsRequest.tools` +uses `default_factory=list`, and `to_responses_request()` always writes +`tools` onto the converted request. That marks the field as set, so +`ResponsesRequest.to_payload()` forwards a synthesized `"tools": []` the +client never sent. The Responses and `/v1/responses` omit path already +avoids this (issue #1184). Chat still does not. + +# What Changes + +- Propagate chat `tools` omission through `to_responses_request()`. +- Keep an explicit client-sent `[]` on the mapped Responses payload. +- Leave source-routed chat sanitization as a second, independent omit. + +# Capabilities + +### Modified Capabilities + +- `chat-completions-compat`: omitted chat `tools` must stay omitted on the + mapped Responses wire payload. + +# Impact + +Source-routed chat already drops empty `tools` in +`sanitize_source_chat_payload`. Codex-backend chat inherits the same omit +rule as `/v1/responses`. Explicit tools, including `[]`, stay intact. diff --git a/openspec/changes/omit-unset-chat-tools/specs/chat-completions-compat/spec.md b/openspec/changes/omit-unset-chat-tools/specs/chat-completions-compat/spec.md new file mode 100644 index 0000000000..6d642cb8cb --- /dev/null +++ b/openspec/changes/omit-unset-chat-tools/specs/chat-completions-compat/spec.md @@ -0,0 +1,18 @@ +## ADDED Requirements + +### Requirement: Chat Completions omit unset tools on the mapped Responses payload + +When a `/v1/chat/completions` request omits the top-level `tools` field, the mapped Responses request MUST leave `tools` unset and the forwarded upstream payload MUST NOT include `tools`. An explicit client-sent empty `tools` array MUST still be forwarded as `[]`. + +#### Scenario: Omitted chat tools stay omitted upstream + +- **GIVEN** a Chat Completions request with `messages` and no `tools` field +- **WHEN** the service maps the request to Responses and forwards it +- **THEN** `tools` is absent from the mapped request field set +- **AND** the upstream payload does not include `tools` + +#### Scenario: Explicit empty chat tools stay explicit + +- **GIVEN** a Chat Completions request that sends `"tools": []` +- **WHEN** the service maps the request to Responses +- **THEN** the mapped payload includes `"tools": []` diff --git a/openspec/changes/omit-unset-chat-tools/tasks.md b/openspec/changes/omit-unset-chat-tools/tasks.md new file mode 100644 index 0000000000..21dfdcb40f --- /dev/null +++ b/openspec/changes/omit-unset-chat-tools/tasks.md @@ -0,0 +1,19 @@ +## 1. Implementation + +- [x] 1.1 Omit `tools` from `ChatCompletionsRequest.to_responses_request()` + when the client did not send the field. +- [x] 1.2 Keep an explicit client-sent empty `tools` array on the mapped + Responses request. + +## 2. Regression coverage + +- [x] 2.1 Assert omitted chat `tools` stay out of `model_fields_set` and + `to_payload()`. +- [x] 2.2 Assert explicit `tools: []` still appears on the mapped payload. +- [x] 2.3 Assert `/v1/chat/completions` does not forward synthesized + `tools` to the Codex stream. + +## 3. Validation + +- [x] 3.1 Run the new mapping and chat-completions tests. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/openspec/changes/persist-durable-response-transition-manifests/design.md b/openspec/changes/persist-durable-response-transition-manifests/design.md deleted file mode 100644 index 75af42bfb3..0000000000 --- a/openspec/changes/persist-durable-response-transition-manifests/design.md +++ /dev/null @@ -1,100 +0,0 @@ -## Context - -The current durable checkpoint binds a response anchor to the input item count, -input fingerprint, and pending tool-call map. This proves the history before the -response and identifies calls that require client settlement, but it does not -prove which response-owned items appeared between that stored input and a later -tool output. The verifier therefore accepts only a small set of positional -suffix layouts. - -codex-lb already receives the complete terminal response output. A content-free -manifest of that output is stronger evidence than client-layout inference and -does not require retaining conversation content. - -## Goals - -- Make durable exact-prefix recovery independent of incidental Codex item - ordering around reasoning, commentary, and retry metadata. -- Preserve the existing same-account, one-send, pre-response-event recovery - boundary. -- Keep all persisted evidence free of prompt, response, reasoning, tool - argument, and tool-output content. -- Leave legacy checkpoints and unknown item types fail closed. - -## Non-goals - -- Automatically approve legacy rowless authorities. -- Recover a request whose exact stored prefix, task identity, account, call - ledger, or response manifest cannot be proven. -- Treat client-provided response metadata as a substitute for a gateway-recorded - completion manifest. - -## Manifest - -The `qk_http_bridge_response_transition_manifest_v1` document contains: - -- schema and canonicalization version; -- terminal response status; -- ordered normalized output item descriptors; -- for each descriptor, item kind plus a SHA-256 fingerprint of its canonical - normalized representation; -- bounded identity hashes needed to correlate response-owned items without - persisting raw response text or metadata payloads; and -- the pending tool-call manifest digest. - -The manifest MUST NOT contain raw text, reasoning, encrypted content, tool -arguments, tool output, credentials, request headers, or request bodies. The -existing pending-call map remains authoritative for exact call/output type -matching. - -Known semantics-free transport artifacts use the existing replay -normalization. Unsupported output item types or inconsistent added/done/terminal -events make the manifest unavailable rather than partially trusted. - -## Verification - -For a complete-context resend, the verifier: - -1. matches the stored input count and fingerprint exactly; -2. matches the next ordered response-owned items against the persisted manifest; -3. proves every durable pending call has exactly one self-contained matching - output and no unresolved, orphaned, duplicate, or type-mismatched calls; -4. groups later canonical user/developer retry metadata by bounded response-owned - turn identity instead of relying on positional developer/user counts; -5. requires at least one fresh user turn and rejects assistant/tool output that - is not covered by another recorded manifest; and -6. seals the full request fingerprint, manifest digest, session, task, account, - rejected anchor, and wire fingerprint into the existing immutable recovery - proof. - -The replay remains eligible only before any upstream response event and is sent -without the rejected proxy anchor at most once on the same account. Ambiguous -send outcomes remain UNKNOWN and non-replayable. - -## Persistence And Compatibility - -Durable sessions and recovery markers receive nullable manifest columns. -Completion updates the input checkpoint, pending-call map, and transition -manifest atomically. Marker copy/takeover paths carry the same manifest. - -Rows with a null, malformed, unsupported-version, or digest-inconsistent -manifest retain current legacy behavior. They MUST NOT become automatically -eligible through a broader rowless shape rule. Rollback is refused while a -non-null v1 manifest exists unless the target image declares manifest support. - -## Observability - -Logs expose only schema version, item count, structural item-kind sequence, -manifest digest prefix, and a stable rejection reason. They never expose item -content or raw identities. Metrics distinguish manifest missing, malformed, -prefix mismatch, item mismatch, pending settlement mismatch, retry identity -mismatch, and successful in-place recovery. - -## Testing - -Tests use canonical synthetic items and sanitized golden traces. A model-based -transition suite permutes reasoning/commentary, multiple calls and outputs, -Lite/non-Lite developer placement, repeated retry turns, cancellation, stale -anchors, and duplicate/missing items. Every accepted trace must retain zero -unresolved calls and exactly one physical send; every one-item mutation outside -documented normalization must fail closed. diff --git a/openspec/changes/persist-durable-response-transition-manifests/proposal.md b/openspec/changes/persist-durable-response-transition-manifests/proposal.md deleted file mode 100644 index fca68fde23..0000000000 --- a/openspec/changes/persist-durable-response-transition-manifests/proposal.md +++ /dev/null @@ -1,43 +0,0 @@ -## Why - -The durable HTTP Responses checkpoint records the completed input fingerprint -and pending tool-call map, but not the ordered response output that produced -that pending-call state. After an idle reconnect, official Codex can resend the -exact stored prefix followed by response-owned reasoning, assistant commentary, -the pending call and its output, and one or more later retry turns. The current -verifier cannot bind the reasoning/commentary items to the completed response, -so it falls back to rowless semantic inference. - -Rowless automatic recovery has consequently accumulated separate allowlists for -root retries, settled-tool tails, staged partial responses, Lite developer -ordering, and other incident-specific layouts. Each new official client layout -requires another grammar branch even though codex-lb observed the original -`response.completed` output and could have retained a content-free proof of it. - -## What Changes - -- Build a versioned, content-free response transition manifest from every - eligible `response.completed` output and bind it to the durable checkpoint. -- Persist only ordered canonical item fingerprints and bounded structural - metadata; never persist response text, reasoning, tool arguments, or tool - output. -- Verify a full resend by matching its exact stored prefix and completed-output - manifest, then proving the durable pending calls are settled and later turns - are canonical, task-bound, self-contained, and ledger-clean. -- Permit one same-request, same-account replay without the rejected proxy anchor - when that manifest proof remains exact. -- Keep checkpoints without a valid manifest on the existing fail-closed operator - recovery path. Do not add another rowless sequence allowlist for this change. -- Cover official Codex retry layouts with table-driven and generated transition - tests so ordering variations are validated by identities and state, not by a - growing list of positional patterns. - -## Impact - -- A database migration adds nullable manifest fields to durable HTTP bridge - sessions and recovery markers. Existing rows remain valid legacy checkpoints. -- The response completion and stale-anchor recovery paths gain manifest build, - persistence, matching, and observability. -- Automatic recovery becomes more general for newly completed responses while - remaining exactly-once, account-bound, and fail-closed for missing or - ambiguous evidence. diff --git a/openspec/changes/persist-durable-response-transition-manifests/specs/responses-api-compat/spec.md b/openspec/changes/persist-durable-response-transition-manifests/specs/responses-api-compat/spec.md deleted file mode 100644 index e98ca5aa19..0000000000 --- a/openspec/changes/persist-durable-response-transition-manifests/specs/responses-api-compat/spec.md +++ /dev/null @@ -1,80 +0,0 @@ -## ADDED Requirements - -### Requirement: Durable completions record a content-free response transition manifest - -For every eligible HTTP Responses bridge `response.completed`, the service MUST -atomically persist a versioned content-free manifest of the ordered normalized -response output together with the durable input checkpoint and pending -tool-call manifest. The manifest MUST bind its canonicalization version, -terminal status, ordered item kinds and fingerprints, bounded response-owned -identity hashes, and pending-call digest. It MUST NOT persist prompt text, -response text, reasoning, encrypted content, tool arguments, tool output, -credentials, headers, or request bodies. Unsupported or inconsistent terminal -output MUST produce no manifest rather than a partial manifest. - -#### Scenario: eligible completion records durable proof - -- **WHEN** a bridge response completes with internally consistent output events and terminal output -- **THEN** its durable session and active recovery marker store the same transition manifest atomically with the response anchor and input checkpoint -- **AND** the stored document contains no conversation or tool content - -#### Scenario: unsupported output remains legacy - -- **WHEN** a completion contains an unsupported item type or inconsistent output lifecycle -- **THEN** the response still completes normally -- **AND** its durable transition manifest is null -- **AND** later stale-anchor recovery does not infer a partial manifest - -### Requirement: Manifest-backed full resends recover stale proxy anchors without layout allowlists - -When a complete-context request exactly matches a durable input checkpoint and -then contains the response-owned output recorded by its transition manifest, -the service MUST validate the transition by manifest and call-ledger state -rather than by a fixed positional retry layout. It MUST require every durable -pending call to have exactly one matching self-contained output, group later -canonical user/developer inputs by response-owned task and turn identity, -require at least one fresh user turn, and reject any uncovered assistant/tool -output, unresolved, orphaned, duplicate, type-mismatched, account-scoped, or -identity-conflicting item. - -If upstream rejects the proxy-injected anchor before any response event, a -matching request MAY be replayed exactly once without that anchor on the same -account. The proof MUST bind the session, task, account, rejected anchor, stored -prefix, transition-manifest digest, pending-call manifest, complete request, and -actual wire fingerprint. - -#### Scenario: commentary and pending call settle through the recorded manifest - -- **GIVEN** a durable checkpoint recorded reasoning, assistant commentary, and a pending custom-tool call in its transition manifest -- **AND** a complete resend exactly matches the stored prefix and manifest, supplies the matching tool output, and contains canonical later retry turns -- **WHEN** upstream rejects the proxy-injected anchor before any response event -- **THEN** the service sends the sealed anchor-free request once on the same account -- **AND** the client receives the replacement response stream instead of an administrator-approval loop - -#### Scenario: repeated developer and user items are identity-grouped - -- **GIVEN** official Codex hoists multiple canonical developer messages ahead of their user messages -- **AND** every item has a unique bounded response-owned identity that groups into a task-matching retry turn -- **WHEN** manifest-backed transition verification runs -- **THEN** eligibility is determined from identity and ledger state rather than a fixed developer/user positional count - -#### Scenario: one-item drift fails closed - -- **GIVEN** a complete resend changes, omits, duplicates, or reorders a manifest-bound output item or pending-call settlement -- **WHEN** stale-anchor recovery is evaluated -- **THEN** the service performs no unanchored send -- **AND** returns the existing stable operator-gated recovery result without exposing request content - -### Requirement: Legacy checkpoints do not broaden rowless automatic recovery - -A checkpoint with no supported transition manifest MUST retain the existing -operator-gated recovery behavior whenever exact durable proof cannot be made. -This capability MUST NOT add a new positional rowless allowlist as a substitute -for missing manifest evidence. - -#### Scenario: pre-migration checkpoint requires operator evidence - -- **GIVEN** a durable checkpoint predates transition manifests -- **WHEN** its proxy anchor is rejected and existing exact durable proof is insufficient -- **THEN** automatic manifest recovery is unavailable -- **AND** the service preserves the existing rowless authority and exactly-once operator gate diff --git a/openspec/changes/persist-durable-response-transition-manifests/tasks.md b/openspec/changes/persist-durable-response-transition-manifests/tasks.md deleted file mode 100644 index 3d82e71389..0000000000 --- a/openspec/changes/persist-durable-response-transition-manifests/tasks.md +++ /dev/null @@ -1,21 +0,0 @@ -## 1. Specification And Model - -- [x] 1.1 Define the content-free manifest, verification contract, compatibility floor, and failure semantics. -- [x] 1.2 Add nullable durable session and recovery-marker manifest storage with migration coverage. -- [x] 1.3 Add canonical manifest build/decode/digest helpers with content-exclusion tests. - -## 2. Runtime - -- [x] 2.1 Capture the manifest from eligible terminal response output and persist it atomically with the checkpoint. -- [x] 2.2 Carry the manifest through lookup, takeover, marker, replacement, and completion paths. -- [x] 2.3 Verify exact-prefix resends through the manifest-backed transition state machine. -- [x] 2.4 Seal the manifest digest into request-local and durable automatic recovery proofs. -- [x] 2.5 Keep legacy or malformed manifests on the existing operator-gated path with bounded diagnostics. - -## 3. Verification - -- [x] 3.1 Add unit tests for canonicalization, privacy, matching, mutation rejection, and retry grouping. -- [x] 3.2 Add integration tests for stale-anchor replay, multiple tool loops, cancellation, ambiguity, and legacy fallback. -- [x] 3.3 Add migration upgrade/downgrade and rollback-floor tests. -- [x] 3.4 Run strict OpenSpec validation, formatter, lint, focused unit/integration tests, and the full relevant suite. -- [ ] 3.5 Verify production canary logs show manifest-backed recovery and no new authorization-required loop for the Fourcam trace class. diff --git a/openspec/changes/persist-file-account-pins/design.md b/openspec/changes/persist-file-account-pins/design.md new file mode 100644 index 0000000000..a86c060939 --- /dev/null +++ b/openspec/changes/persist-file-account-pins/design.md @@ -0,0 +1,45 @@ +## Context + +`ProxyService` records upstream-issued file ownership in a process-local dictionary. File finalize and Responses input-file routing already consume ownership through `_pin_file_account`, `_resolve_file_account`, and `_lookup_file_pin`, but another replica cannot observe that state. The application database and `SessionLocal` are already the shared coordination substrate. + +## Goals / Non-Goals + +**Goals:** + +- Make live file ownership visible to every replica. +- Preserve the existing 30-minute expiry and opaque unknown-file compatibility. +- Keep account-owner routing fail-closed and the change behind existing service boundaries. +- Support the repository's PostgreSQL production path and SQLite test/development path. + +**Non-Goals:** + +- Change upload/finalize API payloads, routing precedence, or retry policy. +- Add operator configuration or background cleanup infrastructure. +- Backfill pins that existed only in memory before migration. + +## Decisions + +1. Add a `file_account_pins` table keyed by `file_id`, with an account identifier and absolute UTC expiry. A dedicated repository owns idempotent ownership claims and live lookup. A live claim is immutable across accounts; same-owner claims renew it, while an expired ID can be claimed again. This is smaller and more explicit than overloading sticky-session namespaces. +2. Use one short-lived `SessionLocal` session per pin/read operation. `ProxyService` is process-scoped, so retaining a request-scoped session would be unsafe; the established durable bridge/ring pattern already uses a session factory. +3. Do not cache durable owner decisions in process memory. `_pin_file_account` writes through the repository, while every `_resolve_file_account` call reads the shared database. Multi-file resolution uses one database query so all referenced IDs are classified from the same repository operation. An authenticated inter-replica forwarding value only corroborates the receiver's fresh database result; it cannot replace that read. +4. Evaluate expiry inside each database statement. PostgreSQL claim, reclaim, and live lookup use `clock_timestamp()`, and every successful claim performs an owner-guarded expiry refresh in the same transaction. The refresh gives a full post-wait TTL even when a new-row insert blocked behind an uncommitted unique contender that later rolled back. PostgreSQL cleanup uses the DB-authoritative, stable `statement_timestamp()` cutoff so the expiry index remains usable. SQLite uses its statement-native UTC clock with the same fractional width as stored `DateTime` values and the same guarded post-claim refresh. All expiry decisions therefore stay in the database clock domain without replica-clock skew or exact-expiry ambiguity. +5. Translate persistence failures at the ownership boundary into the stable fail-closed proxy error. Run finalize lookup and post-upstream pin persistence inside the existing file request-log lifecycle, and keep Responses lookup errors inside the existing startup-error lifecycle, so failures neither leak API-key reservations nor record a failed request as successful. +6. Keep exactly one owner for a Responses usage reservation across API startup, direct or compact service settlement, and HTTP-bridge forwarding. Within one replica, the API layer owns cleanup through the durable file-owner lookup and any following preflight outside the service settlement guard. The direct stream service signals when it enters its settlement-guarded `try/finally`; the local HTTP-bridge service signals only after a successful request submit installs the request-state finalizer; and compact service settlement signals after its one cancellation-safe settlement attempt. From those exact boundaries the service finalizer or settlement attempt owns cleanup even if no upstream event has arrived. For an authenticated cross-replica forward carrying the origin reservation, the receiver delays its successful HTTP 200 until its own service has reached one of those settlement-owned boundaries. The 200 response is the receiver's cleanup-handoff acknowledgement. The origin records dispatch only after local payload and header construction and immediately before entering the request transport. A definitive non-200 response leaves cleanup at the origin. If dispatch occurred but no HTTP status can be observed, the origin must not actively release or replay because the receiver may already own settlement; the receiver finalizer or the existing stale-reservation reaper resolves the ambiguity. The reaper releases reservations whose `updated_at` is older than six hours, or whose `created_at` is older than 24 hours even if a heartbeat keeps refreshing `updated_at` (`#1600`). A `DISPATCH_AMBIGUOUS` reservation can therefore hold quota until the 24-hour hard ceiling plus one hourly janitor loop, but a premature origin release cannot discard receiver-recorded usage. A receiver-side owner-revalidation failure or cancellation before dispatch or after a definitive non-200 propagates with origin cleanup intact. An initial client-facing SSE heartbeat or any other frame does not transfer ownership. Cancellation or owner-lookup failure while cleanup remains at the API layer schedules one tracked release, including when a bounded startup probe has handed pending preflight work to the response body; the origin cancels and awaits that pending startup task before releasing. Each cleanup owner makes one cancellation-safe attempt and, if that persistence write fails, schedules one follow-up release instead of abandoning the reservation. A cleanup-database failure must not mask the original `file_owner_unavailable` error or cancellation. + Once compact settlement has transferred ownership, later receiver-side output validation or a `usage_settlement_failed` raise cannot safely turn the response back into a non-200 rejection. The receiver therefore preserves HTTP 200 and emits a terminal `response.failed` SSE event with the stable error code; the origin keeps the handoff and does not release or replay. Cancellation after owner-forward transport begins, including during response-header wait, is dispatch-ambiguous. Definitive connector failures stay not-dispatched. + +## Risks / Trade-offs + +- [Every hard ownership decision adds a database read] → use one indexed lookup for one file and one batched indexed lookup for multi-file requests; correctness across replicas takes precedence over process-local locality. +- [A database outage can make file create/finalize unavailable] → fail closed because silently selecting another account can disclose or corrupt account-scoped operations. +- [An idle installation can retain expired rows until the next upload] → the rows are inert and every new claim performs indexed opportunistic cleanup. +- [Migration overlaps another branch] → base the revision on the current single head and report any later head conflict rather than editing another track. + +## Migration Plan + +Upgrade creates the empty ownership table and index; new uploads populate it immediately. Downgrade drops only the new table. Existing in-memory pins cannot be backfilled. + +The behavior change is not safe under an ordinary mixed-version rolling rollout: a legacy replica cannot read pins written by a new replica, and a new replica cannot read a legacy replica's process-local pins. Operators must migrate the database first, stop legacy replicas from accepting new file registrations, drain the legacy upload/finalize window for up to the 30-minute pin TTL (or explicitly accept retrying those in-flight uploads), and then cut all file-serving replicas over without mixed-version file traffic. Deployment automation is intentionally outside this code change. + +## Open Questions + +None. diff --git a/openspec/changes/persist-file-account-pins/proposal.md b/openspec/changes/persist-file-account-pins/proposal.md new file mode 100644 index 0000000000..98e8c3d7da --- /dev/null +++ b/openspec/changes/persist-file-account-pins/proposal.md @@ -0,0 +1,27 @@ +## Why + +File ownership pins currently live only in a replica-local dictionary, so a file finalize or Responses request handled by another replica can select an account that does not own the upstream file. The confirmed P1 bug must be fixed by making the existing ownership boundary durable and shared. + +## What Changes + +- Persist live `file_id -> account_id` ownership pins in the application database with the existing 30-minute lifetime. +- Resolve file ownership through the durable store so finalize and input-file routing remain account-bound across replicas. +- Keep the existing `_pin_file_account` and `_resolve_file_account` service boundaries and fail closed when durable ownership cannot be established safely. +- Add migration and targeted repository/service regression coverage. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: Replace process-local best-effort file ownership with durable, replica-shared ownership for file finalize and Responses input-file routing. +- `files-upload-protocol`: Require file finalization to resolve its durable owner through the shared database and fail closed when that lookup is unavailable. +- `sticky-session-operations`: Require a remote HTTP-bridge owner to corroborate signed file-owner metadata with its own fresh durable lookup instead of trusting origin process memory. +- `sticky-session-operations`: Require every receiving HTTP-bridge transport, including terminal compaction, to revalidate forwarded file ownership against the durable database. + +## Impact + +The change affects the proxy file operations mixin, a small proxy persistence repository, the database model and Alembic graph, and focused proxy/database tests. It adds no dependency or public API surface. diff --git a/openspec/changes/persist-file-account-pins/specs/files-upload-protocol/spec.md b/openspec/changes/persist-file-account-pins/specs/files-upload-protocol/spec.md new file mode 100644 index 0000000000..500215080c --- /dev/null +++ b/openspec/changes/persist-file-account-pins/specs/files-upload-protocol/spec.md @@ -0,0 +1,25 @@ +## ADDED Requirements + +### Requirement: File finalize uses durable replica-shared ownership + +When `POST /backend-api/files/{file_id}/uploaded` references a live durable file pin, the service MUST resolve that pin from the shared database and route finalization only through the owning account. The owner decision MUST NOT use a process-local cache. Expiry, reclaim, and cleanup MUST use database-authoritative time. If durable owner resolution fails, the service MUST fail closed before selecting or invoking an unpinned fallback account. + +#### Scenario: another replica finalizes through the durable owner + +- **GIVEN** one replica registered `file_xyz` through `account_a` +- **WHEN** another replica handles `POST /backend-api/files/file_xyz/uploaded` +- **THEN** it MUST resolve the shared durable pin +- **AND** it MUST finalize only through `account_a` + +#### Scenario: finalize owner lookup failure does not fall back + +- **GIVEN** `file_xyz` requires a durable owner decision +- **WHEN** the shared database lookup fails +- **THEN** finalization MUST fail before any unpinned account selection or upstream invocation + +#### Scenario: an expired identifier can be reclaimed using database time + +- **GIVEN** the durable pin for `file_xyz` has expired according to the database clock +- **WHEN** a later upload claims `file_xyz` through `account_b` +- **THEN** the durable owner MUST become `account_b` +- **AND** every replica's next finalize decision MUST resolve `account_b` diff --git a/openspec/changes/persist-file-account-pins/specs/responses-api-compat/spec.md b/openspec/changes/persist-file-account-pins/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..af8667cd6e --- /dev/null +++ b/openspec/changes/persist-file-account-pins/specs/responses-api-compat/spec.md @@ -0,0 +1,161 @@ +## MODIFIED Requirements + +### Requirement: Responses requests with input_file.file_id route to the upload's account + +A `/v1/responses`, `/backend-api/codex/responses`, or `/responses/compact` request that references an `{type: "input_file", file_id}` content item SHALL be routed to the upstream account that registered the file via `POST /backend-api/files` when a durable, unexpired pin for that `file_id` exists. The pin MUST be visible to every replica that shares the application database. A live file pin is hard ownership evidence: it MUST override prompt-cache or bare process-session locality and MUST agree with independently resolved turn-state, previous-response, bridge, or other hard ownership. + +When multiple `file_id`s are referenced, all live pins MUST resolve to the same account. If at least one ID has a live pin and another ID has no live pin, the request MUST fail with `file_owner_unavailable`; if live pins resolve to different accounts, it MUST fail with `continuity_owner_conflict`. If none of the referenced IDs has a live pin, the proxy MUST preserve compatibility with files registered directly upstream or before durable ownership was observed by forwarding the opaque IDs verbatim under ordinary unpinned routing. + +A live durable pin MUST NOT be reassigned to another account. Repeating the claim for the same account MUST be idempotent and MAY renew its expiry; an expired identifier MAY be claimed by a later upload. + +Every hard file-owner decision MUST read the shared database and MUST NOT rely on a process-local owner cache. Authenticated inter-replica forwarding metadata MAY corroborate the freshly resolved durable owner but MUST NOT replace the receiver's database read. A missing or conflicting receiver-side durable owner MUST fail closed before account selection or upstream invocation. Pin expiry, reclaim, and cleanup MUST use database-authoritative statement time rather than a replica's application clock. + +For a streaming Responses request whose durable file-owner lookup runs in the stream service, any API-key usage reservation acquired before that lookup MUST have exactly one cleanup owner if resolution fails or the request is cancelled. Within one replica, the API layer MUST own cleanup until the direct stream service enters its settlement-guarded `try/finally` or the local HTTP-bridge service successfully submits the request and installs its request-state finalizer. The service finalizer MUST own cleanup after that explicit boundary so those layers cannot both release the reservation. Merely completing the durable lookup MUST NOT transfer cleanup before a service finalizer is active, and an initial SSE heartbeat MUST NOT transfer ownership to the client. + +When an authenticated HTTP-bridge origin forwards that reservation to another replica, the receiver MUST delay its successful HTTP 200 response until its service finalizer is active. That 200 response MUST be the cleanup-handoff acknowledgement that transfers ownership from the origin to the receiver. The origin MUST distinguish a request that has not been dispatched, a dispatch with no observed response status, a successful HTTP 200 acknowledgement, and a definitive non-200 rejection. Before dispatch or after a definitive non-200, receiver-side owner-revalidation failure or cancellation MUST propagate with cleanup remaining at the origin. After dispatch when no response status can be observed, the origin MUST NOT actively release or replay the reservation because the receiver may already own settlement; receiver settlement or bounded stale-reservation cleanup MUST resolve that ambiguity. After the acknowledgement, the receiver service finalizer MUST remain authoritative even if no upstream event has arrived. If a bounded startup probe hands pending preflight work to the response body and the body closes first, the active owner MUST cancel and await that work before scheduling one cancellation-safe release attempt. If that persistence write fails, the same cleanup owner MUST schedule one follow-up release attempt instead of abandoning the reservation. An SSE heartbeat or another frame MUST NOT transfer cleanup ownership. Compact service settlement MUST likewise suppress a second API-layer release after its single settlement attempt. Once a forwarded compact service has made that settlement attempt, including when both the primary finalize and the fallback release fail, a later receiver-side output validation failure or a `usage_settlement_failed` error MUST preserve HTTP 200 as the cleanup-handoff acknowledgement and surface a terminal `response.failed` event with the stable error code; it MUST NOT become a non-200 rejection that permits origin release or replay. A client disconnect after the initial SSE heartbeat MUST close the service stream even when the startup probe already completed. A cleanup-store failure MUST NOT replace a stable owner-resolution error. A cleanup-store failure MUST NOT mask the original stable owner error or cancellation. Owner-lookup failure or cancellation MUST NOT trigger account failover or another upstream attempt. + +#### Scenario: file_id pin drives routing for an input_file response + +- **GIVEN** a `POST /backend-api/files` registered `file_xyz` through `account_a` on one replica +- **WHEN** a `/v1/responses` request references `{"type": "input_file", "file_id": "file_xyz"}` on another replica +- **THEN** the proxy MUST route the request to `account_a` + +#### Scenario: file_id pin overrides prompt-cache locality + +- **GIVEN** a pinned `file_xyz -> account_a` +- **WHEN** a `/v1/responses` request references `file_xyz` AND sets an explicit `prompt_cache_key` +- **THEN** the proxy MUST route to `account_a` and MUST NOT send the account-scoped file to the prompt-cache account + +#### Scenario: opaque file_id without a live pin remains compatible + +- **GIVEN** a request references a `file_id` registered directly upstream or before the system durably observed its upload +- **AND** no referenced file has a live durable pin +- **WHEN** the request is routed +- **THEN** the proxy MUST forward the `file_id` verbatim under ordinary unpinned routing +- **AND** it MUST NOT reject the request solely because owner metadata is absent + +#### Scenario: file finalize resolves ownership across replicas + +- **GIVEN** one replica registered `file_xyz` through `account_a` +- **WHEN** another replica handles `POST /backend-api/files/file_xyz/uploaded` +- **THEN** the proxy MUST finalize the file through `account_a` +- **AND** it MUST NOT fall back to a different eligible account + +#### Scenario: concurrent live ownership claims do not overwrite + +- **GIVEN** `file_xyz` has a live durable pin to `account_a` +- **WHEN** another replica attempts to pin `file_xyz` to `account_b` +- **THEN** the claim MUST fail with `continuity_owner_conflict` +- **AND** subsequent routing MUST still resolve `file_xyz` to `account_a` + +#### Scenario: a replica observes an expired pin reclaimed by another replica + +- **GIVEN** a replica previously resolved `file_xyz` to `account_a` +- **AND** the durable pin expires and another replica claims `file_xyz` for `account_b` +- **WHEN** the first replica resolves `file_xyz` again +- **THEN** it MUST read the durable owner and return `account_b` +- **AND** it MUST NOT return `account_a` from process-local state + +#### Scenario: durable owner lookup failure fails closed + +- **GIVEN** a request references a file whose owner decision requires the shared database +- **WHEN** the durable owner lookup fails +- **THEN** the request MUST fail before selecting or invoking an unpinned fallback account + +#### Scenario: cancellation during owner lookup releases admission state + +- **GIVEN** a request has acquired an API-key usage reservation before durable file-owner resolution completes +- **WHEN** the request is cancelled while the owner lookup is pending +- **THEN** exactly one cleanup owner MUST attempt to release or settle the reservation +- **AND** no account selection, upstream invocation, retry, or failover may occur + +#### Scenario: delayed owner failure after stream handoff releases admission state + +- **GIVEN** the streaming startup probe expires while durable file-owner resolution is still pending +- **WHEN** the lookup later fails or the response body is closed +- **THEN** the origin API MUST cancel and await any still-pending lookup +- **AND** the origin API MUST make exactly one release attempt +- **AND** a lookup failure MUST be represented by the stable `file_owner_unavailable` error + +#### Scenario: failed reservation release is retried + +- **GIVEN** a startup or disconnect cleanup owns an API-key reservation +- **WHEN** the first persistence release fails +- **THEN** the cleanup owner MUST schedule one follow-up release attempt +- **AND** it MUST NOT leave the reservation reserved with no later cleanup path + +#### Scenario: forwarded owner metadata is revalidated against durable ownership + +- **GIVEN** a replica receives authenticated forwarding metadata that identifies `account_a` as a referenced file's owner +- **WHEN** the receiver's fresh durable lookup has no live owner or identifies a different owner +- **THEN** the receiver MUST fail closed +- **AND** it MUST NOT route using the forwarded value alone +- **AND** it MUST propagate the preflight failure to the origin without releasing the origin reservation +- **AND** the originating request path MUST remain the sole cleanup owner because no successful handoff acknowledgement was sent + +#### Scenario: forwarded stream acknowledges cleanup ownership before HTTP 200 + +- **GIVEN** the origin forwards a file-pinned streaming request and its API-key reservation to the authenticated owner replica +- **WHEN** the receiver completes durable owner revalidation and installs its service settlement finalizer +- **THEN** the receiver MAY return HTTP 200 as the cleanup-handoff acknowledgement +- **AND** the origin MUST stop releasing the reservation after receiving that acknowledgement +- **AND** cancellation before the first upstream event MUST invoke only the receiver's service finalizer + +#### Scenario: ambiguous owner dispatch defers active origin cleanup + +- **GIVEN** the origin has begun dispatching a signed forwarded request carrying its reservation +- **WHEN** the transport fails before the origin can observe an HTTP status +- **THEN** the origin MUST NOT actively release or replay the reservation +- **AND** receiver settlement or stale-reservation cleanup MUST remain the only recovery paths + +#### Scenario: definitive owner rejection retains origin cleanup + +- **GIVEN** the origin dispatches a signed forwarded request carrying its reservation +- **WHEN** the receiver returns a non-200 response without acknowledging cleanup handoff +- **THEN** the origin MUST make exactly one cancellation-safe release attempt +- **AND** the receiver MUST NOT settle the origin reservation + +#### Scenario: owner non-200 remains a rejection after body-read failure + +- **GIVEN** the origin has observed a non-200 owner-forward status +- **WHEN** reading the rejection body then fails +- **THEN** the origin MUST treat the outcome as a definitive rejection +- **AND** it MUST NOT reclassify the dispatch as ambiguous + +#### Scenario: compact service settlement is not released twice + +- **GIVEN** terminal or direct compaction receives an API-key usage reservation +- **WHEN** the compact service makes its single settlement or release attempt +- **THEN** the API layer MUST NOT issue another release for that reservation +- **AND** a pre-service failure MUST still leave exactly one release attempt at the API layer + +#### Scenario: malformed compact output after settlement preserves handoff + +- **GIVEN** a forwarded terminal compact request whose receiver service has made its single settlement attempt +- **WHEN** the settled response lacks a valid compaction output item +- **THEN** the receiver MUST return HTTP 200 as the cleanup-handoff acknowledgement +- **AND** it MUST emit a terminal `response.failed` event +- **AND** the origin MUST NOT release or replay the reservation + +#### Scenario: compact settlement failure after fallback preserves handoff + +- **GIVEN** a forwarded terminal compact request whose receiver service has made its single settlement attempt +- **WHEN** usage settlement fails after a successful fallback release +- **THEN** the receiver MUST return HTTP 200 as the cleanup-handoff acknowledgement +- **AND** it MUST emit a terminal `response.failed` event with code `usage_settlement_failed` +- **AND** the origin MUST NOT release or replay the reservation + +#### Scenario: compact settlement attempt preserves handoff when both writes fail + +- **GIVEN** a forwarded terminal compact request whose receiver service attempts settlement +- **WHEN** both reservation finalization and the fallback release fail +- **THEN** the receiver MUST still return HTTP 200 as the cleanup-handoff acknowledgement +- **AND** it MUST emit a terminal `response.failed` event with code `usage_settlement_failed` +- **AND** the origin MUST NOT release or replay the reservation + +#### Scenario: completed startup probe still closes the service stream + +- **GIVEN** the streaming startup probe already obtained the first service event +- **WHEN** the client disconnects after the initial SSE heartbeat +- **THEN** the origin MUST close the service stream +- **AND** reservation cleanup MUST still run if ownership has not transferred diff --git a/openspec/changes/persist-file-account-pins/specs/sticky-session-operations/spec.md b/openspec/changes/persist-file-account-pins/specs/sticky-session-operations/spec.md new file mode 100644 index 0000000000..9d0ce463f8 --- /dev/null +++ b/openspec/changes/persist-file-account-pins/specs/sticky-session-operations/spec.md @@ -0,0 +1,145 @@ +## MODIFIED Requirements + +### Requirement: Hard continuity remains owner-bound and bounded + +Requests that depend on `previous_response_id`, hard turn-state, nonblank `conversation`, account-scoped `input_file.file_id` pins, live or durable bridge ownership, replay/reattach state, or another required owner continuity source MUST NOT silently reroute to an account that cannot preserve continuity. A resolved required owner MUST override bare process-session locality and MUST be selected without consulting or rewriting that soft mapping. A `previous_response_id` is a stored-object continuation reference and remains owner-bound even when the same request also carries a session header, `prompt_cache_key`, or another soft locality key. If independently resolved hard sources identify different accounts, if live durable referenced-file pins identify different accounts, or if a request has partial live durable file-pin coverage, the service MUST fail closed before upstream dispatch. A request for which no referenced file has a live durable pin MUST preserve opaque `file_id` compatibility and proceed without inventing ownership evidence. If the owner account/session is unavailable or saturated, the service MUST fail closed with an explicit retryable continuity/local overload reason instead of flooding the owner queue indefinitely. + +Every HTTP, compact, direct WebSocket, and HTTP-bridge transport MUST resolve explicit turn state against both live and durable bridge aliases. Live, durable, previous-response, file, and explicit turn-state evidence MUST be compared independently; source ordering MUST NOT choose the first match when distinct sessions or accounts resolve. A reused direct WebSocket MUST repeat nonblank `conversation` ownership validation for each response-create frame because the existing socket account proves only the current route. Single-account routing MUST constrain effective routing without narrowing the ownership-candidate pool used by that validation. + +When an HTTP-bridge owner is on another replica, the origin MUST forward its resolved durable file owner in authenticated full-context metadata. The receiving owner MUST perform its own fresh shared-database lookup and MUST require that durable result to match the forwarded owner. A missing or conflicting receiver-side durable owner MUST fail closed before account selection or upstream invocation. A retired direct WebSocket's upstream turn-state token MUST NOT be sent to a different account selected for a later movable bare-session request. + +A nonblank `conversation` without a dedicated resolved owner MUST proceed only when an explicit hard Codex mapping proves ownership or exactly one account remains in the model/API-key/security-scoped selection pool before transient additional-quota availability, retry exclusions, runtime health, budget, or account-cap filtering. A temporarily quota-filtered, excluded, unhealthy, or capped candidate MUST remain part of this ambiguity check because it may be the actual owner. A bare process-session mapping MUST NOT prove conversation ownership. + +#### Scenario: Previous-response owner queue is saturated + +- **WHEN** a `/v1/responses` follow-up requires a previous-response owner +- **AND** the owner session queue or account cap is saturated +- **THEN** the service fails closed with `hard_affinity_saturated`, `previous_response_owner_unavailable`, or the applicable stable `account_stream_cap` / `account_response_create_cap` code +- **AND** it does not route to an unrelated account that lacks continuity state + +#### Scenario: File-pinned request owner is capped + +- **WHEN** a `/v1/responses` request references an `input_file.file_id` pinned to an owner account +- **AND** the owner account is at its account stream or response-create cap +- **THEN** the service returns a local account-cap overload for the owner +- **AND** it does not route the file reference to another account + +#### Scenario: File-pinned request owner overrides process-session locality + +- **GIVEN** a request carries a bare process-session header mapped to account A +- **AND** its `input_file.file_id` is durably pinned to account B +- **WHEN** the request is routed +- **THEN** account B is treated as the required owner +- **AND** the process-session mapping is neither consulted as an owner nor rewritten + +#### Scenario: Conflicting hard owners fail closed + +- **GIVEN** a turn state, previous response, bridge, or input file resolves to account A +- **AND** another hard source on the same request resolves to account B +- **WHEN** the request is routed +- **THEN** the service fails with `continuity_owner_conflict` before upstream dispatch +- **AND** source ordering does not choose either owner + +#### Scenario: Partial or cross-account file pins fail closed + +- **GIVEN** a request references multiple account-scoped input files +- **AND** at least one file has a live durable owner pin +- **AND** another file has no live durable owner pin or the live pins resolve to different accounts +- **WHEN** the request is routed +- **THEN** the service fails with `file_owner_unavailable` or `continuity_owner_conflict` +- **AND** it does not route the files using a soft affinity account + +#### Scenario: Opaque file IDs with no live durable pins preserve compatibility + +- **GIVEN** a request references one or more `input_file.file_id` values +- **AND** none of those IDs has a live durable owner pin +- **WHEN** the request is routed +- **THEN** the service forwards the opaque file references under ordinary unpinned routing +- **AND** it does not invent a hard owner or fail solely because durable pin metadata is absent + +#### Scenario: Ambiguous conversation fails closed + +- **GIVEN** a request carries nonblank `conversation` continuity and only bare process-session affinity +- **AND** more than one account is eligible +- **WHEN** no dedicated or hard-mapping owner can be resolved +- **THEN** the request fails with a stable owner-unavailable error before upstream dispatch + +#### Scenario: Account-cap pressure does not manufacture a conversation owner + +- **GIVEN** two accounts remain in the model/API-key/security-scoped selection pool +- **AND** one account is temporarily at its local account cap +- **WHEN** a request carries nonblank `conversation` continuity without a dedicated or hard-mapping owner +- **THEN** the request still fails with a stable owner-unavailable error +- **AND** the uncapped account is not treated as the unique owner + +#### Scenario: Retry or additional-quota filtering does not manufacture a conversation owner + +- **GIVEN** two accounts remain in the model/API-key/security-scoped selection pool +- **AND** retry exclusion or transient additional-quota availability removes one from the effective routing pool +- **WHEN** a request carries nonblank `conversation` continuity without a dedicated or hard-mapping owner +- **THEN** the request still fails with a stable owner-unavailable error +- **AND** the remaining effective account is not treated as the unique owner + +#### Scenario: Account status does not manufacture a conversation owner + +- **GIVEN** two accounts are in the model/API-key/security ownership pool +- **AND** one account is paused, requires reauthentication, deactivated, or otherwise unavailable for routing +- **WHEN** a request carries nonblank `conversation` continuity without a dedicated or hard-mapping owner +- **THEN** the request still fails with a stable owner-unavailable error +- **AND** the active account is not treated as the unique owner + +#### Scenario: Preferred file owner does not manufacture a conversation owner + +- **GIVEN** a request carries nonblank `conversation` continuity and a file durably pinned to account B +- **AND** another account remains in the model/API-key/security ownership pool +- **WHEN** no dedicated conversation owner can be resolved +- **THEN** file ownership does not narrow the conversation ambiguity check to account B +- **AND** the request fails closed before upstream dispatch + +#### Scenario: Bridge turn state is owner-bound across transports + +- **GIVEN** an HTTP bridge registered a turn-state alias for account A +- **WHEN** the alias is reused through compact, plain HTTP streaming, or direct WebSocket transport +- **THEN** each transport treats account A as the required owner +- **AND** it does not fall back to unrelated sticky affinity + +#### Scenario: Independent bridge aliases conflict + +- **GIVEN** a live or durable turn-state alias resolves to one bridge session +- **AND** a previous-response alias on the same request resolves to a distinct session or account +- **WHEN** the request is routed +- **THEN** the service fails with `continuity_owner_conflict` +- **AND** alias lookup order does not select either session + +#### Scenario: Reused WebSocket revalidates conversation ownership + +- **GIVEN** a direct upstream WebSocket is already open on account A +- **AND** a later response-create frame carries nonblank `conversation` +- **WHEN** more than one account remains in the ownership-candidate pool +- **THEN** the later frame fails with a stable owner-unavailable error before upstream send +- **AND** the existing socket account is not treated as ownership proof + +#### Scenario: Single-account routing does not manufacture conversation ownership + +- **GIVEN** single-account routing selects account A +- **AND** multiple accounts remain in the model/API-key/security ownership pool +- **WHEN** a request carries nonblank `conversation` without dedicated owner evidence +- **THEN** the request remains ambiguous and fails closed +- **AND** only the effective routing states are constrained to account A + +#### Scenario: Remote bridge owner revalidates forwarded file ownership + +- **GIVEN** origin replica A durably resolves an input file to account A +- **AND** the request's HTTP bridge owner runs on replica B +- **WHEN** replica A forwards the request to replica B with authenticated file-owner metadata +- **THEN** replica B MUST freshly resolve the shared durable pin +- **AND** it MUST accept the forwarded owner only when both owner values match +- **AND** a missing, conflicting, tampered, or legacy-unbound proof MUST be rejected before upstream invocation + +#### Scenario: Retired WebSocket turn state does not cross accounts + +- **GIVEN** a closed upstream WebSocket on account A supplied an account-scoped turn-state token +- **AND** a later movable bare-session frame selects account B +- **WHEN** the proxy opens the replacement WebSocket +- **THEN** it removes account A's stale turn-state token before connect +- **AND** account B never receives that token diff --git a/openspec/changes/persist-file-account-pins/tasks.md b/openspec/changes/persist-file-account-pins/tasks.md new file mode 100644 index 0000000000..0ae0b2f25d --- /dev/null +++ b/openspec/changes/persist-file-account-pins/tasks.md @@ -0,0 +1,22 @@ +## 1. Durable ownership storage + +- [x] 1.1 Add the file-account pin ORM model and forward Alembic migration on the current head. +- [x] 1.2 Implement a focused repository for durable upsert and unexpired owner lookup. + +## 2. Proxy integration + +- [x] 2.1 Wire the repository through the existing file pin/resolve boundaries. +- [x] 2.2 Make multi-file ownership resolution use the durable lookup boundary and preserve fail-closed conflict behavior. + +## 3. Verification + +- [x] 3.1 Add targeted repository and cross-replica service regression tests, including expiry and multi-file behavior. +- [x] 3.2 Run focused tests, migration checks, OpenSpec validation, and inspect the final diff/status. + +## 4. DB-authoritative ownership repair + +- [x] 4.1 Remove process-local caching from hard file-owner decisions and batch multi-file resolution through the repository. +- [x] 4.2 Use database-authoritative time for claim expiry, reclaim, live lookup, and opportunistic cleanup. +- [x] 4.3 Add the file-finalize ownership contract to `files-upload-protocol` and replace the stale process-local forwarding contract in `sticky-session-operations`. +- [x] 4.4 Add hermetic race and fail-closed regression coverage, then rerun focused validation. +- [x] 4.5 Make compact settlement and ambiguous owner-forward dispatch single-owner, and add stream/collect/non-200/lost-status regression coverage. diff --git a/openspec/changes/preserve-dashboard-cancelled-count/design.md b/openspec/changes/preserve-dashboard-cancelled-count/design.md new file mode 100644 index 0000000000..678cf611d8 --- /dev/null +++ b/openspec/changes/preserve-dashboard-cancelled-count/design.md @@ -0,0 +1,44 @@ +## Context + +The backend overview response already carries a nullable `cancelledCount`. +`DashboardOverviewSchema` is the frontend trust boundary, and Zod strips +unknown object keys there. The omission is therefore isolated to the typed +consumer contract; no calculation or transport change is required. + +## Goals / Non-Goals + +**Goals:** + +- Keep the frontend overview contract aligned with the backend response. +- Lock the documented requests/error/cancelled breakdown with a focused test. + +**Non-Goals:** + +- Change cancellation classification or aggregation. +- Add a new dashboard card or navigation surface. +- Change backward compatibility for payloads that omit the field. + +## Decisions + +- Declare `cancelledCount` as nullable and optional, matching the additive + backend response and preserving compatibility with older servers. +- Test the field through `DashboardOverviewSchema.parse`, the actual API + boundary, rather than testing the nested schema in isolation. + +Alternative considered: configure the metrics object with `.passthrough()`. +That would weaken the trust boundary for every unknown metric, so the explicit +field is the smaller and safer change. + +## Risks / Trade-offs + +- [Risk] Frontend and backend optionality drift → Mirror the existing additive + metric pattern and cover a payload that includes the field. + +## Migration Plan + +Ship as an additive frontend contract change. Rollback is removal of the field; +backend responses remain compatible in either direction. + +## Open Questions + +None. diff --git a/openspec/changes/preserve-dashboard-cancelled-count/proposal.md b/openspec/changes/preserve-dashboard-cancelled-count/proposal.md new file mode 100644 index 0000000000..32a6c5939f --- /dev/null +++ b/openspec/changes/preserve-dashboard-cancelled-count/proposal.md @@ -0,0 +1,28 @@ +## Why + +The dashboard overview API emits `cancelledCount`, but the frontend Zod +boundary omits that field and silently strips it from otherwise valid metrics +payloads. Operators therefore cannot distinguish a window with no +cancellations from one whose cancellation count was discarded client-side. + +## What Changes + +- Preserve `cancelledCount` when the dashboard parses overview metrics. +- Add a frontend contract regression covering the backend's documented + requests/error/cancelled breakdown. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `usage-error-metrics`: The dashboard frontend preserves the overview + cancellation count emitted by the API. + +## Impact + +Frontend dashboard schema and its focused tests only. No backend, database, or +navigation changes. diff --git a/openspec/changes/preserve-dashboard-cancelled-count/specs/usage-error-metrics/spec.md b/openspec/changes/preserve-dashboard-cancelled-count/specs/usage-error-metrics/spec.md new file mode 100644 index 0000000000..46b338d1f8 --- /dev/null +++ b/openspec/changes/preserve-dashboard-cancelled-count/specs/usage-error-metrics/spec.md @@ -0,0 +1,20 @@ +## MODIFIED Requirements + +### Requirement: Cancelled counts surface alongside error counts + +Metric surfaces that expose an error count MUST also expose the window's +cancelled count as an additive field: the dashboard overview metrics +(`cancelledCount`), the usage summary metrics (`cancelled7d`), the reports +daily rows (`cancelled_count`) and summary (`total_cancelled`), and the fleet +pressure metrics (`cancelledCount`). The dashboard overview cancelled total +MUST be sourced from the demand quarter rollup (status grain) for the folded +segment plus the raw tail, so it stays accurate across history already folded +without the hourly `cancelled_count` measure. The dashboard frontend MUST +preserve `cancelledCount` when parsing the overview response. + +#### Scenario: Dashboard overview preserves the status breakdown + +- **GIVEN** the dashboard overview API returns `requests=4`, `errorCount=1`, + and `cancelledCount=2` +- **WHEN** the frontend parses the overview response +- **THEN** the parsed metrics expose all three values unchanged diff --git a/openspec/changes/preserve-dashboard-cancelled-count/tasks.md b/openspec/changes/preserve-dashboard-cancelled-count/tasks.md new file mode 100644 index 0000000000..a6b8e110cb --- /dev/null +++ b/openspec/changes/preserve-dashboard-cancelled-count/tasks.md @@ -0,0 +1,11 @@ +## 1. Frontend contract + +- [x] 1.1 Add a failing dashboard schema test that requires `cancelledCount` + to survive parsing +- [x] 1.2 Add `cancelledCount` to the dashboard metrics Zod schema + +## 2. Validation + +- [x] 2.1 Run the focused dashboard schema tests +- [x] 2.2 Run frontend typecheck and build +- [x] 2.3 Validate OpenSpec diff --git a/openspec/changes/recognize-canonical-invalid-previous-response-anchor/proposal.md b/openspec/changes/recognize-canonical-invalid-previous-response-anchor/proposal.md deleted file mode 100644 index f1222dc204..0000000000 --- a/openspec/changes/recognize-canonical-invalid-previous-response-anchor/proposal.md +++ /dev/null @@ -1,19 +0,0 @@ -# Recognize canonical invalid previous-response anchors - -## Why - -Production upstream now emits a stale Responses anchor as `code=invalid_request_error` with the exact message `Invalid previous_response_id.` and no `param`. The existing classifier only recognizes `previous_response_not_found` or the older `param=previous_response_id` plus “not found” shape. As a result, the HTTP bridge forwards a raw 400, then treats the socket close as `stream_incomplete` and opens the per-session retry circuit instead of entering the existing proof-gated stale-anchor recovery path. - -## What Changes - -- Recognize only the exact canonical invalid-`previous_response_id` message when `param` is absent or matches `previous_response_id`. -- Route that shape through the existing proof-gated recovery, quarantine, masking, and settlement behavior. -- Keep unrelated `invalid_request_error` shapes request-owned and non-replayable. -- Add unit and public HTTP bridge regressions for the exact production event. -- Close the already-specified durable-owner fencing gap exposed by the cloud regression: a replacement for a reader-retired local session advances the owner epoch or reclaims the already-released row, while an atomic expected-owner check prevents a stale pre-connect lookup from stealing a lease claimed by another replica or process incarnation. - -## Non-Goals - -- Do not broaden recovery to arbitrary invalid requests. -- Do not drop client-supplied anchors without the existing complete-context proof. -- Do not weaken duplicate-replay, account-ownership, or durable-settlement fences. diff --git a/openspec/changes/recognize-canonical-invalid-previous-response-anchor/specs/responses-api-compat/spec.md b/openspec/changes/recognize-canonical-invalid-previous-response-anchor/specs/responses-api-compat/spec.md deleted file mode 100644 index 852ffb581f..0000000000 --- a/openspec/changes/recognize-canonical-invalid-previous-response-anchor/specs/responses-api-compat/spec.md +++ /dev/null @@ -1,56 +0,0 @@ -## MODIFIED Requirements - -### Requirement: Single HTTP bridge previous-response misses recover or fail closed -When an HTTP bridge session receives an anonymous upstream `previous_response_not_found` error for a single pending follow-up request, the service MUST treat the error as an internal continuity-loss signal. The same treatment MUST apply when upstream returns `code=invalid_request_error` with the exact canonical message `Invalid previous_response_id.` (allowing only quote/backtick and terminal-period variations) and either omits `param` or sets `param=previous_response_id`. It MUST either recover through the existing previous-response rebind path or rewrite the error to a retryable continuity failure instead of forwarding the raw upstream invalid-request error. Other `invalid_request_error` messages or conflicting `param` values MUST retain their ordinary request-error classification. - -#### Scenario: single pending HTTP bridge follow-up loses previous-response continuity -- **WHEN** an HTTP `/v1/responses` or `/backend-api/codex/responses` bridge session has exactly one pending request with `previous_response_id` -- **AND** upstream emits `previous_response_not_found` without a `response.id` -- **THEN** the service attempts the existing previous-response recovery path -- **AND** if recovery is unavailable, it emits a retryable continuity failure for that request -- **AND** the downstream error code is not `previous_response_not_found` - -#### Scenario: canonical invalid previous-response anchor omits param -- **WHEN** an HTTP `/v1/responses` or `/backend-api/codex/responses` bridge session has a pending request with `previous_response_id` -- **AND** upstream emits `code=invalid_request_error`, no `param`, and the exact message `Invalid previous_response_id.` before `response.created` -- **THEN** the service classifies the event as previous-response continuity loss -- **AND** it uses the same proof-gated recovery or fail-closed path as `previous_response_not_found` -- **AND** it does not forward the raw invalid-request response - -#### Scenario: unrelated invalid request remains a request error -- **WHEN** upstream emits `code=invalid_request_error` -- **AND** its `param` conflicts with `previous_response_id` or its message contains additional request-validation text -- **THEN** the service MUST NOT classify that error as previous-response continuity loss - -### Requirement: Public Responses errors mask previous-response misses -Public Responses endpoints MUST NOT return an OpenAI-shaped previous-response continuity error to clients. If a lower layer still raises or collects that error, the API layer MUST rewrite it to a retryable `stream_incomplete` continuity failure and remove the missing response id from the public payload. The exact canonical `Invalid previous_response_id.` invalid-request shape MUST use the same masking and recovery path only when its `param` is absent or matches `previous_response_id`; unrelated invalid requests MUST remain unchanged. - -#### Scenario: API layer receives an upstream previous-response miss -- **WHEN** a public `/responses`, `/v1/responses`, `/responses/compact`, or `/v1/responses/compact` handler receives an error with `code=previous_response_not_found` -- **OR** it receives `code=invalid_request_error` with `param=previous_response_id` and a message saying the previous response was not found -- **THEN** the response status is retryable -- **AND** the public error code is `stream_incomplete` -- **AND** the missing `previous_response_id` is not exposed in the response body - -#### Scenario: canonical invalid anchor is not exposed -- **WHEN** a public Responses handler receives `code=invalid_request_error` with an absent or matching `param` and the exact canonical message `Invalid previous_response_id.` -- **THEN** the raw invalid-request response is not exposed -- **AND** the request uses the existing proof-gated recovery or retryable continuity failure path - -## ADDED Requirements - -### Requirement: Reader-retired replacements preserve cross-replica ownership -When a local reader-retired HTTP bridge session is replaced before its bounded close releases the durable row, the replacement MUST advance the durable owner epoch if this replica still owns the row and MUST be able to reclaim a row that the old close already released. The pre-connect owner instance and process identity MUST be rechecked atomically under the durable row lock. A stale pre-connect ownership lookup MUST NOT grant takeover permission if another replica or newer process incarnation claims an active lease while the replacement opens its upstream connection. - -#### Scenario: another replica claims during replacement connect -- **GIVEN** the pre-connect durable lookup identifies the current replica as owner -- **AND** the old bounded close releases that owner while a replacement upstream connection is opening -- **WHEN** another replica claims an active lease before the replacement commits its durable claim -- **THEN** the replacement fails closed with an owner mismatch -- **AND** the other replica remains the durable owner at the same fencing epoch - -#### Scenario: old close releases before replacement claim -- **GIVEN** the pre-connect durable lookup identifies the current replica and process as owner -- **WHEN** the old bounded close releases the row before the replacement commits its durable claim -- **THEN** the replacement reclaims the released row -- **AND** it advances the fencing epoch so the old close cannot affect the replacement diff --git a/openspec/changes/recognize-canonical-invalid-previous-response-anchor/tasks.md b/openspec/changes/recognize-canonical-invalid-previous-response-anchor/tasks.md deleted file mode 100644 index c40dd8159c..0000000000 --- a/openspec/changes/recognize-canonical-invalid-previous-response-anchor/tasks.md +++ /dev/null @@ -1,10 +0,0 @@ -# Tasks - -- [x] Extend the shared previous-response classifier for the exact canonical invalid-anchor message and fail closed on conflicting params or additional text. -- [x] Add unit coverage for accepted and rejected error shapes. -- [x] Add a public HTTP bridge regression that quarantines the exact production shape and recovers only after a verified complete resend. -- [x] Add a deterministic old-close/replacement-claim race regression and fence the replacement durable owner with a new epoch. -- [x] Atomically bind same-instance epoch fencing to the pre-connect owner identity; allow released-row reclaim but reject another replica/process claim during upstream connect. -- [x] Update the canonical Responses compatibility specification. -- [x] Run focused tests, strict OpenSpec validation, lint, and typecheck. -- [ ] Obtain terminal cloud checks and an exact-head clean Codex review before merge. diff --git a/openspec/changes/recover-proxied-websocket-early-close/proposal.md b/openspec/changes/recover-proxied-websocket-early-close/proposal.md deleted file mode 100644 index da8c0938c0..0000000000 --- a/openspec/changes/recover-proxied-websocket-early-close/proposal.md +++ /dev/null @@ -1,22 +0,0 @@ -## Why - -The production Responses bridge observed an HTTP-proxy WebSocket connection -closing while TLS setup was still transferring the transport to -`websockets.ClientConnection`. The dependency invoked `connection_lost()` -before `connection_made()` initialized its receive assembler, raised an -uncaught `AttributeError`, and left the affected streamed request without a -terminal `response.completed` event. A later retry succeeded, but the first -request was already broken. - -## What Changes - -- Use a narrowly scoped `ClientConnection` adapter for proxied upstream - WebSockets. -- Treat a close before `connection_made()` as a pre-dispatch transport failure, - complete the dependency's waiter without touching uninitialized state, and - retry one fresh tunnel on the same account. -- Keep established connections on the dependency's normal close path. -- Keep shared environment-proxy failures account-neutral: an exhausted retry - returns a typed connection error without backing off or rotating accounts. -- Add adapter-level and public HTTP Responses regressions for the exact - pre-`connection_made()` close shape. diff --git a/openspec/changes/recover-proxied-websocket-early-close/specs/outbound-http-clients/spec.md b/openspec/changes/recover-proxied-websocket-early-close/specs/outbound-http-clients/spec.md deleted file mode 100644 index e86547774f..0000000000 --- a/openspec/changes/recover-proxied-websocket-early-close/specs/outbound-http-clients/spec.md +++ /dev/null @@ -1,38 +0,0 @@ -## ADDED Requirements - -### Requirement: Proxied WebSocket setup closes fail as pre-dispatch transport errors - -When an upstream WebSocket uses an HTTP proxy and the transport closes while -TLS setup is transferring the transport to the WebSocket protocol, before the -protocol's `connection_made()` initializes receive state, the service MUST -complete connection-lost bookkeeping without dereferencing uninitialized -receive or transport attributes. The service MUST retry exactly one fresh -tunnel on the same account because no application frame was dispatched. If -that retry also fails, it MUST return a typed pre-dispatch transport error -without penalizing or rotating accounts and MUST NOT leave an HTTP Responses -stream pending without a terminal event. Once `connection_made()` has run, -the dependency's established-connection close semantics MUST remain unchanged. - -#### Scenario: proxy transport closes before connection setup completes - -- **GIVEN** a secure upstream Responses WebSocket is routed through an HTTP proxy -- **AND** the proxy transport closes before `connection_made()` initializes the receive assembler -- **WHEN** the WebSocket dependency reports `connection_lost()` -- **THEN** the service completes the connection-lost waiter without raising an event-loop callback exception -- **AND** retries one fresh proxy tunnel on the same account -- **AND** no request is treated as having reached upstream - -#### Scenario: shared proxy setup retry is exhausted - -- **GIVEN** the fresh same-account proxy tunnel also closes before setup completes -- **WHEN** the service returns the connection failure -- **THEN** the failure identifies exhausted shared-proxy setup -- **AND** the selected account is not backed off or excluded -- **AND** no other account is tried through the same failing shared proxy - -#### Scenario: established proxied connection keeps normal close semantics - -- **GIVEN** the proxied WebSocket completed `connection_made()` and initialized receive state -- **WHEN** the established connection closes -- **THEN** the dependency's normal close path handles pending receives, pings, and drain waiters -- **AND** the adapter does not weaken or suppress the established-connection failure classification diff --git a/openspec/changes/recover-proxied-websocket-early-close/tasks.md b/openspec/changes/recover-proxied-websocket-early-close/tasks.md deleted file mode 100644 index 3c74670bee..0000000000 --- a/openspec/changes/recover-proxied-websocket-early-close/tasks.md +++ /dev/null @@ -1,7 +0,0 @@ -- [x] Define the proxied WebSocket pre-dispatch close contract. -- [x] Add adapter coverage for a close before and after `connection_made()`. -- [x] Add a public HTTP Responses regression proving shared-proxy failures stay account-neutral. -- [x] Implement the narrow proxied `ClientConnection` adapter. -- [x] Run strict OpenSpec, formatting, lint, type, unit, and integration gates. -- [ ] Merge through repository gates, publish the exact image, and deploy through immutable GitOps. -- [ ] Verify production continuity and both affected long-running tasks without recurrent bridge errors. diff --git a/openspec/changes/recover-rate-limited-on-confirmed-reset/.openspec.yaml b/openspec/changes/recover-rate-limited-on-confirmed-reset/.openspec.yaml new file mode 100644 index 0000000000..a8821c74d0 --- /dev/null +++ b/openspec/changes/recover-rate-limited-on-confirmed-reset/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-11 diff --git a/openspec/changes/recover-rate-limited-on-confirmed-reset/context.md b/openspec/changes/recover-rate-limited-on-confirmed-reset/context.md new file mode 100644 index 0000000000..bc5870ed3d --- /dev/null +++ b/openspec/changes/recover-rate-limited-on-confirmed-reset/context.md @@ -0,0 +1,7 @@ +## Warm-up settings lifecycle + +`limit_warmup_exhausted_threshold_percent` no longer controls reset-confirmed warm-up. A real selected-window reset must be eligible regardless of prior usage, so retaining that threshold in candidate selection would contradict this change's behavioral contract. + +This PR intentionally leaves the persisted setting, settings API, and dashboard field in place to keep the recovery fix to one concern. Their end-to-end removal will be handled as a dedicated OpenSpec follow-up under the settings-surface reduction tracked by #1340, including the database model and migration, API schemas, dashboard control, tests, and operator documentation. + +`limit_warmup_cooldown_seconds` remains active but intentionally applies only to staggered idle warm-up. Reset-confirmed warm-up uses the atomic account/window/reset claim instead: an attempt for the same tuple is deduplicated, while a distinct real reset is not suppressed merely because another attempt happened recently. diff --git a/openspec/changes/recover-rate-limited-on-confirmed-reset/design.md b/openspec/changes/recover-rate-limited-on-confirmed-reset/design.md new file mode 100644 index 0000000000..3bfe257b8a --- /dev/null +++ b/openspec/changes/recover-rate-limited-on-confirmed-reset/design.md @@ -0,0 +1,78 @@ +## Context + +Persisted `status`, `reset_at`, and `blocked_at` are the cross-replica authority for account availability. A 429 therefore survives process restarts and peer selection, but the current recovery gate also treats every future `reset_at` as authoritative even when usage history proves that the exact monthly window associated with the block has reset. Free accounts can consequently remain `rate_limited` through a fresh monthly window, and the warm-up layer correctly refuses to contact them because they are not `active`. + +The scheduler already captures usage immediately before and after a selected-account refresh, usage history survives restarts, account status writes support compare-and-set guards, and warm-up attempts are deduplicated by account/window/reset. The design uses those existing primitives and does not introduce a setting or a second recovery mechanism. + +## Goals / Non-Goals + +**Goals:** + +- Recover a Free account from a stale future rate-limit marker when a temporal monthly transition proves that the blocked window reset and fresh quota is available. +- Preserve the 30-second post-429 floor and the full persisted cooldown for generic 429, Retry-After, jitter-only, stale, mismatched, or exhausted evidence. +- Make recovery atomic and order it before ordinary active-only warm-up. +- Preserve recovery and warm-up behavior across scheduler/process restarts by using persisted usage history. +- Reuse one confirmed reset tuple for status recovery and durable warm-up deduplication. + +**Non-Goals:** + +- Probe or warm a still-blocked account as a way to infer whether a throttle ended. +- Generalize early recovery to Plus/Pro primary-window exhaustion, `quota_exceeded`, auth failures, paused/deactivated accounts, model-scoped throttles, or generic Retry-After cooldowns. +- Add configuration, schema, migration, dashboard, API, or new background-worker behavior. +- Change the existing recovery rules after a persisted cooldown has naturally elapsed. + +## Decisions + +### Resolve one canonical monthly reset-evidence tuple + +For the selected account, the scheduler will first test its in-memory monthly before/after samples with the existing temporal reset-confirmation predicate. A blocked Free account must still query persisted monthly history recorded since `blocked_at` when that current pair confirms a transition whose baseline does not anchor to the current block marker. That history must contain a baseline recorded strictly after `blocked_at` whose reset deadline matches the account marker; a matching row at the exact block timestamp is not eligible and cannot shadow a later valid baseline. Only adjacent pairs at or after the eligible baseline are then evaluated, and the most recent pair that passes the temporal reset predicate becomes the transition evidence. This handles upstream monthly deadlines that slide between samples without comparing non-neighboring rows, while preventing newer unanchored evidence from masking a valid persisted recovery path. + +The persisted lookup makes the evidence restart-safe: a process that starts after the transition can recover from the same history rather than waiting for the stale account deadline. Both paths retain the matching baseline alongside the canonical `(before, after)` transition, and the normal reset predicate remains the single authority for scheduled-boundary crossing or a quota-recovery re-anchor within the observation interval. Warm-up consumes only the adjacent transition pair; recovery additionally checks the matching baseline. + +Alternative considered: infer recovery from repeated `monthly < 100%` snapshots. Availability alone does not identify which 429 or quota window ended and would weaken generic Retry-After protection. + +### Require the reset evidence to identify the current block + +Early recovery applies only to a `rate_limited` Free account with both markers present and a still-future persisted `reset_at`. The scheduler requires all of the following: + +- at least 30 seconds have elapsed since `blocked_at`; +- the evidence is a monthly-to-monthly temporal reset transition; +- persisted post-block history contains a monthly baseline whose `reset_at` matches the persisted account `reset_at` within five seconds; +- the reset evidence is an adjacent monthly pair at or after that matching baseline; +- the transition's after sample and the latest monthly sample were recorded after `blocked_at` and both report usage below 100 percent. + +The deadline match binds the usage transition to the block being recovered, while the latest-sample check prevents an earlier good sample from reactivating an account that exhausted the new window again. Free-plan scoping ensures a Plus account with an exhausted primary window cannot be released by unrelated long-window availability. + +Alternative considered: let any fresh available usage override a future deadline after 30 seconds. That cannot distinguish a quota-window reset from a generic throttle or a model/account restriction. + +### Recover through marker-guarded compare-and-set before warm-up + +The scheduler will perform recovery before invoking warm-up. The status transition compares the current status, deactivation reason, `reset_at`, and `blocked_at`; on success it writes `active`, clears the reason and both block markers, and updates the selected detached account passed to warm-up. A miss leaves that object blocked, so a concurrent newer 429 or operator change wins and no warm-up candidate is evaluated from the stale snapshot. + +Warm-up candidate selection remains restricted to `active`, and the sender independently reloads account state and requires `active` immediately before network I/O. These two checks cover both a failed recovery CAS and a re-block that lands after candidate creation. + +Alternative considered: send warm-up while `rate_limited` and promote on a 2xx response. A model- or scope-specific success would not prove that the account-wide throttle ended and would bypass the persisted cross-replica gate. + +### Reuse reset evidence for normal warm-up and deduplication + +After a successful recovery, the scheduler will pass the same resolved monthly before/after pair into the existing selected-window warm-up evaluation. Candidate construction therefore derives the new monthly reset tuple even after a restart, while the existing atomic attempt claim continues to enforce at most one account/window/reset attempt across workers. + +Reset confirmation no longer requires the previous sample to be exhausted. A real temporal reset with newly available quota is eligible regardless of how much of the old window was used, subject to existing opt-in and availability gates. Timestamp jitter without a real boundary crossing or re-anchor remains ineligible. + +Alternative considered: add a recovery-specific sender or dedupe key. That would duplicate safety checks and could create two attempts for the same reset. + +## Risks / Trade-offs + +- [Risk] A persisted deadline could coincidentally resemble a monthly reset. → Restrict the exception to Free accounts and require a five-second deadline match, temporal reset proof, post-block samples, fresh availability, and the 30-second floor. +- [Risk] A newer block or operator state change can race with recovery. → Guard every persisted marker in the compare-and-set and warm only after it succeeds; re-read active status in the sender. +- [Risk] Retention may remove the transition pair before a restarted scheduler observes it. → Fail closed and preserve the ordinary persisted cooldown; do not synthesize evidence from availability alone. +- [Risk] The new monthly window can be exhausted after the transition. → Require both the transition after sample and latest monthly sample to remain below 100 percent. +- [Trade-off] Recovery may perform one bounded history lookup for a selected blocked Free account. The lookup is account/window/time scoped and avoids fleet-wide work. + +## Migration Plan + +No data or configuration migration is required. Deploy the scheduler and warm-up changes together, then verify that qualifying Free accounts transition to `active`, clear both block markers, and create at most one monthly warm-up attempt while genuinely exhausted Plus accounts remain blocked. Rollback restores the prior conservative behavior; already recovered account rows remain valid active state and require no data repair. + +## Open Questions + +None. diff --git a/openspec/changes/recover-rate-limited-on-confirmed-reset/proposal.md b/openspec/changes/recover-rate-limited-on-confirmed-reset/proposal.md new file mode 100644 index 0000000000..8578301bae --- /dev/null +++ b/openspec/changes/recover-rate-limited-on-confirmed-reset/proposal.md @@ -0,0 +1,28 @@ +## Why + +A real quota-window reset can leave a `rate_limited` account unavailable until a stale persisted cooldown deadline, even though fresh usage already proves that the blocked window reset and quota is available. This also prevents the normal reset warm-up from starting the new window, so recovery must precede warm-up without weakening generic 429 and Retry-After protection. + +## What Changes + +- Allow background usage refresh to recover a `rate_limited` account before its persisted `reset_at` only when a post-block monthly-window transition proves the specific blocked window reset, the new window has available quota, and the minimum 30-second post-429 floor has elapsed. +- Require recovery to use a compare-and-set transition that matches the blocked status and markers, clears `reset_at` and `blocked_at`, and completes before normal warm-up evaluation. +- Keep generic 429 and Retry-After cooldowns protected when no qualifying reset transition exists, and keep exhausted or unsafe account states blocked. +- Restore warm-up traffic to `active` accounts only while triggering one deduplicated warm-up after every confirmed selected-window reset, regardless of the previous window's usage percentage. +- Add regression coverage for restart-persisted reset evidence, stale or mismatched markers, concurrent re-blocking, exhausted fresh windows, the cooldown floor, unsafe states, and reset-tuple deduplication. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `usage-refresh-policy`: Define recovery-before-warm-up behavior for a reset-confirmed monthly window and require warm-up after every real selected-window reset rather than only exhausted-to-available transitions. +- `account-routing`: Permit the strict reset-confirmed recovery exception to a future persisted cooldown while preserving cross-replica enforcement for generic 429 and Retry-After cooldowns. + +## Impact + +- Affected code: background usage refresh scheduling, recoverable-status reconciliation, limit warm-up candidate construction, and the compare-and-set account-status repository path. +- Affected tests: focused scheduler, status-recovery, warm-up, and repository-backed integration coverage. +- No API, schema, migration, setting, dependency, dashboard, or deployment contract changes. diff --git a/openspec/changes/recover-rate-limited-on-confirmed-reset/specs/account-routing/spec.md b/openspec/changes/recover-rate-limited-on-confirmed-reset/specs/account-routing/spec.md new file mode 100644 index 0000000000..a1b6e0b7b1 --- /dev/null +++ b/openspec/changes/recover-rate-limited-on-confirmed-reset/specs/account-routing/spec.md @@ -0,0 +1,71 @@ +## MODIFIED Requirements + +### Requirement: Rate-limit cooldowns are enforced across replicas + +A replica that did not observe the upstream 429 MUST NOT transition a `RATE_LIMITED` account to `ACTIVE` while the persisted `reset_at` deadline is in the future unless background usage refresh proves that the exact blocked Free monthly window reset under the strict exception below. For `RATE_LIMITED` rows with `blocked_at` set but no persisted `reset_at` (legacy rows written before cooldown persistence), replicas MUST hold the account `RATE_LIMITED` until at least `blocked_at + RATE_LIMITED_MIN_COOLDOWN_SECONDS`. Recovery transitions MUST be written through the compare-and-set status update (`update_status_if_current`) so a stale snapshot cannot clobber a newer marking. + +The reset-confirmed exception SHALL apply only to a Free account with a still-future persisted deadline after the 30-second minimum floor has elapsed. Post-block monthly history MUST contain a baseline whose reset deadline matches the persisted account deadline within five seconds, and an adjacent monthly before/after pair at or after that baseline MUST prove a real temporal reset. Both the after sample and latest monthly sample MUST be post-block and below `100%`. The recovery compare-and-set MUST match the persisted status, deactivation reason, `reset_at`, and `blocked_at`, then clear both markers when it writes `ACTIVE`. The evidence MAY be loaded from persisted history after a process restart, but availability alone and comparisons between non-neighboring rows MUST NOT satisfy the exception. + +This constraint applies to every recovery path that writes account status, including the usage-refresh reconcile path. A usage refresh that observes available quota for a `RATE_LIMITED` account with `blocked_at` set MUST NOT rewrite the account to `ACTIVE` or clear its markers while the effective persisted cooldown is running unless the strict reset-confirmed exception succeeds. The replica that observed the current 429 MAY still recover earlier through its runtime-cooldown-gated fresh-usage path only when its runtime block marker is at least as recent as the effective persisted `blocked_at`; leftover runtime state from an earlier 429 MUST NOT unlock early recovery of a newer block. `RATE_LIMITED` rows without `blocked_at` keep the existing fresh-usage recovery. Generic 429 and Retry-After cooldowns without matching reset evidence, reset timestamp jitter, exhausted post-reset windows, and non-Free account exhaustion MUST remain protected until their ordinary recovery condition is met. + +#### Scenario: Usage refresh does not clear a running Retry-After cooldown + +- **GIVEN** an account marked `RATE_LIMITED` by a 429 whose Retry-After hint persisted `reset_at` 20 minutes in the future and `blocked_at` set +- **WHEN** a periodic usage refresh fetches fresh usage showing available quota before that deadline +- **AND** no qualifying Free monthly reset transition matches the persisted deadline +- **THEN** the persisted row keeps status `RATE_LIMITED` with its `reset_at` and `blocked_at` intact +- **AND** once the deadline elapses, a later refresh may recover the account to `ACTIVE` through the compare-and-set path + +#### Scenario: Confirmed blocked Free monthly reset permits peer recovery + +- **GIVEN** replica A marked a Free account `RATE_LIMITED` with `blocked_at` and a persisted deadline matching that account's monthly window +- **AND** the 30-second minimum floor has elapsed +- **WHEN** replica B observes a real post-block transition from the matching monthly baseline into a new available monthly window +- **AND** the latest monthly sample remains below `100%` +- **THEN** replica B may compare-and-set the account to `ACTIVE` before the old persisted deadline +- **AND** a successful transition clears `reset_at` and `blocked_at` + +#### Scenario: Generic 429 without matching reset evidence remains protected + +- **GIVEN** an account has a future persisted cooldown from an upstream 429 or Retry-After hint +- **AND** fresh usage reports availability but no temporal monthly reset whose baseline matches that deadline +- **WHEN** any replica evaluates recovery +- **THEN** the account remains `RATE_LIMITED` until an ordinary recovery condition is met + +#### Scenario: Peer replica does not flip a cooling account back + +- **GIVEN** balancer instance A marked account X `RATE_LIMITED` from a 429 with no reset metadata +- **AND** account X's recorded usage is below 100% +- **WHEN** a second balancer instance sharing the same database runs account selection +- **THEN** account X is not selected +- **AND** the persisted row remains `RATE_LIMITED` with its `reset_at` deadline intact until the deadline elapses or strict reset-confirmed recovery succeeds + +#### Scenario: Stale runtime cooldown does not unlock early recovery of a newer block + +- **GIVEN** a replica holds expired runtime cooldown state left over from an earlier 429 of account X +- **AND** account X was since re-marked `RATE_LIMITED` by a peer replica with a newer `blocked_at` and a future persisted `reset_at` +- **WHEN** the replica evaluates account X with usage recorded after the newer `blocked_at` +- **AND** no strict reset-confirmed transition matches the newer block +- **THEN** account X stays `RATE_LIMITED` and is not selected until the persisted deadline elapses + +#### Scenario: Concurrent newer block wins the recovery race + +- **GIVEN** reset evidence qualifies a blocked Free account for early recovery +- **AND** another replica changes its status or either block marker before recovery commits +- **WHEN** the recovery compare-and-set evaluates the older snapshot +- **THEN** it does not overwrite the newer account row +- **AND** the account is not made routable from the stale evidence + +#### Scenario: Exhausted Plus primary window remains protected + +- **GIVEN** a Plus account is `RATE_LIMITED` with primary usage at `100%` +- **WHEN** a replica observes available long-window usage or a long-window reset +- **THEN** the Free monthly reset exception does not apply +- **AND** the account remains unavailable until its ordinary recovery condition is met + +#### Scenario: Legacy row without reset_at is floored + +- **GIVEN** a persisted `RATE_LIMITED` row with `blocked_at` five seconds ago and `reset_at` NULL +- **WHEN** a fresh balancer instance evaluates it during selection +- **THEN** the account stays `RATE_LIMITED` and is not selected +- **AND** once the 30-second floor has elapsed, recovery back to `ACTIVE` is permitted through the compare-and-set path diff --git a/openspec/changes/recover-rate-limited-on-confirmed-reset/specs/usage-refresh-policy/spec.md b/openspec/changes/recover-rate-limited-on-confirmed-reset/specs/usage-refresh-policy/spec.md new file mode 100644 index 0000000000..8c3fd5925d --- /dev/null +++ b/openspec/changes/recover-rate-limited-on-confirmed-reset/specs/usage-refresh-policy/spec.md @@ -0,0 +1,243 @@ +## MODIFIED Requirements + +### Requirement: Background usage refresh reconciles recoverable blocked statuses + +Background usage refresh SHALL reconcile persisted `rate_limited` and `quota_exceeded` accounts back to `active` after it writes fresh usage snapshots that prove the blocked window has recovered. This reconciliation SHALL be recovery-only and SHALL NOT promote `active` accounts into blocked statuses. For `rate_limited` accounts, recovery evidence SHALL come from the most recently recorded main-window row: when a post-block refresh no longer reports a short primary window and the last primary sample's own reset deadline has elapsed (or no primary sample exists), a fresh long-window row recorded after the block that still reports usage below `100%` proves recovery. While the last primary sample still claims an unexpired window (or omits reset metadata), or the newer long-window row is itself exhausted, primary freshness SHALL keep gating recovery. + +A future persisted `reset_at` SHALL continue to block ordinary recovery except for a `rate_limited` Free account whose monthly usage history proves that the specific monthly window associated with the current block reset. This exception MUST require `blocked_at` and a future persisted `reset_at`, at least 30 seconds elapsed after `blocked_at`, a monthly baseline recorded strictly after `blocked_at` whose `reset_at` matches the persisted marker within five seconds, a real temporal reset in an adjacent monthly pair at or after that baseline, and both the transition's after sample and the latest monthly sample recorded after `blocked_at` with usage below `100%`. The reset pair MAY come from the current refresh or be selected from adjacent persisted post-block samples so recovery survives a process restart and tolerates sliding reset deadlines without comparing non-neighboring rows. Availability without that matching anchored transition, reset timestamp jitter, an exhausted latest window, or evidence for a non-Free account MUST NOT override the persisted cooldown. + +Every recovery write MUST compare the current status, deactivation reason, `reset_at`, and `blocked_at`. A successful write SHALL set the account to `active` and clear the deactivation reason and both block markers. A compare-and-set miss MUST preserve the newer row and MUST NOT make the stale account snapshot eligible for warm-up. + +#### Scenario: Scheduler recovers a stale rate-limited account from fresh primary usage +- **WHEN** an account is persisted as `rate_limited` +- **AND** the persisted rate-limit reset deadline has already elapsed +- **AND** a later background usage refresh writes a fresh primary usage row recorded after the persisted block marker +- **AND** that primary usage row reports usage below `100%` +- **THEN** the scheduler marks the account `active` +- **AND** it clears persisted `reset_at` and `blocked_at` + +#### Scenario: Scheduler recovers a rate-limited account that never had a primary row +- **WHEN** an account is persisted as `rate_limited` with no stored primary-slot row at all +- **AND** the persisted rate-limit reset deadline has already elapsed +- **AND** a later background usage refresh records a fresh long-window row below `100%` after the persisted block marker +- **THEN** the scheduler marks the account `active` +- **AND** it clears persisted `reset_at` and `blocked_at` + +#### Scenario: Scheduler recovers a rate-limited account when upstream stops reporting the primary window +- **WHEN** an account is persisted as `rate_limited` +- **AND** the persisted rate-limit reset deadline has already elapsed +- **AND** the last primary usage sample's own reset deadline has also elapsed +- **AND** a later background usage refresh records only a long-window usage row after the persisted block marker +- **AND** that long-window row reports usage below `100%` +- **THEN** the scheduler marks the account `active` +- **AND** it clears persisted `reset_at` and `blocked_at` + +#### Scenario: Unexpired primary sample keeps gating recovery evidence +- **WHEN** an account is persisted as `rate_limited` +- **AND** the last primary usage sample predates the block but still claims an unexpired reset deadline +- **AND** a later refresh recorded only a fresh long-window row +- **AND** no qualifying reset-confirmed Free monthly transition matches the current block +- **THEN** the account stays `rate_limited` until fresh primary evidence arrives, the primary sample's reset deadline elapses, or a qualifying monthly reset is confirmed + +#### Scenario: Scheduler recovers a legacy rate-limited account without a block marker +- **WHEN** an account is persisted as `rate_limited` +- **AND** the persisted rate-limit reset deadline has already elapsed +- **AND** the account has no persisted block marker +- **AND** a later background usage refresh writes a recent primary usage row that reports usage below `100%` +- **THEN** the scheduler marks the account `active` +- **AND** it clears persisted `reset_at` + +#### Scenario: Scheduler preserves legacy rate-limited accounts without recent primary usage +- **WHEN** an account is persisted as `rate_limited` +- **AND** the persisted rate-limit reset deadline has already elapsed +- **AND** the account has no persisted block marker +- **AND** the latest primary usage row is not recent enough to prove background refresh recovery +- **AND** no newer long-window row proves a post-block refresh +- **THEN** the scheduler leaves the account `rate_limited` + +#### Scenario: Scheduler preserves an unexpired rate-limit cooldown +- **WHEN** an account is persisted as `rate_limited` +- **AND** its persisted rate-limit reset deadline is still in the future +- **AND** a later background usage refresh writes fresh available usage +- **AND** no qualifying reset-confirmed Free monthly transition matches the current block +- **THEN** the scheduler leaves the account `rate_limited` + +#### Scenario: Confirmed Free monthly reset recovers before a stale deadline +- **GIVEN** a Free account is persisted as `rate_limited` with `blocked_at` more than 30 seconds ago and a future `reset_at` +- **AND** a monthly baseline recorded strictly after `blocked_at` has a reset deadline within five seconds of the persisted marker +- **WHEN** background usage refresh confirms a real transition in an adjacent monthly pair at or after that matching baseline +- **AND** the transition's after sample and latest monthly sample were recorded after `blocked_at` and report usage below `100%` +- **THEN** the scheduler atomically marks the account `active` before the stale persisted deadline +- **AND** it clears `reset_at`, `blocked_at`, and the deactivation reason + +#### Scenario: Persisted monthly transition recovers after scheduler restart +- **GIVEN** a qualifying Free monthly reset transition was persisted after `blocked_at` +- **AND** the scheduler process restarts after the transition is no longer the current in-memory before/after pair +- **WHEN** the restarted scheduler refreshes the still-`rate_limited` account before its stale persisted deadline +- **THEN** it may use a matching persisted baseline plus a later adjacent monthly transition pair as reset evidence +- **AND** it recovers the account through the same marker-guarded transition + +#### Scenario: A baseline at the exact block timestamp cannot shadow a later valid baseline +- **GIVEN** persisted monthly history contains a reset-matching row recorded exactly at `blocked_at` +- **AND** a later row recorded strictly after `blocked_at` matches the same persisted reset marker +- **AND** an adjacent reset transition follows that later row +- **WHEN** the restarted scheduler resolves persisted recovery evidence +- **THEN** it MUST ignore the row recorded exactly at `blocked_at` +- **AND** it MUST use the later matching baseline to evaluate the qualifying transition + +#### Scenario: An unanchored current transition cannot mask persisted recovery evidence +- **GIVEN** the current monthly before/after pair confirms a reset whose baseline does not match the blocked Free account's persisted reset marker +- **AND** persisted monthly history contains an eligible post-block baseline plus a qualifying adjacent reset transition +- **WHEN** the scheduler resolves monthly reset evidence +- **THEN** it MUST scan persisted history instead of short-circuiting on the unanchored current pair +- **AND** it MUST use evidence anchored to the persisted block marker for recovery and warm-up + +#### Scenario: Minimum post-block floor prevents immediate recovery +- **GIVEN** a Free account was marked `rate_limited` less than 30 seconds ago +- **AND** monthly samples otherwise appear to prove a reset with available quota +- **WHEN** background usage refresh evaluates recovery +- **THEN** the account remains `rate_limited` with both block markers intact + +#### Scenario: Mismatched monthly baseline does not recover the current block +- **GIVEN** a Free account has a future persisted rate-limit deadline +- **AND** monthly history contains a real reset transition whose baseline deadline differs from that marker by more than five seconds +- **WHEN** background usage refresh evaluates recovery +- **THEN** the transition is not treated as evidence for the current block +- **AND** the account remains `rate_limited` + +#### Scenario: Later exhausted monthly state defeats older recovery evidence +- **GIVEN** a Free account has a qualifying post-block monthly reset transition whose after sample reports available quota +- **AND** its latest monthly sample reports usage at or above `100%` +- **WHEN** background usage refresh evaluates recovery +- **THEN** the account remains blocked + +#### Scenario: Plus primary exhaustion is not released by monthly evidence +- **GIVEN** a Plus account is persisted as `rate_limited` +- **AND** its current primary usage reports `100%` +- **WHEN** background usage refresh observes available long-window usage or an unrelated reset transition +- **THEN** the account remains `rate_limited` + +#### Scenario: Scheduler recovers a stale quota-exceeded account from fresh secondary usage +- **WHEN** an account is persisted as `quota_exceeded` +- **AND** a later background usage refresh writes a fresh secondary usage row that reports usage below `100%` +- **THEN** the scheduler marks the account `active` +- **AND** it clears persisted `reset_at` and `blocked_at` + +#### Scenario: Scheduler does not tighten active accounts into blocked statuses +- **WHEN** background usage refresh evaluates an account currently persisted as `active` +- **THEN** the scheduler does not change that account to `rate_limited` or `quota_exceeded` + +#### Scenario: Scheduler ignores stale pre-block recovery evidence +- **WHEN** an account is persisted as `rate_limited` +- **AND** the latest primary usage row was recorded before the persisted block marker +- **AND** no newer long-window row or qualifying post-block monthly reset transition proves recovery +- **THEN** the scheduler leaves the account blocked + +#### Scenario: Scheduler skips recovery when the account row changed concurrently +- **WHEN** background usage refresh determines that a blocked account is recoverable +- **AND** the persisted account status, reason, or reset markers change before the scheduler writes recovery +- **THEN** the scheduler skips the stale recovery write +- **AND** warm-up does not use that stale recovery decision + +#### Scenario: Scheduler clears stale deactivation reasons on recovery +- **WHEN** background usage refresh recovers a `rate_limited` or `quota_exceeded` account to `active` +- **THEN** the scheduler writes `deactivation_reason` as `NULL` + +### Requirement: Reset-confirmed limit warm-up + +The system SHALL support an optional limit warm-up mechanism that is disabled by default. When enabled globally and for an account, background usage refresh MAY send one minimal upstream Responses request after it confirms that a selected quota window moved into a newly available reset window. Eligibility SHALL depend on a real reset transition and the configured post-reset availability gate, not on whether the previous window was exhausted. The legacy `limit_warmup_exhausted_threshold_percent` setting MUST NOT gate reset-confirmed eligibility. + +Background usage refresh MUST complete any applicable blocked-status reconciliation before warm-up evaluation. Candidate evaluation and the sender's fresh preflight check MUST both require the account to be `active`; paused, deactivated, `reauth_required`, `rate_limited`, and `quota_exceeded` accounts MUST NOT receive warm-up traffic. When a reset-confirmed recovery uses persisted transition evidence, warm-up SHALL reuse that same before/after pair so the new account/window/reset tuple enters the ordinary durable deduplication path. + +The configured `limit_warmup_cooldown_seconds` SHALL gate only staggered idle warm-up candidates. It MUST NOT suppress a reset-confirmed candidate for a distinct account/window/reset tuple, which remains protected by the durable atomic attempt claim for that tuple. + +#### Scenario: Warm-up follows a real reset regardless of prior usage +- **GIVEN** limit warm-up is enabled globally and for an active account +- **AND** the account's previous usage sample for a selected window reports any usage below or at exhaustion +- **WHEN** background usage refresh records a newer sample that proves a real reset for that window and satisfies the configured availability gate +- **THEN** the system sends at most one warm-up request for that account/window/reset tuple + +#### Scenario: Staggered idle cooldown does not suppress a distinct reset tuple +- **GIVEN** an account has a recent warm-up attempt inside `limit_warmup_cooldown_seconds` +- **AND** background usage refresh confirms a different selected account/window/reset tuple +- **WHEN** reset-confirmed warm-up evaluates the new tuple +- **THEN** the staggered idle cooldown MUST NOT suppress that candidate +- **AND** the durable attempt claim MUST still prevent another send for an already claimed identical tuple + +#### Scenario: Warm-up is skipped unless reset is confirmed +- **GIVEN** limit warm-up is enabled globally and for an account +- **WHEN** background usage refresh records a newer available sample without a real selected-window reset transition +- **THEN** the system MUST NOT send a reset-confirmed warm-up request for that sample + +#### Scenario: Warm-up is not triggered by upstream reset_at timestamp jitter +- **GIVEN** limit warm-up is enabled globally and for an account +- **WHEN** background usage refresh records a newer sample whose `reset_at` advanced by less than 60 seconds as upstream timestamp jitter +- **THEN** the system MUST NOT send a warm-up request for that account/window/reset tuple + +#### Scenario: Warm-up is opt-in and safe by default +- **GIVEN** background usage refresh is preparing to evaluate limit warm-up candidates +- **WHEN** global limit warm-up is disabled +- **OR** the account is not opted in +- **THEN** background usage refresh MUST NOT send warm-up traffic + +#### Scenario: Warm-up uses fresh opt-in state after usage refresh +- **GIVEN** an account was loaded before a background usage refresh cycle +- **AND** the account's limit warm-up opt-in changes while the refresh cycle is running +- **WHEN** the scheduler evaluates warm-up candidates after writing usage samples +- **THEN** the scheduler MUST evaluate the latest persisted opt-in value rather than the stale in-session account object + +#### Scenario: Warm-up respects unsafe account states +- **WHEN** an account is paused, deactivated, `reauth_required`, rate-limited, quota-exceeded, or in an auth-refresh failure path +- **THEN** limit warm-up MUST NOT send traffic for that account + +#### Scenario: Reset recovery completes before warm-up +- **GIVEN** an opted-in Free account is `rate_limited` and has qualifying monthly reset evidence +- **WHEN** marker-guarded recovery succeeds +- **THEN** the scheduler first persists the account as `active` and clears its block markers +- **AND** only then may it evaluate the same monthly reset tuple for warm-up + +#### Scenario: Recovery race prevents warm-up from stale evidence +- **GIVEN** reset evidence makes a blocked account appear recoverable +- **AND** a concurrent write changes its status or block markers before recovery persists +- **WHEN** the recovery compare-and-set misses +- **THEN** the stale scheduler snapshot remains ineligible for warm-up + +#### Scenario: Sender rejects an account re-blocked after candidate creation +- **GIVEN** an active account produced a valid warm-up candidate +- **AND** the account becomes blocked before upstream warm-up traffic begins +- **WHEN** the sender reloads the account state +- **THEN** it does not send the warm-up request + +#### Scenario: Warm-up attempts are durable and deduplicated +- **WHEN** multiple refresh workers observe the same account/window/reset candidate +- **THEN** the database permits at most one persisted attempt for that tuple +- **AND** later refresh cycles skip that tuple after a prior attempt exists + +#### Scenario: Persisted recovery evidence shares the warm-up tuple +- **GIVEN** a scheduler restart causes recovery to use a persisted monthly before/after transition +- **WHEN** the recovered active account reaches warm-up evaluation +- **THEN** warm-up derives the candidate from that same transition's new reset deadline +- **AND** an existing attempt for the account/monthly/reset tuple prevents another send + +#### Scenario: Staggered idle warm-up pre-starts rolling primary windows +- **GIVEN** limit warm-up and staggered idle warm-up are enabled globally +- **AND** multiple active accounts are opted into limit warm-up +- **AND** an opted-in account has a healthy idle short-window primary usage sample (any sample reporting a duration over 24 hours is not eligible) with `used_percent` at or below the configured `limit_warmup_idle_threshold_percent` +- **AND** no prior warm-up attempt places the account inside the configured cooldown +- **AND** the usage sample was refreshed for the current cycle +- **WHEN** background usage refresh evaluates that account inside its deterministic stagger slot +- **THEN** the system MUST attempt to send one minimal upstream warm-up request for that account's current rolling-window cycle, whose length is the account's observed primary window duration (defaulting to 300 minutes when duration metadata is missing) +- **AND** the system MUST NOT send another staggered idle warm-up for that same account/cycle tuple +- **AND** account slots MUST be spread deterministically across the account's rolling window so restarts do not align all opted-in accounts into the same phase + +#### Scenario: Staggered idle warm-up is skipped for accounts with real usage +- **GIVEN** staggered idle warm-up is enabled globally +- **AND** an active opted-in account has a short-window primary usage sample with `used_percent` above the configured `limit_warmup_idle_threshold_percent` +- **WHEN** background usage refresh evaluates that account +- **THEN** the system MUST NOT send staggered idle warm-up traffic for that account + +#### Scenario: Staggered idle warm-up remains opt-in +- **GIVEN** limit warm-up is enabled globally and for an account +- **AND** staggered idle warm-up is disabled +- **WHEN** background usage refresh observes an idle short-window primary sample that is not a reset-confirmed transition +- **THEN** limit warm-up MUST NOT send synthetic traffic for that idle sample diff --git a/openspec/changes/recover-rate-limited-on-confirmed-reset/tasks.md b/openspec/changes/recover-rate-limited-on-confirmed-reset/tasks.md new file mode 100644 index 0000000000..2d762983da --- /dev/null +++ b/openspec/changes/recover-rate-limited-on-confirmed-reset/tasks.md @@ -0,0 +1,32 @@ +## 1. Reset Evidence + +- [x] 1.1 Use the account/window/time-scoped usage-history lookup to require a monthly baseline recorded strictly after the block and matching the blocked deadline within tolerance. +- [x] 1.2 Resolve one canonical monthly reset-evidence tuple from an anchored current refresh pair or by scanning only adjacent persisted pairs at or after the matching baseline with the existing temporal reset predicate, including fallback from an unanchored current transition. +- [x] 1.3 Feed the resolved tuple to both blocked-status recovery and selected-window warm-up without widening the scheduler's selected-account scope. + +## 2. Safe Status Recovery + +- [x] 2.1 Add the Free monthly early-recovery predicate with the future-marker, matching-baseline, 30-second floor, post-block timestamp, and current availability gates. +- [x] 2.2 Run marker-guarded recovery before warm-up, clearing the reason and both block markers only when the status/reason/reset/blocked compare-and-set succeeds. +- [x] 2.3 Preserve ordinary recovery for elapsed cooldowns and reject early recovery for generic Retry-After, mismatched or jitter-only transitions, exhausted latest usage, non-Free accounts, and unsafe statuses. + +## 3. Active-Only Warm-up + +- [x] 3.1 Make reset-confirmed candidates eligible after every real selected-window reset regardless of previous usage or the legacy exhaustion-threshold setting while retaining post-reset availability, opt-in, and jitter gates. +- [x] 3.2 Restrict both warm-up candidate evaluation and the sender's fresh preflight to `active` accounts so a failed CAS or later re-block prevents upstream traffic. +- [x] 3.3 Reuse the recovered monthly reset tuple with the existing atomic account/window/reset attempt claim so restart recovery cannot duplicate warm-up. +- [x] 3.4 Keep `limit_warmup_cooldown_seconds` scoped to staggered idle candidates and record the follow-up removal plan for the now-unused exhaustion-threshold setting. + +## 4. Regression Coverage + +- [x] 4.1 Add a scheduler regression for a Free account stuck behind a future legacy deadline that recovers and warms after a confirmed monthly reset. +- [x] 4.2 Cover recovery from persisted transition history after restart, including an ineligible matching row exactly at `blocked_at` and an unanchored current transition before fallback to a later valid baseline, and prove the same monthly reset tuple is deduplicated. +- [x] 4.3 Cover the 30-second floor, missing or mismatched markers, stale/pre-block evidence, timestamp jitter, exhausted after/latest usage, generic Retry-After cooldown, and a Plus account with primary usage at `100%`. +- [x] 4.4 Cover compare-and-set contention and re-block-after-candidate races, proving neither stale recovery nor warm-up traffic occurs. +- [x] 4.5 Cover active-only warm-up plus a non-exhausted-to-available real reset to prove prior exhaustion is no longer required. + +## 5. Verification + +- [x] 5.1 Run focused scheduler, usage-repository, recoverable-status, and limit-warm-up unit/integration tests. +- [x] 5.2 Run the proportional regression suite plus Ruff, formatting, type checking, and `git diff --check`. +- [x] 5.3 Validate OpenSpec strictly and semantically verify every changed requirement and scenario against implementation and tests. diff --git a/openspec/changes/recover-repeated-clean-close/proposal.md b/openspec/changes/recover-repeated-clean-close/proposal.md deleted file mode 100644 index 6e9bc35b2f..0000000000 --- a/openspec/changes/recover-repeated-clean-close/proposal.md +++ /dev/null @@ -1,101 +0,0 @@ -## Why - -The HTTP Responses bridge currently opens its retry circuit after a clean -upstream WebSocket close even when the replacement socket also closes before -producing any response event. A downstream idle-recovery task can also replace -the upstream socket without restarting its reader. Closing the old socket then -wakes that stale reader, which misclassifies the proxy-initiated close as an -upstream failure and retires work already moved to the replacement socket. -Together these behaviors make a transient handoff issue visible as a reconnect -loop and require the Codex client to be restarted. - -## What Changes - -- Permit one additional pre-visible replay when the replacement upstream - WebSocket closes cleanly before any response event. -- Add bounded, configurable jitter before that additional replay to avoid - synchronized reconnects. -- Emit a dedicated diagnostic event for the additional clean-close replay. -- Keep the allowance hard-capped at one and preserve all existing no-replay - behavior after downstream-visible output or continuity-sensitive state. -- Treat an unclassified WebSocket receive error after dispatch as ambiguous - delivery and fail closed without reconnecting or resending the request. -- Distinguish a generic receive failure on an idle bridge from an active - `stream_incomplete` failure without logging request content or identifiers. -- When recovery is initiated outside the upstream reader, cancel and await the - old reader before closing its socket, then start exactly one reader for the - replacement socket. -- Keep the shared session live while the replacement socket opens so concurrent - idle pruning cannot evict and fail its pending response during the handoff. -- Start silent pre-response recovery with enough headroom to reconnect before - the downstream client's request timeout boundary. -- Do not let a proxy-initiated close of a superseded socket retire pending work - on the replacement socket or increment the retry circuit. -- Detect a stuck pre-response gate from the absence of upstream activity and - response creation, rather than admission flags alone. Give requests with a - prior continuity anchor a bounded two-threshold grace period, and emit - diagnostic state when the watchdog skips a candidate. -- Detect a socket that is already closed before the transport adapter invokes - its send primitive, reconnect once, and dispatch the request exactly once on - the replacement socket. Preserve fail-closed handling for every exception - raised after the send primitive is invoked because delivery is then - ambiguous. -- Return one atomic retry-circuit decision containing the failure class and - remaining cooldown. Use it for both the HTTP `Retry-After` header and an - accurate operator-facing message instead of describing every WebSocket - failure as a timeout. -- Treat an upstream rejection of a proxy-injected `previous_response_id`, or - an explicit client anchor accompanied by the same immutable owner-bound - complete-context proof, as a stale continuity anchor rather than a reason - to inject the same rejected identifier into another physical WebSocket. - Replay immediately only when the proof covers the complete client context; - otherwise quarantine the logical key and recover on the next complete - client resend while keeping delta-only requests fail-closed. -- Recognize a narrowly validated completed Codex `agent_message` as the - retained-output boundary in an exact owner-bound full resend. Keep this - recovery pinned to the existing owning account, require the response-owned - `amsg_` identity plus canonical agent paths, timestamp/turn metadata, and one - self-contained text part, and continue rejecting malformed or client-shaped - lookalikes. -- Recover one additional abandoned-tool boundary only after upstream rejects - the exact durable anchor before emitting any response event: the durable - manifest must be nonempty, none of its call ids may occur anywhere in the - exact client resend, and the fresh suffix must begin with one canonical - response-owned `agent_message` followed only by new user input. This proves - the client never accepted or executed the orphan call while keeping ordinary - pending-tool replay fail-closed. -- For that exception only, recognize and omit strictly shaped response-owned - reasoning bookkeeping immediately before the new agent boundary, omit - already sealed historical agent deliveries from the stored prefix, and send - the one recovery attempt from the same verified projection. Do not serialize - stale reasoning identifiers or historical agent identifiers into the fresh - request. -- Recognize Codex's exact persisted user-message bookkeeping (`msg_` UUID plus - `turn_id` and finite `create_time`) in that proved suffix. Validate the raw - shape before removing the response-owned message id and timestamp from the - one-shot projection; malformed lookalikes remain fail-closed. - -## Impact - -- Repeated clean handoffs can recover transparently without an immediate - terminal circuit-open response. -- The retry remains bounded and does not create an unbounded replay loop. -- Reader ownership follows the active socket across idle recovery, preventing - locally generated close frames from being counted as upstream instability. -- Adds the `http_bridge_retry_circuits` durable table and migration so retry - cooldown state survives cross-replica clean-close and incomplete-stream - failures. -- Adds a forward-only request-usage rollup repair migration for deployments - already stamped at the previous merge head, so changing migration ancestry - cannot leave startup schema-drift checks failing. -- Clients receive a consistent integer cooldown hint and failure-class copy; - clients remain responsible for honoring `Retry-After` rather than exhausting - their local retry budget inside the advertised cooldown. -- A stale durable response anchor can no longer strand a long-lived desktop - task in an `Invalid previous_response_id` / reconnect / cooldown loop. -- Long-lived multi-agent Codex tasks whose latest completed turn ends in a - sub-agent delivery can use the same one-shot, owner-bound stale-anchor - recovery without weakening cross-account replay eligibility. -- A durable pending call that was never delivered to the client no longer - strands that task after the corresponding upstream anchor has disappeared; - the exception remains same-owner, stale-anchor-triggered, and one-shot. diff --git a/openspec/changes/recover-repeated-clean-close/specs/responses-api-compat/spec.md b/openspec/changes/recover-repeated-clean-close/specs/responses-api-compat/spec.md deleted file mode 100644 index 6e46df7ed6..0000000000 --- a/openspec/changes/recover-repeated-clean-close/specs/responses-api-compat/spec.md +++ /dev/null @@ -1,587 +0,0 @@ -## MODIFIED Requirements - -### Requirement: Clean upstream close before any response event fails fast - -When the HTTP Responses bridge observes an upstream WebSocket close with `close_code = 1000` before any `response.*` event has been surfaced for the pending request, the proxy MUST preserve its existing pre-visible replay guards. -If the request has already used exactly one eligible pre-visible -replay and the replacement upstream WebSocket also closes cleanly before any -response event, the proxy MAY perform exactly one additional replay. The -additional replay MUST be hard-capped at one per request, and the configured -maximum MUST NOT raise that cap. - -The proxy MUST NOT replay after downstream-visible output, after a terminal -response event, or when continuity-sensitive request state makes replay unsafe. -Before the additional replay, the proxy MAY sleep for bounded configured -jitter. The proxy MUST emit a dedicated low-cardinality diagnostic event for -the additional replay. - -When a downstream HTTP stream task initiates pre-response recovery while the -upstream reader is blocked on the superseded socket, the proxy MUST cancel and -await that reader before locally closing the socket. It MUST then start exactly -one reader for the replacement socket. A close caused by replacing the socket -MUST NOT be recorded as an upstream clean-close failure, MUST NOT increment the -retry circuit, and MUST NOT retire pending work moved to the replacement. The -cancelled reader's socket-generation finalizer MUST NOT leave the shared session -marked closed while the replacement socket is being selected or opened, so idle -pruning MUST NOT evict the handoff in progress. - -The default pre-response idle-recovery window MUST leave bounded headroom -before the downstream client's request timeout. With the default ten-second -keepalive interval, the proxy MUST initiate eligible recovery after no more -than six silent intervals so replacement connection and first output can occur -before a 120-second client deadline. - -The stuck pre-response watchdog MUST judge staleness using elapsed time since -the last upstream activity and the absence of a response identifier or -`response.created` latency, not admission flags alone. A request with a prior -continuity anchor MUST receive at most two retire-thresholds of grace before -being considered stale. When the watchdog skips a candidate, it MUST emit a -low-cardinality diagnostic containing the session-closed state, candidate -count, and pending-state verdicts. - -#### Scenario: clean close before response.created is not retried - -- **GIVEN** an HTTP bridge request is not eligible under the pre-visible replay guards -- **WHEN** upstream closes the bridge with `close_code = 1000` before any - `response.*` event for the pending request -- **THEN** the proxy returns HTTP 502 through the existing rejected-input path -- **AND** does not transparently replay the request - -#### Scenario: clean close before response output receives one bounded additional replay - -- **GIVEN** an HTTP bridge request has no surfaced `response.*` events -- **AND** its first pre-visible replay has already been used -- **WHEN** the replacement upstream WebSocket closes with code `1000` -- **THEN** the proxy performs one additional pre-visible replay -- **AND** the request replay count increases by one -- **AND** the proxy emits a `retry_precreated_clean_close` diagnostic event - -#### Scenario: repeated clean closes do not create an unbounded replay loop - -- **GIVEN** the additional clean-close replay has already been used -- **WHEN** another upstream WebSocket closes cleanly before response output -- **THEN** the proxy does not replay the request again -- **AND** the existing terminal or circuit handling is used - -#### Scenario: visible output still prevents clean-close replay - -- **GIVEN** the pending request has surfaced any response event downstream -- **WHEN** the upstream WebSocket closes with code `1000` -- **THEN** the proxy does not replay the request - -#### Scenario: clean-close retry jitter is bounded - -- **GIVEN** clean-close retry jitter is configured -- **WHEN** the additional clean-close replay is scheduled -- **THEN** the delay is no greater than the configured jitter maximum -- **AND** the hard replay cap remains one regardless of the configured value - -#### Scenario: downstream idle recovery transfers reader ownership - -- **GIVEN** the upstream reader is blocked on the current bridge socket -- **AND** the downstream HTTP stream task initiates eligible pre-response recovery -- **WHEN** the bridge replaces the upstream socket -- **THEN** the old reader is cancelled and awaited before its socket is closed -- **AND** the shared session remains live while the replacement socket opens -- **AND** idle pruning retains the registered session while the handoff is in progress -- **AND** exactly one reader owns the replacement socket -- **AND** the local close does not open or increment the retry circuit -- **AND** pending work remains attached to the replacement session - -#### Scenario: silent pre-response recovery precedes the client timeout - -- **GIVEN** the upstream has produced no response event -- **AND** the default ten-second keepalive interval is active -- **WHEN** six silent intervals elapse -- **THEN** the proxy initiates eligible pre-response recovery -- **AND** at least sixty seconds remain before a 120-second client request timeout - -#### Scenario: anchored stuck-gate grace is bounded - -- **GIVEN** a pending HTTP bridge request has a prior continuity anchor -- **AND** no response identifier or `response.created` latency has been recorded -- **WHEN** less than two retire thresholds have elapsed since the gate began waiting -- **THEN** the watchdog does not classify the request as stale -- **WHEN** two retire thresholds elapse without upstream activity -- **THEN** the watchdog may classify the request as stale - -#### Scenario: upstream activity resolves admission-flag ambiguity - -- **GIVEN** a pending request has not acquired the response-created gate -- **AND** upstream activity has not produced a response identifier or `response.created` -- **WHEN** the staleness threshold elapses -- **THEN** the watchdog classifies the request as stale -- **AND** emits pending-state verdict inputs when it skips a watchdog pass - -### Requirement: Durable retry-circuit state protects repeated hard-affinity failures - -For a hard-affinity bridge key, the proxy MUST scope retry-circuit state by -affinity kind, affinity key, and API-key scope (using a stable anonymous scope -when no API key is present). The proxy MUST record only the documented -pre-response failure classes (`stream_incomplete`, `clean_close`, and -`stream_idle_timeout`). - -The default circuit MUST open after two consecutive recorded failures. Once -open, it MUST suppress pre-created replay until the persisted cooldown expires, -using exponential backoff from sixty seconds up to ten minutes. Clean-close -failures MUST cap their cooldown at thirty seconds. The proxy MUST persist -failure count, cooldown deadline, last failure detail, and update time in the -`http_bridge_retry_circuits` table and MUST merge conflict updates so concurrent -replicas cannot shorten an existing cooldown. - -The clean-close retry jitter maximum MUST be read from the -`http_responses_session_bridge_clean_close_retry_jitter_max_seconds` runtime -setting and MUST be bounded to the inclusive range 0–30 seconds. - -The proxy MUST evict process-local circuit entries and their loaded/persisted -markers after one hour without use, independently of durable-row cleanup, so -one-shot hard-affinity keys cannot grow the worker's memory without bound. - -Before every hard-affinity retry decision, the proxy MUST refresh the durable -row so a cooldown opened by another replica is observed even when this process -has already loaded the key. A durable lookup or persistence failure MUST NOT -crash the request; the proxy MUST continue using available local state and -record the failure for observability. Rows older than one hour MUST be treated -as expired and removed. A successful terminal response MUST clear the local -and durable circuit state. - -When a request is suppressed, the proxy MUST obtain the retry decision from one -atomic process-local snapshot after refreshing durable state. The decision MUST -include the normalized last failure class and remaining suppression interval. -State below the configured failure threshold MUST NOT suppress a request or -enter half-open mode, even if a stale durable read carries an older cooldown -timestamp. Once one half-open probe is admitted, its own streaming path MUST -NOT treat that probe lease as an active cooldown; the lease only fences later -submissions. -The HTTP response MUST expose the ceiling of that interval as `Retry-After`, and -the error message MUST name the recorded class (`stream_incomplete`, -`clean_close`, or `stream_idle_timeout`) accurately rather than calling every -failure a timeout. The same integer interval MUST be used in the message and -the response metadata. A client MAY retry after that interval; retrying earlier -MUST remain suppressed without dispatching another upstream request. - -#### Scenario: the second hard-key failure opens a durable circuit - -- **GIVEN** a hard-affinity key has one recorded pre-response failure -- **WHEN** a second eligible failure is recorded -- **THEN** the proxy opens the retry circuit -- **AND** persists at least two consecutive failures and a cooldown deadline -- **AND** subsequent pre-created replay is suppressed until that deadline - -#### Scenario: retry decisions observe a cooldown opened by another replica - -- **GIVEN** this replica previously looked up a hard-affinity key with no row -- **AND** another replica persists an open cooldown for that same key and API-key scope -- **WHEN** this replica evaluates the next pre-created retry -- **THEN** it refreshes durable state before deciding -- **AND** suppresses the retry for the persisted cooldown - -#### Scenario: circuit state remains isolated by key and API-key scope - -- **GIVEN** one hard-affinity key has an open circuit -- **WHEN** a different affinity key or API-key scope evaluates a retry -- **THEN** that request is not suppressed by the first key's circuit - -#### Scenario: durable circuit lookup failure does not fail the request - -- **GIVEN** durable retry-circuit lookup or persistence is unavailable -- **WHEN** the proxy evaluates or records a retry-circuit event -- **THEN** the request continues using any available local circuit state -- **AND** the failure is logged and exposed through retry-circuit observability - -#### Scenario: incomplete streams produce an accurate cooldown response - -- **GIVEN** a hard-affinity circuit is open with `last_detail = stream_incomplete` -- **WHEN** a new HTTP request is suppressed -- **THEN** the proxy returns HTTP 503 without dispatching upstream -- **AND** the message identifies repeated incomplete WebSocket streams -- **AND** `Retry-After` equals the ceiling of the same remaining interval named in the message - -#### Scenario: terminal success clears the circuit - -- **GIVEN** a hard-affinity circuit has recorded prior failures -- **WHEN** a request on that bridge reaches `response.completed` -- **THEN** local and durable circuit state are cleared -- **AND** the next request is not suppressed by the settled failures - -#### Scenario: stale sub-threshold state is not an open circuit - -- **GIVEN** a local or durable retry state has fewer failures than the open threshold -- **AND** it carries a stale cooldown timestamp -- **WHEN** the proxy evaluates admission and streaming startup -- **THEN** both paths admit the request without a cooldown response -- **AND** the state does not enter half-open mode - -#### Scenario: admitted half-open probe does not suppress itself - -- **GIVEN** an open circuit cooldown expires -- **WHEN** one half-open probe is admitted -- **THEN** later submissions remain fenced by the half-open lease -- **AND** the admitted probe's own stream startup proceeds without a synthetic 503 - -### Requirement: Proven pre-dispatch closes recover without duplicate turns - -The upstream WebSocket adapter MUST distinguish a transport that is already in -a terminal closed state before the adapter invokes its underlying send -primitive. That condition MUST use a dedicated account-neutral error class and -MUST prove that the application send primitive was not called. - -For an HTTP Responses request with no upstream response event, the bridge MAY -replace the closed socket once and dispatch the request on the replacement. A -continuity-bound request MUST still satisfy the existing proof-gated fresh-body -replay contract before its anchor can be removed or its account can change. The -replacement attempt MUST consume the request's single fresh-upstream retry -allowance. - -A complete fresh-body replay proof and an account-neutral replay proof are -independent. If the retained fresh body still contains any account-scoped -identifier, the replacement MUST remain on the owning account even after the -continuity anchor is safely removed. Cross-account replacement is permitted -only when the retained body separately satisfies the account-neutral replay -contract and the replacement also uses a new account-neutral logical key with -all prior session and turn-state affinity headers removed. A hard-affinity -session that has not performed that explicit fork MUST remain on its owning -account even when its retained body is account neutral. A physical-socket-only -replacement that preserves the current logical key and reconnect handshake -MUST remain on the owning account for soft-affinity keys as well. It MUST NOT -treat body neutrality alone as proof that old turn state may cross accounts. - -Any error raised after the underlying send primitive is invoked MUST remain an -ambiguous send failure. The proxy MUST NOT reconnect and resend from that path, -even when the exception reports a clean WebSocket close, because the complete -frame may already have crossed the kernel boundary. - -For an exact stored-prefix full resend owned by the same durable or live -session, a completed Codex inter-agent delivery MAY serve as the retained -prior-output boundary when it has the exact response-owned `agent_message` -shape: an `amsg_` UUID identity, distinct canonical absolute agent paths for -author and recipient, exact `turn_id` plus finite `create_time` metadata, and -one self-contained `input_text` content part. The proof MUST reject missing, -extra, malformed, reordered, or client-shaped fields. This shape MUST remain -owner-bound and MUST NOT by itself make a request eligible for account-neutral -reallocation. The proof MUST additionally bind a persisted tool-call manifest -that is present and exactly empty; a missing manifest or any unsettled call -MUST reject the inter-agent boundary. A matching request MAY remove only the -proxy-injected stale `previous_response_id`, replay the complete context once -on the same account, and publish a replacement anchor only after -`response.completed`. - -For the abandoned-pending-call exception, the new suffix MAY contain zero or -more exact response-owned reasoning items immediately before the canonical -`agent_message`. Each such item MUST carry an `rs_` identity, encrypted -content, a structured summary, and exact internal turn provenance; malformed -reasoning or reasoning after the agent boundary MUST fail closed. The proof -projection MUST omit those reasoning items and any exact historical -response-owned agent deliveries already covered by the immutable stored-prefix -fingerprint. The actual one-shot unanchored retry MUST use that same projected -input. It MUST NOT resend omitted response bookkeeping or any pending call id. -User follow-ups in this proved suffix MAY carry Codex's exact persisted message -bookkeeping: a `msg_` UUID, `user` role, one self-contained `input_text`, and -exact `turn_id` plus finite nonnegative `create_time` metadata. The proof MUST -validate that complete raw shape before projection. The projected replay MUST -remove the response-owned message id and `create_time` while preserving the -validated `turn_id`; missing, extra, malformed, or non-finite fields MUST fail -closed. - -Rejected-proof observability MUST classify only known string-valued `type` and -`role` fields. Non-string or otherwise malformed values MUST be labeled as an -opaque `other` shape without logging their content and MUST NOT turn the -fail-closed bridge response into an internal server error. Pre-bridge request -inspection, including file-reference extraction, MUST apply the same typed -classification and MUST NOT fail on non-string item types. - -#### Scenario: socket already closed before send recovers once - -- **GIVEN** an HTTP bridge socket is already closed before `response.create` dispatch -- **AND** the request is eligible for fresh-upstream recovery -- **WHEN** the adapter is asked to send the request -- **THEN** the adapter does not invoke the closed socket's send primitive -- **AND** the bridge opens one replacement socket -- **AND** the request is dispatched exactly once on that replacement - -#### Scenario: a second pre-dispatch close does not loop - -- **GIVEN** the request consumed its one fresh-upstream recovery allowance -- **WHEN** the replacement socket is also already closed before send -- **THEN** the proxy does not open a third socket -- **AND** the request fails through the existing terminal transport path - -#### Scenario: completed inter-agent delivery recovers an exact long-session resend - -- **GIVEN** a live or durable session stores an exact completed input prefix -- **AND** no tool-call manifest remains pending -- **AND** the full resend suffix contains response-owned reasoning, one - canonical completed `agent_message`, and one or more later user messages -- **WHEN** upstream rejects the proxy-injected stale `previous_response_id` - before any response event -- **THEN** the proxy sends the original complete request exactly once without - `previous_response_id` on the same owning account -- **AND** preserves the canonical inter-agent message in that request -- **AND** publishes a new anchor only after successful completion - -#### Scenario: malformed inter-agent lookalikes remain anchored - -- **GIVEN** a full-resend-shaped suffix contains an `agent_message` with a - missing or non-`amsg_` identity, invalid agent path, non-finite timestamp, - extra metadata, hosted/account-scoped content, or a user item before it -- **WHEN** stale-anchor recovery is evaluated -- **THEN** the inter-agent item does not satisfy retained-output proof -- **AND** the proxy keeps the continuity request owner-bound and fail-closed -- **AND** it does not dispatch an unanchored replay - -#### Scenario: live, represented, or unknown tool state rejects an inter-agent boundary - -- **GIVEN** a full resend suffix contains a canonical completed `agent_message` -- **AND** the durable or live owner has an unavailable persisted tool-call - manifest, the anchor has not been explicitly rejected, or the exact client - resend contains any pending call id -- **WHEN** stale-anchor recovery is evaluated -- **THEN** the inter-agent item does not satisfy retained-output proof -- **AND** the existing continuity anchor remains attached - -#### Scenario: rejected orphan pending call yields to a later inter-agent boundary - -- **GIVEN** the durable owner has a nonempty pending client-side tool-call manifest -- **AND** an exact-prefix full resend contains none of those pending call ids -- **AND** its fresh suffix begins with one canonical response-owned - `agent_message` (optionally preceded only by exact response-owned reasoning) - followed only by one or more new user inputs -- **WHEN** upstream rejects the exact durable `previous_response_id` before - emitting any response event -- **THEN** the bridge treats the pending call as never accepted or executed by - the client -- **AND** retries the complete request exactly once without the stale anchor - on the same owning account -- **AND** does not synthesize the orphan call or output into the unanchored replay -- **AND** omits the response-owned reasoning and sealed historical agent - deliveries from the one-shot recovery payload -- **AND** publishes a replacement anchor only after successful completion - -#### Scenario: persisted user bookkeeping is validated then normalized - -- **GIVEN** the proved abandoned-pending suffix contains one or more persisted - user messages with canonical `msg_` UUIDs and exact turn/timestamp metadata -- **WHEN** the bridge builds the one-shot same-owner unanchored projection -- **THEN** it first validates every raw user-message field and content part -- **AND** removes each response-owned message id and `create_time` -- **AND** preserves only the validated `turn_id` metadata in the replay -- **AND** any malformed id, timestamp, metadata, content, or extra field rejects - the proof before dispatch - -#### Scenario: malformed diagnostic fields remain fail-closed - -- **GIVEN** a rejected full-resend suffix contains a non-string `type` or - `role` value -- **WHEN** the bridge emits its bounded suffix-shape diagnostic -- **THEN** it records only the `other` classification -- **AND** does not expose the malformed value -- **AND** does not return HTTP 500 from the diagnostic path - -#### Scenario: complete but account-scoped fresh body stays on its owner - -- **GIVEN** a continuity-bound request has a complete fresh-body replay proof -- **AND** that fresh body still contains an account-scoped conversation, - prompt, hosted input item, or file identifier -- **WHEN** the original socket is proven closed before send -- **THEN** the bridge may remove the continuity anchor for the one-shot retry -- **AND** the replacement remains bound to the original owning account - -#### Scenario: only an account-neutral fresh body may change accounts - -- **GIVEN** a continuity-bound request has both a complete fresh-body replay - proof and a separate account-neutral replay proof -- **WHEN** the original socket is proven closed before send -- **AND** the bridge establishes a new account-neutral logical key and strips - all prior session and turn-state affinity headers -- **THEN** the one-shot replacement may select another eligible account - -#### Scenario: hard affinity stays on its owner without an explicit fork - -- **GIVEN** a continuity-bound request has an account-neutral fresh-body proof -- **AND** its current bridge still has a hard session or turn-state affinity key -- **WHEN** the original socket is proven closed before send -- **THEN** the replacement remains on the original owning account -- **AND** no old session or turn-state identifier is sent to another account - -#### Scenario: soft affinity stays on its owner without an explicit fork - -- **GIVEN** a continuity-bound request has an account-neutral fresh-body proof -- **AND** its current bridge has a soft prompt-cache or sticky affinity key -- **WHEN** the original socket is proven closed before send -- **AND** recovery replaces only the physical socket while retaining the - current logical key and reconnect handshake -- **THEN** the replacement remains on the original owning account -- **AND** no old session or turn-state identifier is sent to another account - -#### Scenario: post-dispatch close remains non-replayable - -- **GIVEN** the adapter invoked the underlying send primitive -- **WHEN** that primitive raises a clean-close or transport exception -- **THEN** delivery is treated as ambiguous -- **AND** the proxy does not dispatch the request on another socket - -#### Scenario: unclassified receive error after dispatch remains non-replayable - -- **GIVEN** the adapter has successfully invoked the underlying send primitive -- **AND** the bridge has not observed `response.created` or another response - event -- **WHEN** the WebSocket reader reports a generic transport error without a - classified protocol error code -- **THEN** upstream acceptance is treated as ambiguous -- **AND** the proxy does not reconnect or resend the request -- **AND** it terminally settles the affected request and retires the bridge so - a later client request can create a fresh session - -#### Scenario: explicit clean close keeps its bounded recovery owner - -- **GIVEN** a pre-visible request satisfies the existing clean-close replay - guards -- **WHEN** the reader reports an explicitly classified clean close -- **THEN** the clean-close owner may perform its single bounded recovery -- **AND** the generic receive-error fail-closed rule does not consume or widen - that recovery allowance - -#### Scenario: idle generic receive failure is not an active stream failure - -- **GIVEN** a bridge has no pending request and no create-admission waiter -- **WHEN** its reader reports a generic transport error -- **THEN** the bridge retires its stale aliases without reconnecting or opening - a retry circuit -- **AND** it emits a content-free informational retirement diagnostic rather - than an active `stream_incomplete` warning - -### Requirement: Rejected proxy continuity anchors recover without context loss - -When the upstream explicitly rejects a proxy-injected `previous_response_id` before any response event is observed, the bridge MUST NOT inject the -same rejected identifier into another physical WebSocket. A request that has a -request-bound immutable durable proof covering its complete unanchored input -MAY bypass the rejected anchor and replay exactly once, without that anchor, on -the same account. The bridge MUST retain the previous durable anchor until the -replacement reaches terminal completion and atomically publishes its new -checkpoint. - -When that proof is absent, the bridge MUST quarantine the logical session key, -retire the rejected physical bridge while preserving the durable lease, and -return a stable non-retryable-same-contract error. A later full-resend-shaped -client request MUST take a fresh unanchored path only when its request-bound -durable proof exactly matches the current durable owner and proves complete -conversation context. A merely full-resend-shaped request and a delta-only -request MUST keep the durable anchor and fail closed; the bridge MUST NOT -silently discard prior conversation context. A completed response MUST clear -the bounded quarantine. - -Client-supplied `previous_response_id` values MUST NOT be cleared or replayed -unanchored by this recovery path unless the same request carries a -request-bound immutable durable proof that matches the current durable owner, -the exact stored input prefix, pending-tool manifest, and complete fresh -suffix. When that proof exists and upstream rejects the explicit anchor before -any response event, the bridge MAY quarantine the rejected physical session -and replay the proved complete request exactly once without the anchor on the -same owning account. An incomplete, delta-only, owner-conflicting, or -unproved explicit-anchor request remains fail-closed. - -A nonempty durable pending-tool manifest MAY satisfy this stale-anchor recovery -only when the exact client resend contains none of its call ids, its fresh -suffix begins with one canonical response-owned `agent_message` and then only -new user input, and upstream first rejects the exact anchor before emitting any -response event. This narrow condition proves that the client advanced without -accepting or executing the orphan call. It MUST NOT make the request eligible -for proactive, owner-unavailable, cross-account, or pre-rejection fresh replay. - -#### Scenario: proved complete request recovers in the same turn - -- **GIVEN** the upstream rejects a proxy-injected durable response anchor before any response event -- **AND** immutable durable evidence proves the untrimmed request contains the complete conversation context -- **WHEN** the bridge performs local recovery -- **THEN** it retains the previous durable anchor until replacement completion -- **AND** replays the complete request exactly once without an anchor on the same account -- **AND** terminal completion atomically replaces the durable anchor -- **AND** a failed replacement leaves the previous durable anchor available for a later verified retry - -#### Scenario: unproved request quarantines and only a durably proved full resend recovers - -- **GIVEN** the upstream rejects a proxy-injected durable response anchor before any response event -- **AND** the current request lacks complete-context proof -- **WHEN** the bridge handles the rejection -- **THEN** it returns `previous_response_anchor_unrecoverable` without another upstream dispatch -- **AND** quarantines the logical key without clearing its durable anchor -- **WHEN** the client subsequently supplies a full-resend-shaped request whose immutable proof exactly matches the durable owner and complete context -- **THEN** the fresh bridge sends that request without the rejected anchor -- **AND** a terminal completion clears quarantine - -#### Scenario: delta-only and client-owned anchors remain fail-closed - -- **GIVEN** a client explicitly supplies `previous_response_id` -- **AND** its request omits the prior completed output or otherwise lacks the - immutable owner-bound complete-context proof -- **WHEN** upstream rejects that anchor before execution -- **THEN** the bridge does not clear the anchor or dispatch an unanchored copy -- **GIVEN** a quarantined key receives a delta payload, a full-resend-shaped - payload without exact durable completeness proof, or an unproved - client-supplied anchor -- **WHEN** recovery is evaluated -- **THEN** the proxy does not remove the anchor -- **AND** it does not issue an unanchored replay that could omit prior context - -#### Scenario: proved explicit stale anchor recovers once - -- **GIVEN** a client explicitly supplies `previous_response_id` -- **AND** the same request's immutable durable proof binds the current owner, - exact stored prefix, pending-tool manifest, and complete fresh suffix -- **WHEN** upstream rejects that anchor before any response event -- **THEN** the bridge retires the rejected physical session -- **AND** sends the proved complete request once without - `previous_response_id` on the same owning account -- **AND** a failed replacement cannot authorize a second duplicate replay - -### Requirement: Upstream websocket drops penalize affected accounts - -When an upstream websocket closes while one or more streamed response requests are pending and have not reached a terminal event, the proxy MUST record a -transient upstream error for the account before signaling failure for those -pending requests, except when the close carries a classified process-wide -network failure, is a clean close (`close_code = 1000`) before any -`response.*` event, or carries the classified per-socket -`upstream_keepalive_timeout` transport error. Clean pre-response closes and -keepalive timeouts MUST remain account-neutral while using the bounded retry -and retry-circuit handling above. A classified process-wide network failure -MUST remain account neutral and use its network error code. For other closes, -the proxy MUST surface -`stream_incomplete` to affected pending requests. - -#### Scenario: websocket closes before pending responses complete - -- **GIVEN** a streamed response request is pending on an upstream websocket -- **AND** the direct downstream response has not emitted a numeric sequence, - or the request uses another transport -- **WHEN** the websocket closes before a terminal response event is observed -- **AND** the close is not an account-neutral clean pre-response close, - process-wide network failure, or upstream WebSocket liveness timeout -- **THEN** the pending request fails with `stream_incomplete` -- **AND** the account receives a transient upstream failure signal for routing - -#### Scenario: sequenced direct websocket closes before completion - -- **GIVEN** a direct Responses WebSocket request has successfully emitted a - finite integer `sequence_number` -- **WHEN** the upstream websocket closes before a terminal response event is observed -- **AND** the close is not an account-neutral clean pre-response close, - process-wide network failure, or upstream WebSocket liveness timeout -- **THEN** the request is recorded as failed with `stream_incomplete` -- **AND** no synthetic terminal frame is emitted under the active response id -- **AND** the downstream WebSocket closes with code 1011 -- **AND** the account receives a transient upstream failure signal for routing - -#### Scenario: websocket liveness timeout remains account neutral - -- **GIVEN** a streamed response request is pending on an upstream websocket -- **WHEN** its transport reports `upstream_websocket_liveness_timeout` -- **THEN** the pending request fails with that classified error code -- **AND** the account receives no failure-health signal -- **AND** the request is not transparently replayed - -#### Scenario: clean pre-response close does not penalize the account - -- **GIVEN** a hard-affinity HTTP bridge request is pending with no surfaced response event -- **WHEN** the upstream websocket closes cleanly before response output -- **THEN** the proxy records the clean-close retry-circuit outcome -- **AND** the selected account is not penalized diff --git a/openspec/changes/recover-repeated-clean-close/tasks.md b/openspec/changes/recover-repeated-clean-close/tasks.md deleted file mode 100644 index a5b90414b1..0000000000 --- a/openspec/changes/recover-repeated-clean-close/tasks.md +++ /dev/null @@ -1,46 +0,0 @@ -- [x] Add bounded clean-close replay settings with safe defaults. -- [x] Allow one additional clean-close replay only before visible output. -- [x] Add jitter and dedicated retry diagnostics. -- [x] Add regression coverage for the second replay and retry cap. -- [x] Restart the upstream reader when pre-response recovery is initiated by the downstream stream task. -- [x] Add regression coverage for old-reader cancellation and replacement-reader ownership. -- [x] Keep the shared session live across the cancelled reader's socket-generation finalizer. -- [x] Add regression coverage for concurrent pruning during reader handoff. -- [x] Move the default pre-response recovery threshold ahead of the client timeout boundary. -- [x] Bound anchored stuck-gate grace and evaluate staleness from upstream activity/response creation. -- [x] Emit stuck-watchdog skip diagnostics with pending-state verdict inputs. -- [x] Add a forward-only repair for databases stamped before request-usage rollups were connected to the merge head. -- [x] Validate the OpenSpec change and run the focused and full test suites. -- [x] Build and deploy the validated image, then verify production health and logs. -- [x] Classify an already-closed transport before invoking its send primitive. -- [x] Reconnect once and send exactly once after a proven pre-dispatch close. -- [x] Require a separate account-neutral proof before a fresh-body retry may change accounts. -- [x] Keep physical-socket-only recovery pinned to the owning account until an explicit account-neutral logical fork strips prior affinity state. -- [x] Keep post-dispatch send failures ambiguous and non-replayable. -- [x] Keep unclassified post-dispatch receive failures ambiguous and - non-replayable while preserving the bounded clean-close and typed - closed-before-send recovery owners. -- [x] Classify an idle generic receive failure as content-free bridge - retirement rather than an active `stream_incomplete` failure. -- [x] Add regressions proving an unobserved upstream acceptance cannot cause a - second irreversible execution, while clean-close and typed pre-send recovery - remain bounded and available. -- [x] Return an atomic retry-circuit decision with failure detail and remaining cooldown. -- [x] Render accurate cooldown copy and one matching integer `Retry-After` value on every HTTP 503 cooldown path. -- [x] Keep sub-threshold state out of half-open mode and prevent an admitted probe from suppressing its own stream. -- [x] Add regressions for pre-dispatch recovery, duplicate prevention, circuit opening/reset, and cooldown rendering. -- [x] Quarantine an explicitly rejected proxy-injected anchor and recover the next complete full resend unanchored. -- [x] Keep incomplete/delta-only and client-owned continuity fail-closed. -- [x] Recover a rejected explicit client anchor only when the same immutable durable proof binds the exact owner and complete resend; keep incomplete explicit-anchor requests fail-closed. -- [x] Validate the amended OpenSpec and run formatting, lint, type, focused, unit, and bridge regression gates. -- [x] Build and deploy the exact merged source image, then verify production rollout, health, and recovery observability. -- [x] Treat only a canonical response-owned `agent_message` as a completed output boundary for exact-prefix owner-bound recovery. -- [x] Preserve `agent_message` identifiers during proof classification while keeping the replay ineligible for account-neutral reallocation. -- [x] Add negative shape tests and an end-to-end stale-anchor regression matching reasoning + agent message + repeated user follow-ups. -- [x] Emit content-free proof-rejection shape diagnostics for future long-session incidents. -- [x] Require an explicitly empty persisted tool manifest for inter-agent output boundaries and keep malformed diagnostic fields fail-closed. -- [x] Add a stale-anchor-only proof for an undelivered pending call followed by a canonical inter-agent boundary; keep pending ids, malformed suffixes, and pre-rejection replay fail-closed. -- [x] Project exact pre-boundary reasoning and sealed historical agent deliveries out of both the proof and the one-shot recovery payload; reject malformed or reordered reasoning. -- [x] Validate exact persisted `msg_` user bookkeeping and strip only its response-owned id/create-time fields from the proved one-shot projection; reject malformed lookalikes. -- [x] Validate the amended OpenSpec and rerun focused, unit, integration, lint, type, architecture, and diff gates. -- [ ] Build and deploy the exact merged source image, then verify the previously failing long task and both operator windows remain healthy. diff --git a/openspec/changes/recover-store-context-stale-anchor/.openspec.yaml b/openspec/changes/recover-store-context-stale-anchor/.openspec.yaml deleted file mode 100644 index 6529e830bb..0000000000 --- a/openspec/changes/recover-store-context-stale-anchor/.openspec.yaml +++ /dev/null @@ -1,2 +0,0 @@ -schema: spec-driven -created: 2026-08-22 diff --git a/openspec/changes/recover-store-context-stale-anchor/proposal.md b/openspec/changes/recover-store-context-stale-anchor/proposal.md deleted file mode 100644 index eac554e737..0000000000 --- a/openspec/changes/recover-store-context-stale-anchor/proposal.md +++ /dev/null @@ -1,24 +0,0 @@ -## Why - -The HTTP Responses bridge can receive a complete conversation input, verify -that its prefix exactly matches the live session checkpoint, trim that prefix, -and inject the checkpoint's `previous_response_id`. If upstream then rejects -that proxy-injected anchor before producing any response event, the current -recovery gate recognizes only durable full-resend proof. A live-session-only -request therefore returns `previous_response_anchor_unrecoverable` even though -the bridge still holds the exact complete request it verified and trimmed. -Repeated client retries carry the same stale continuity state and can loop. - -## What Changes - -- Mint an immutable request-local proof only when the bridge itself verifies - and applies an exact stored-prefix trim to a complete-context request. -- Bind the proof to the logical session key, account, response anchor, stored - prefix fingerprint, pending tool-call manifest, and full request fingerprint. -- Permit one same-request, same-account, unanchored replay only when upstream - rejects the proxy-injected anchor before any response event and the proof - still matches the live session. -- Keep incomplete inputs, client-owned anchors, post-event failures, account - changes, and repeated recovery attempts fail closed. -- Preserve the old durable checkpoint until the replacement response completes - and publishes its successor. diff --git a/openspec/changes/recover-store-context-stale-anchor/specs/responses-api-compat/spec.md b/openspec/changes/recover-store-context-stale-anchor/specs/responses-api-compat/spec.md deleted file mode 100644 index c9d7b98511..0000000000 --- a/openspec/changes/recover-store-context-stale-anchor/specs/responses-api-compat/spec.md +++ /dev/null @@ -1,39 +0,0 @@ -## ADDED Requirements - -### Requirement: Exact store-context trims can recover a rejected proxy anchor in place - -When the HTTP Responses bridge receives a complete unanchored request, proves -that its input prefix exactly matches the current live session checkpoint, and -then trims that prefix before injecting the live session's -`previous_response_id`, it MUST retain an immutable request-local completeness -proof. If upstream explicitly rejects that proxy-injected anchor before any -response event, the bridge MAY replay the retained complete request exactly -once without the anchor on the same account. The proof MUST bind the logical -session key, account, rejected response anchor, stored input count and -fingerprint, pending tool-call manifest, and complete request fingerprint. The -existing durable checkpoint MUST remain authoritative until the replacement -request completes. - -#### Scenario: request-local proof recovers an exact-prefix trim - -- **GIVEN** a live bridge completed a response and stored its exact input prefix -- **AND** the next unanchored client request contains that prefix, the prior response output or exact pending tool-call completion, and new input -- **AND** the bridge verifies the prefix, seals the complete request proof, trims the prefix, and injects its response anchor -- **WHEN** upstream rejects that anchor before emitting any response event -- **THEN** the bridge opens one replacement WebSocket on the same account -- **AND** sends the original complete request exactly once without `previous_response_id` -- **AND** a completed replacement atomically supersedes the old checkpoint - -#### Scenario: incomplete or changed request remains fail closed - -- **GIVEN** the stored prefix does not match, prior output or pending tool context is missing, any proof-bound session or request field changed, the anchor was client supplied, or an upstream response event was already observed -- **WHEN** stale-anchor recovery is evaluated -- **THEN** the bridge does not use request-local unanchored replay -- **AND** it preserves the existing durable checkpoint and established fail-closed error behavior - -#### Scenario: recovery remains single-dispatch and account-bound - -- **GIVEN** a request-local proof authorizes stale-anchor recovery -- **WHEN** the replacement dispatch is attempted -- **THEN** it is attempted at most once on the same owning account -- **AND** an ambiguous send or replacement failure does not authorize another replay or an account change diff --git a/openspec/changes/recover-store-context-stale-anchor/tasks.md b/openspec/changes/recover-store-context-stale-anchor/tasks.md deleted file mode 100644 index b41762c789..0000000000 --- a/openspec/changes/recover-store-context-stale-anchor/tasks.md +++ /dev/null @@ -1,7 +0,0 @@ -- [x] Define request-local store-context completeness proof semantics. -- [x] Add proof immutability and exact-binding unit coverage. -- [x] Add a production-shape bridge regression for exact-prefix trim followed by stale-anchor rejection. -- [x] Implement one same-account unanchored replay before any response event. -- [x] Run OpenSpec validation and focused formatting, lint, type, unit, and integration gates. -- [ ] Merge through required repository gates and publish the exact source image. -- [ ] Deploy through immutable GitOps and verify production continuity plus the affected long-running task. diff --git a/openspec/changes/reject-truncated-chat-completions/.openspec.yaml b/openspec/changes/reject-truncated-chat-completions/.openspec.yaml new file mode 100644 index 0000000000..41c30bab88 --- /dev/null +++ b/openspec/changes/reject-truncated-chat-completions/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-19 diff --git a/openspec/changes/reject-truncated-chat-completions/design.md b/openspec/changes/reject-truncated-chat-completions/design.md new file mode 100644 index 0000000000..b493ebc75f --- /dev/null +++ b/openspec/changes/reject-truncated-chat-completions/design.md @@ -0,0 +1,79 @@ +## Context + +`POST /v1/chat/completions` always asks the upstream Responses client for an SSE +iterator. The adapter converts that iterator either into Chat Completions SSE or +into a collected JSON response. Explicit terminal events are already mapped, +but natural iterator exhaustion is not tracked. This leaves the streaming +protocol unterminated and lets the collected path synthesize +`finish_reason=stop`. + +The public `/v1/responses` path already uses `upstream_stream_truncated`, +`server_error`, and HTTP 502 for EOF before a terminal event. The Chat adapter +must match those machine-consumed semantics without depending on the proxy API +module. + +## Goals / Non-Goals + +**Goals:** + +- Detect whether a terminal upstream Responses event was observed. +- Use the canonical `upstream_stream_truncated` code and `server_error` type. +- Finish streaming Chat errors with `data: [DONE]`. +- Keep explicit completion, incomplete, failure, and generator-cleanup behavior + unchanged. + +**Non-Goals:** + +- Change public `/v1/responses` normalization. +- Change retry, account selection, keepalive, or reservation policy. +- Convert explicit `response.incomplete` into a transport error. +- Refactor unrelated Chat payload or tool-call mapping. + +## Decisions + +### Decision: detect truncation at the Chat adapter boundary + +The adapter is the first layer that knows whether it observed a Chat-relevant +terminal Responses event. `stream_chat_chunks` will synthesize the error chunk +and `[DONE]` only when its mapped iterator exhausts without a terminal marker. +`collect_chat_completion` will return the equivalent error envelope before it +assembles a `ChatCompletion`. + +This keeps the behavior correct for both the subscription route and any other +caller of the adapter without changing the public Responses pipeline. + +### Decision: preserve canonical machine semantics + +The synthesized error uses: + +- code: `upstream_stream_truncated` +- type: `server_error` +- message: `Responses stream ended before a terminal event` + +The existing route-level `_status_for_error` fallback maps that envelope to +HTTP 502. No Chat-specific status policy is added. + +### Decision: leave explicit incomplete events successful + +`response.incomplete` is a terminal event with a meaningful finish reason such +as `length` or `content_filter`. It remains a Chat completion. Only EOF without +any terminal event is classified as transport truncation. + +## Risks / Trade-offs + +- A caller that previously relied on partial EOF content will now receive a + retriable error. That is intentional because presenting partial output as + complete is contract-breaking and suppresses retries. +- Streaming may already have emitted partial content before the error. The + terminal error chunk and `[DONE]` make that state explicit without retracting + bytes already delivered. + +## Test Strategy + +- Unit-test streaming delta-then-EOF for error + `[DONE]`. +- Unit-test collected delta-then-EOF for the canonical error envelope. +- Route-test non-streaming HTTP status and error code. +- Manually drive streaming and non-streaming ASGI requests with an inert + delta-then-EOF upstream. +- Keep existing explicit completion, incomplete, error, usage, tool-call, and + generator-close tests green. diff --git a/openspec/changes/reject-truncated-chat-completions/proposal.md b/openspec/changes/reject-truncated-chat-completions/proposal.md new file mode 100644 index 0000000000..97ec561b1f --- /dev/null +++ b/openspec/changes/reject-truncated-chat-completions/proposal.md @@ -0,0 +1,34 @@ +## Why + +The Chat Completions adapter currently treats an upstream Responses iterator +that reaches EOF without a terminal event as success. Streaming callers receive +content without the required `data: [DONE]` marker, while non-streaming callers +receive a successful `chat.completion` whose partial text has +`finish_reason=stop`. The public Responses adapter already classifies the same +condition as `upstream_stream_truncated`. + +## What Changes + +- Detect upstream EOF before any `response.completed`, + `response.incomplete`, `response.failed`, or `error` event. +- Emit an OpenAI error chunk followed by `data: [DONE]` for streaming Chat + Completions. +- Return an OpenAI error envelope that maps to HTTP 502 for non-streaming Chat + Completions. +- Preserve explicit terminal/error handling, usage/tool-call finalization, and + upstream generator cleanup. + +## Capabilities + +### Modified Capabilities + +- `chat-completions-compat`: define deterministic truncation behavior for + streaming and collected Chat Completions. + +## Impact + +- Affected code: `app/core/openai/chat_responses.py` +- Affected route: `POST /v1/chat/completions` +- Affected tests: Chat response mapping and proxy Chat Completions integration +- Compatibility: malformed upstream termination changes from false success to a + stable OpenAI server-error envelope diff --git a/openspec/changes/reject-truncated-chat-completions/specs/chat-completions-compat/spec.md b/openspec/changes/reject-truncated-chat-completions/specs/chat-completions-compat/spec.md new file mode 100644 index 0000000000..c9b7a66737 --- /dev/null +++ b/openspec/changes/reject-truncated-chat-completions/specs/chat-completions-compat/spec.md @@ -0,0 +1,38 @@ +## ADDED Requirements + +### Requirement: Chat Completions reject truncated upstream Responses streams + +`POST /v1/chat/completions` MUST classify upstream Responses iterator +exhaustion before a terminal `response.completed`, `response.incomplete`, +`response.failed`, or `error` event as `upstream_stream_truncated`. The error +MUST use OpenAI error type `server_error`. Partial content received before the +exhaustion MUST NOT be presented as a successfully completed non-streaming Chat +Completion. + +#### Scenario: Streaming upstream EOF emits error and done + +- **WHEN** a streaming Chat Completions request receives zero or more + non-terminal upstream Responses events +- **AND** the upstream iterator reaches EOF before a terminal event +- **THEN** the proxy MUST emit an OpenAI error chunk with code + `upstream_stream_truncated` +- **AND** the proxy MUST terminate the stream with `data: [DONE]` + +#### Scenario: Collected upstream EOF returns an error envelope + +- **WHEN** a non-streaming Chat Completions request receives zero or more + non-terminal upstream Responses events +- **AND** the upstream iterator reaches EOF before a terminal event +- **THEN** the proxy MUST return HTTP 502 +- **AND** the response body MUST be an OpenAI error envelope with code + `upstream_stream_truncated` and type `server_error` +- **AND** the proxy MUST NOT return a `chat.completion` success object + +#### Scenario: Explicit terminal events retain existing behavior + +- **WHEN** the upstream iterator emits `response.completed`, + `response.incomplete`, `response.failed`, or `error` +- **THEN** the proxy MUST preserve the existing Chat Completions mapping for + that event +- **AND** the proxy MUST preserve existing usage, tool-call, and upstream + generator cleanup behavior diff --git a/openspec/changes/reject-truncated-chat-completions/tasks.md b/openspec/changes/reject-truncated-chat-completions/tasks.md new file mode 100644 index 0000000000..6fbf870873 --- /dev/null +++ b/openspec/changes/reject-truncated-chat-completions/tasks.md @@ -0,0 +1,23 @@ +## 1. Specification + +- [x] 1.1 Define streaming and non-streaming EOF truncation requirements. +- [x] 1.2 Validate the scoped OpenSpec change. + +## 2. Regression Coverage + +- [x] 2.1 Add a streaming adapter regression for error chunk plus `[DONE]`. +- [x] 2.2 Add a collected adapter regression for the canonical error envelope. +- [x] 2.3 Add a non-streaming route regression for HTTP 502 and + `upstream_stream_truncated`. + +## 3. Implementation + +- [x] 3.1 Track terminal event observation in the Chat adapter. +- [x] 3.2 Synthesize canonical truncation errors without changing explicit + terminal behavior. + +## 4. Verification + +- [x] 4.1 Run focused Chat adapter and route tests. +- [x] 4.2 Manually verify streaming and non-streaming HTTP surfaces. +- [x] 4.3 Run lint, type, diagnostic, and strict OpenSpec gates. diff --git a/openspec/changes/relay-unmodified-sse-frames-verbatim/design.md b/openspec/changes/relay-unmodified-sse-frames-verbatim/design.md new file mode 100644 index 0000000000..4cffc70cd3 --- /dev/null +++ b/openspec/changes/relay-unmodified-sse-frames-verbatim/design.md @@ -0,0 +1,115 @@ +# Design — relay-unmodified-sse-frames-verbatim + +## Context + +Follow-up to `validate-stream-lifecycle-events-only` (stacked on it) and the +second half of the deferral in `2026-07-13-optimize-sse-single-parse`: after +lifecycle-only validation, every frame still pays one `json.loads` per owning +layer and an unconditional `format_sse_event` re-encode in the streaming +mixin. Delta frames after the first visible token have no per-event consumer, +so their bytes can relay verbatim. + +## Goals / Non-Goals + +**Goals:** zero JSON parse and zero re-encode for canonically framed frames no +consumer needs parsed; preserve every consumer's trigger surface (terminal +settlement, error rewrite, tool-call rewrite/dedupe, text-done suppression, +service-tier attribution, TTFT, reservation touches); keep the +`5ee532cb` framing guarantee (data-only blocks are re-framed with +`event: ` for EventSource clients); fix the stale-`event:`-line alias +bug that verbatim relay would otherwise expose. + +**Non-Goals:** the chat/completions bridge (cross-dialect translation keeps +full parsing); the `/v1` public normalizer (independent per-chunk consumer — +verbatim blocks satisfy its identity gate unchanged, see below); the websocket +relay and bridge upstream reader (every ws frame is parsed for response-id +multiplexing; covered by the stacked lifecycle change); the first-event block +in the streaming mixin (once per stream; retry classification lives there). + +## Decisions + +- **Cheap type = strict canonical shape.** `sse_event_type_from_block` matches + only `event: \ndata: {…}\n\n` (single data line, LF framing, + JSON-object data). SSE legally allows the `event:` field after `data:`, + CR/CRLF framing, comments, and multi-line data — all of those return `None` + and take the existing full-parse path, so only blocks byte-shaped like + `format_sse_event` output (which is what the upstream Codex backend and our + own re-encodes emit) are eligible for verbatim relay. This resolves the + field-ordering checklist item. +- **Must-parse set** = lifecycle/terminal frames (`response.created`, + `response.in_progress`, `response.completed`, `response.failed`, + `response.incomplete`, `error`) + tool-call item frames + (`response.output_item.added`, `response.output_item.done` — rewrite, + duplicate suppression, and TTFT item inspection) + text-done frames + (`response.output_text.done`, `response.content_part.done` — suppression + reads the payload's `part`). Everything else has no payload consumer outside + the gated windows below. +- **TTFT window trigger.** The full parse also runs while + `latency_first_token_ms is None` **or** `ttft_reasoning_deltas` is + non-empty. Verified by reading `support.py`: `_ttft_event_latency_ms` (and + therefore all mutation of the pending reasoning-delta state) is invoked only + under the `latency_first_token_ms is None` guard (mixin loop and first-event + block), and the stream-end `_finalize_ttft_latency_ms` is gated on the same + condition — so the first clause alone already covers the pending window; the + explicit non-empty check is a defensive belt (pending entries can outlive + TTFT settlement, e.g. a second reasoning summary stream, but are never read + after it). +- **Service-tier gate** stays on the raw line (`'"service_tier"'` substring), + not the event type, so a moved snapshot field still full-parses; false + positives (the substring inside delta text) just take the parse path. +- **Verbatim branch bookkeeping.** The reservation touch (non-terminal frames + keep reservations alive), `saw_text_delta`, `settlement.downstream_visible`, + and `settlement.downstream_text_visible` are preserved; text flags derive + from the cheap type, which for canonical frames equals the payload type. +- **Client normalizer laziness.** `_normalize_stream_payload_for_http_block` + returns the cheap type without parsing only when the block is canonical, the + type is not `error` and not a legacy alias, and the block has no `"error"` + substring. The substring guard is load-bearing: `parse_error_payload` + rewrites any payload carrying a top-level `error` envelope regardless of its + `type` (`OpenAIErrorEnvelope.error` is optional, so only an actual `error` + key triggers it), and response snapshots legitimately carry `"error":null` — + both stay on the full-parse path. +- **Alias gate + stale `event:` line.** `_normalize_sse_event_block` now gates + on the three bare alias names (matching both `"type":""` in data + lines and `event: ` framing lines) instead of `'"type":'`, and + rewrites the `event:` line too. Previously only the data line was rewritten + and the mixin's unconditional re-encode masked the mismatch; under verbatim + relay the stale line would reach clients, so the fix lands in the same + change. +- **/v1 identity gate verified for raw UTF-8.** `api.py` pass-through compares + parsed-payload *object identity* (`normalized_payload is parsed_payload`) + plus `_has_canonical_event_framing`, which checks only the + `event: \n` prefix — no comparison against a re-serialization — so + verbatim raw-UTF-8 blocks pass through byte-identically (regression test + added). + +## Accepted limitations (documented drift) + +- **Byte-visible output change.** Unmodified delta frames now carry upstream + bytes (raw UTF-8, upstream key order/spacing) instead of the `ensure_ascii` + canonical re-encode. JSON-equivalent and SSE-valid; codified in the spec + delta. +- **The `event:` framing line is trusted for non-parsed frames.** A + hypothetical upstream frame whose `event:` line disagrees with its payload + `type` (never emitted by upstream; our own re-encodes are consistent by + construction) would be classified by the framing line: a terminal payload + disguised under a delta `event:` line would relay verbatim and settle as + `stream_incomplete` at EOF instead of a terminal settlement. Today's + behavior for such frames differs only in which side wins; the `/v1` + normalizer still parses independently and enforces its own contract. +- **Malformed-JSON canonical frames.** A canonical-looking block whose data is + not valid JSON relays verbatim (today it is also yielded unchanged — the + parse failure path skips the re-encode) but now sets `saw_text_delta` / + text-visibility from the framing line, which the parse path would not. + Only reachable from a misbehaving upstream. +- **Usage on delta frames** would relay verbatim without settlement capture, + as under the stacked lifecycle change (upstream emits usage only on + terminal frames) — unchanged hedge, inherited. + +## Stacking note (delta-merge hazard) + +This change is stacked on `validate-stream-lifecycle-events-only` and MODIFIES +the same `responses-api-compat` requirement. To avoid the concurrent-MODIFIED +last-writer-wins loss (#1772), this change's delta contains the **union** text: +the lifecycle-only validation clauses from the stacked change plus the +verbatim-relay condition, so syncing/archiving in either order preserves both. diff --git a/openspec/changes/relay-unmodified-sse-frames-verbatim/proposal.md b/openspec/changes/relay-unmodified-sse-frames-verbatim/proposal.md new file mode 100644 index 0000000000..0ba2cc2b08 --- /dev/null +++ b/openspec/changes/relay-unmodified-sse-frames-verbatim/proposal.md @@ -0,0 +1,78 @@ +# Relay Unmodified SSE Frames Verbatim + +## Why + +Even after lifecycle-only validation (`validate-stream-lifecycle-events-only`), +every streamed SSE frame still pays one `json.loads` per owning layer plus an +unconditional `format_sse_event` re-encode in the streaming mixin, and the +core client parses every frame's payload just to read its `type` for terminal +detection. The dominant traffic — text/reasoning/tool-argument delta frames +after the first visible token — has no per-event consumer at all: tool-call +rewrite and duplicate suppression act only on `response.output_item.*`, +text-done suppression only on `response.output_text.done` / +`response.content_part.done`, service-tier attribution only on response +snapshots carrying `"service_tier"`, TTFT only while the first-token window is +open, and settlement/error handling only on lifecycle frames. This is the +remaining half of the follow-up the `2026-07-13-optimize-sse-single-parse` +design doc deferred, and py-spy attributes the `format_sse_event` `json.dumps` +leaf plus 2–3 redundant `json.loads` per chunk to it. + +## What Changes + +- `app/core/utils/sse.py` gains `sse_event_type_from_block`: cheap event-type + extraction that matches only the exact canonical block shape + `format_sse_event` emits (leading `event: ` line, single JSON-object + `data:` line, LF framing). Data-only blocks, multi-line data, CR/CRLF + framing, and `event:` fields appearing after `data:` (legal SSE, but not + canonical here) return `None` so callers fall back to a full parse. +- Streaming mixin hot loop: compute the cheap type first; run the full + parse only when the type is unavailable, is in the must-parse set + (lifecycle/terminal frames, `response.output_item.added`/`done`, + `response.output_text.done`, `response.content_part.done`), the TTFT + first-token window is open (including a pending reasoning-delta window), or + the raw line carries the `"service_tier"` marker. Otherwise the upstream + block is yielded verbatim — raw UTF-8 and upstream key order/spacing instead + of the `ensure_ascii` canonical re-encode — with text-visibility accounting + set from the cheap type. The first-event block stays fully parsed. +- Core client `_normalize_stream_payload_for_http_block` becomes lazy: a + canonical non-error block without an `"error"` substring returns its cheap + type with no JSON parse; error frames, error-envelope payloads, alias types, + and non-canonical framing keep the full parse + rewrite path. +- Core client `_normalize_sse_event_block` narrows its gate from `'"type":'` + (matches every event, gates nothing) to the three legacy alias substrings, + and — in the same change, because verbatim relay would otherwise expose the + latent bug — rewrites the stale `event:` framing line alongside the `data:` + payload when an alias fires. +- The chat/completions bridge and the `/v1` public normalizer are untouched; + the `/v1` identity pass-through gate (parsed-payload object identity + + canonical framing prefix) accepts verbatim upstream blocks unchanged. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `responses-api-compat`: the single-parse streaming requirement gains the + verbatim-relay condition — a canonically framed frame that no per-event + consumer needs parsed MAY be relayed with upstream bytes verbatim + (JSON-equivalent, SSE-valid); all other frames keep the parse + + canonical-re-serialization path, and legacy alias rewrites MUST cover both + the `data:` payload and the `event:` framing line. + +## Impact + +- **Code**: `app/core/utils/sse.py`, `app/core/clients/proxy.py`, + `app/modules/proxy/_service/streaming/mixin.py`. +- **Behavior**: byte-visible but JSON-equivalent — unmodified delta frames now + carry upstream bytes (raw UTF-8, upstream key order/spacing) instead of the + `ensure_ascii` canonical re-encode. Framing stays SSE-valid and named-event + clients keep seeing `event:` lines (non-canonical blocks still get + re-framed). Legacy `response.text.delta`-style upstreams now get a correct + `event:` line after alias rewrite (previously stale, masked by the mixin + re-encode). +- **Performance**: removes the per-delta `json.loads` in the core client and + the mixin plus the per-delta `format_sse_event` re-encode for the dominant + post-first-token delta traffic. diff --git a/openspec/changes/relay-unmodified-sse-frames-verbatim/specs/responses-api-compat/spec.md b/openspec/changes/relay-unmodified-sse-frames-verbatim/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..83dddff9fc --- /dev/null +++ b/openspec/changes/relay-unmodified-sse-frames-verbatim/specs/responses-api-compat/spec.md @@ -0,0 +1,68 @@ +# responses-api-compat Delta + +## MODIFIED Requirements + +### Requirement: Streaming events are parsed once and re-serialized only when modified + +Within each streaming layer (core client consumer, streaming mixin, bridge upstream reader, websocket relay, /v1 normalizers), an SSE event's JSON payload MUST be parsed at most once and reused by that layer's consumers, and an event that no consumer modified MUST NOT be re-serialized by the /v1 normalizers. Schema validation of the parsed payload MUST run only for stream lifecycle frames (`response.created`, `response.completed`, `response.incomplete`, `response.failed`, `error`); all other frames MUST be classified from the parsed payload's `type` field (with a typeless payload carrying an `error` object classifying as `error`). + +A canonically framed SSE block — a leading `event: ` line followed by a single JSON-object `data:` line with LF framing — whose type requires no per-event consumer MAY skip payload parsing entirely and be relayed downstream with the upstream bytes verbatim (raw UTF-8 and upstream key order/spacing preserved; JSON-equivalent to the canonical re-encode). A frame MUST take the parse path when any consumer needs it: lifecycle/terminal frames, tool-call item frames (`response.output_item.added`, `response.output_item.done`), text-done frames (`response.output_text.done`, `response.content_part.done`), frames arriving while the TTFT first-token window is open (including a pending reasoning-delta window), frames carrying a `"service_tier"` marker, and any block without canonical framing (data-only blocks, multi-line data, or an `event:` field that does not lead the block). A parsed frame MUST be re-serialized with canonical `event: ` + `data:` framing when modified or when its source block lacked canonical framing. Legacy event-type alias rewrites MUST cover both the `data:` payload type and the `event:` framing line. Event framing, payload contents, dedupe/rewrite semantics, usage settlement, and error normalization MUST be unchanged. + +#### Scenario: Unmodified events pass through the /v1 normalizer verbatim + +- **GIVEN** a canonical stream event that no normalizer branch rewrites +- **WHEN** the /v1 response normalizer processes it +- **THEN** the original block is yielded byte-identically without re-serialization + +#### Scenario: Tool-call rewrite reuses the parsed event on the no-change path + +- **GIVEN** an event without duplicate parallel tool calls +- **WHEN** the rewrite step runs with the caller's parsed event +- **THEN** it returns the original line, payload, and event without re-parsing or re-validating + +#### Scenario: Rewritten events stay consistent + +- **WHEN** the rewrite step removes duplicate tool calls +- **THEN** the returned line, payload, and validated event all reflect the rewritten content + +#### Scenario: Delta frames skip schema validation + +- **GIVEN** a stream of `response.output_text.delta` frames between `response.created` and `response.completed` +- **WHEN** the streaming mixin, websocket relay, or bridge upstream reader processes the stream +- **THEN** only the lifecycle frames are schema-validated, the delta frames are classified from the parsed payload dict, and downstream output, usage settlement, and error normalization are unchanged + +#### Scenario: Identity websocket relay frames are forwarded without re-encoding + +- **GIVEN** a websocket frame matched to a request whose downstream response-id rewrite does not apply +- **WHEN** the relay forwards the frame downstream +- **THEN** the upstream frame text is forwarded as-is instead of a canonical JSON re-encode + +#### Scenario: Unmodified canonical delta frames relay upstream bytes verbatim + +- **GIVEN** a canonically framed `response.output_text.delta` frame containing raw UTF-8, arriving after the first visible token settled the TTFT window +- **WHEN** the streaming mixin processes it +- **THEN** the upstream block is yielded byte-identically without a JSON parse or `ensure_ascii` re-encode, and downstream text-visibility accounting still updates + +#### Scenario: Data-only frames regain canonical framing + +- **GIVEN** a delta frame without a leading `event:` line +- **WHEN** the streaming mixin processes it after the TTFT window settles +- **THEN** the frame is parsed and re-serialized with the canonical `event: ` line so named-event (EventSource) clients keep seeing the event name + +#### Scenario: Legacy alias frames are rewritten on both lines + +- **GIVEN** an upstream block whose `event:` line and `data:` payload both carry the legacy `response.text.delta` type +- **WHEN** the core client normalizes the block +- **THEN** both the `event:` framing line and the payload `type` read `response.output_text.delta` + +#### Scenario: Error frames keep the full parse and rewrite path + +- **GIVEN** a canonically framed `error` frame, or a frame whose payload carries a top-level `error` envelope +- **WHEN** the core client normalizes the stream for the SDK contract +- **THEN** the frame is parsed and rewritten to a terminal `response.failed` event exactly as before verbatim relay + +#### Scenario: /v1 identity pass-through accepts verbatim raw-UTF-8 blocks + +- **GIVEN** an upstream-verbatim canonical delta block containing raw UTF-8 +- **WHEN** the /v1 normalizer leaves the parsed payload unmodified +- **THEN** the block passes through byte-identically (the identity gate compares parsed-payload object identity and the `event:` framing prefix, not re-serialized bytes) diff --git a/openspec/changes/relay-unmodified-sse-frames-verbatim/tasks.md b/openspec/changes/relay-unmodified-sse-frames-verbatim/tasks.md new file mode 100644 index 0000000000..3c0b3945b8 --- /dev/null +++ b/openspec/changes/relay-unmodified-sse-frames-verbatim/tasks.md @@ -0,0 +1,34 @@ +# Tasks — relay-unmodified-sse-frames-verbatim + +## 1. Implementation + +- [x] 1.1 `sse_event_type_from_block` in `app/core/utils/sse.py`: strict + canonical-shape matcher (leading `event:` line, single JSON-object + `data:` line, LF framing); `None` otherwise +- [x] 1.2 Streaming mixin hot loop: verbatim relay branch gated on cheap type + ∉ must-parse set, TTFT window settled (`latency_first_token_ms` set and + no pending reasoning deltas), and no `"service_tier"` marker; keeps + reservation touch + text-visibility accounting; first-event block stays + fully parsed +- [x] 1.3 `_normalize_stream_payload_for_http_block`: lazy cheap-type return + for canonical non-error, non-alias blocks without an `"error"` + substring +- [x] 1.4 `_normalize_sse_event_block`: gate narrowed from `'"type":'` to the + three alias substrings; alias rewrite covers the `event:` framing line + in addition to the `data:` payload + +## 2. Validation + +- [x] 2.1 Unit coverage for `sse_event_type_from_block` (canonical, raw + UTF-8, data-only, trailing `event:` ordering, CRLF/multi-line, + non-object data) +- [x] 2.2 Mixin regressions: raw-UTF-8 delta relayed byte-identically with no + JSON parse after TTFT settles; data-only delta re-framed with + `event: ` (5ee532cb regression class); usage settlement unchanged +- [x] 2.3 Client normalizer regressions: canonical frames skip `json.loads`; + `error` frames and top-level error envelopes still rewritten; alias + rewrite covers both lines; non-alias blocks skip the alias parse +- [x] 2.4 `/v1` identity pass-through accepts verbatim raw-UTF-8 blocks + byte-identically +- [x] 2.5 Existing streaming/contract/dedupe/TTFT suites pass; `uvx ruff + format --check`, `uv run ruff check` on changed files diff --git a/openspec/changes/remove-codex-review-label-gate/.openspec.yaml b/openspec/changes/remove-codex-review-label-gate/.openspec.yaml new file mode 100644 index 0000000000..0c73c8f54e --- /dev/null +++ b/openspec/changes/remove-codex-review-label-gate/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-15 diff --git a/openspec/changes/remove-codex-review-label-gate/design.md b/openspec/changes/remove-codex-review-label-gate/design.md new file mode 100644 index 0000000000..0a60fe427d --- /dev/null +++ b/openspec/changes/remove-codex-review-label-gate/design.md @@ -0,0 +1,29 @@ +## Context + +The retiring gate is implemented as one GitHub Actions workflow, one synchronization script, and its dedicated unit test module. Contributor guidance and workflow comments also describe that automation. Branch protection requires stable CI check contexts, not Codex labels, and the simplicity-budget workflow still needs label events for its independent override label. + +## Goals / Non-Goals + +**Goals:** + +- Remove the Codex review label automation as one coherent unit. +- Replace its merge-gate documentation with current-head CodeRabbit evidence. +- Preserve required CI check names and simplicity-budget override behavior. + +**Non-Goals:** + +- Changing branch-protection rules or required CI jobs. +- Removing the local Codex review harness or optional local review command. +- Replacing the removed `needs rebase` label with another synchronization workflow. + +## Decisions + +- Delete the workflow, script, and dedicated tests instead of disabling them. This prevents dormant automation from remaining a maintenance surface; retaining a disabled compatibility shim was rejected because there are no protected status checks or consumers to preserve. +- Remove every main-spec requirement whose behavior is implemented by the deleted synchronizer, including apply-time reclassification. The unrelated CI path-filtering and simplicity-budget requirements remain outside the delta. +- Keep `labeled` and `unlabeled` events on the simplicity-budget workflow because they re-evaluate `simplicity-budget-approved`, independent of the deleted label churn. +- Treat GitHub's live `mergeable` API field as triage evidence instead of replacing the removed `needs rebase` label sync. + +## Risks / Trade-offs + +- [Risk] Stale `needs rebase` labels may remain after automation removal. → Triage must use the live `mergeable` field, which is already the accepted source of truth. +- [Risk] Documentation could imply that optional local Codex review is still mandatory. → State consistently that CodeRabbit is the gate and local Codex review is only encouraged. diff --git a/openspec/changes/remove-codex-review-label-gate/proposal.md b/openspec/changes/remove-codex-review-label-gate/proposal.md new file mode 100644 index 0000000000..e85b9e1237 --- /dev/null +++ b/openspec/changes/remove-codex-review-label-gate/proposal.md @@ -0,0 +1,27 @@ +## Why + +The repository is adopting CodeRabbit on its OSS plan as the always-on mechanical reviewer under issue #1756. The auto-posted `@codex review` and `🤖 codex: ok` label gate is therefore redundant and should be retired. + +## What Changes + +- Replace the documented Codex cloud-review merge gate with a CodeRabbit gate that requires actionable findings to be fixed or explicitly addressed or dismissed in-thread on the merge-target head. +- Remove the Codex review label synchronization workflow, script, and unit tests. +- Remove the associated `needs rebase` label synchronization as accepted collateral; triage uses GitHub's live `mergeable` API field as its source of truth. +- Discard the in-flight `label-sync-rate-limit-fallback` change because it only patches the machinery being removed. +- Keep local `codex review --base origin/main` runs as an encouraged extra tool, not a merge gate. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `github-automation`: remove the requirements governing the retired Codex review and needs-rebase label synchronization machinery. + +## Impact + +- GitHub automation no longer auto-posts Codex review requests or maintains Codex review and needs-rebase labels. +- Contributor guidance uses CodeRabbit review evidence for the mechanical-review merge gate. +- Branch-protection status checks and simplicity-budget override label behavior remain unchanged. diff --git a/openspec/changes/remove-codex-review-label-gate/specs/github-automation/spec.md b/openspec/changes/remove-codex-review-label-gate/specs/github-automation/spec.md new file mode 100644 index 0000000000..0fe96196e1 --- /dev/null +++ b/openspec/changes/remove-codex-review-label-gate/specs/github-automation/spec.md @@ -0,0 +1,348 @@ +## REMOVED Requirements + +### Requirement: Needs-rebase label sync + +The Codex label synchronization script MUST add `needs rebase` when GitHub +reports a confirmed merge conflict, MUST remove it when GitHub reports a known +mergeable or non-conflict state, and MUST preserve its current value only when +GitHub reports `UNKNOWN`. It MUST NOT infer a conflict from the pull request +merely being behind the base branch. + +#### Scenario: Confirmed conflict gains the label + +- **WHEN** GitHub reports the pull request as `CONFLICTING` or `DIRTY` +- **THEN** the synchronizer adds `needs rebase` + +#### Scenario: Review-blocked pull request loses a stale label + +- **GIVEN** a pull request has `needs rebase` +- **WHEN** GitHub reports it as `BLOCKED` by review or status requirements +- **THEN** the synchronizer removes `needs rebase` + +#### Scenario: Base lag alone removes a stale label + +- **WHEN** GitHub reports a pull request as `BEHIND` without a confirmed conflict +- **THEN** the synchronizer removes `needs rebase` when present +- **AND** it does not add the label to an unlabelled pull request + +#### Scenario: Other mergeable statuses remove a stale label + +- **GIVEN** a pull request has `needs rebase` +- **WHEN** GitHub reports `CLEAN`, `DRAFT`, `HAS_HOOKS`, or `UNSTABLE` +- **THEN** the synchronizer removes `needs rebase` + +#### Scenario: Unknown state preserves current evidence + +- **WHEN** GitHub reports `UNKNOWN` +- **THEN** the synchronizer preserves the current `needs rebase` value + +### Requirement: Codex review label sync write-token fallback + +The `Codex review labels` workflow MUST execute the label synchronization script from the trusted default branch and MUST prefer a dedicated GitHub App installation token, then a repository-provided write token, before falling back to the default `github.token`. + +#### Scenario: GitHub App credentials are configured + +- **WHEN** the repository defines the `CODEX_LABEL_SYNC_APP_ID` variable and the `CODEX_LABEL_SYNC_APP_PRIVATE_KEY` secret +- **THEN** the workflow mints a short-lived installation token for that App before the sync step +- **AND** the mint requests only the label-sync permission subset (actions write, checks read, contents read, issues write, pull requests read, statuses read) rather than inheriting all installation permissions +- **AND** the sync step uses that token ahead of `CODEX_LABEL_SYNC_TOKEN`, `RELEASE_PLEASE_TOKEN`, and `github.token` + +#### Scenario: App token mint fails or is not configured + +- **WHEN** the mint step fails, or the `CODEX_LABEL_SYNC_APP_ID` variable is absent +- **THEN** the job does not fail because of the mint step +- **AND** the sync step falls back to the next available token in the chain + +#### Scenario: Privileged token is configured + +- **WHEN** the workflow synchronizes Codex review labels and no App token was minted +- **THEN** it uses `CODEX_LABEL_SYNC_TOKEN` when present +- **AND** it falls back to `RELEASE_PLEASE_TOKEN` before `github.token` +- **AND** it checks out the default branch with persisted checkout credentials disabled + +### Requirement: Codex review label sync write-denial resilience + +The Codex label synchronization script MUST distinguish GitHub write-permission denials from classification/read failures. + +#### Scenario: GitHub App token cannot mutate a PR resource + +- **WHEN** a label, comment, or workflow-run approval write returns `Resource not accessible by integration (HTTP 403)` +- **THEN** the workflow logs a per-PR warning for the skipped mutation +- **AND** it continues processing remaining selected PRs +- **AND** it exits successfully if no read/classification errors occurred + +#### Scenario: PR state cannot be read or classified + +- **WHEN** the script cannot read required PR state, check state, merge state, or Codex review evidence +- **THEN** the workflow fails rather than silently treating the PR as synchronized + +### Requirement: Codex review label sync review-thread state + +The Codex label synchronization script MUST grant `🤖 codex: ok` only when the +current pull-request head has green required checks, a clean Codex review for +that head, and no unresolved current-head Codex finding threads. It MUST treat +unresolved, non-outdated Codex inline review findings on the current head as +needs-work evidence, and MUST NOT treat inline Codex findings from resolved or +outdated review threads as active needs-work evidence. It MUST attribute an +unresolved thread to the current head only when the thread's current commit, +original commit, or body text ties it to the current head, and MUST treat +stale unresolved Codex inline threads as non-blocking when none of those tie +them to the current head. + +#### Scenario: Resolved inline finding no longer blocks the ok label + +- **WHEN** a current-head inline Codex finding comment belongs to a resolved + review thread +- **AND** a clean current-head Codex review exists +- **THEN** the script does not classify that inline finding as active + needs-work evidence + +#### Scenario: Unresolved inline finding still blocks the ok label + +- **WHEN** a current-head inline Codex finding comment belongs to an unresolved, + non-outdated review thread +- **THEN** the script classifies that inline finding as active needs-work + evidence + +#### Scenario: stale rebased inline thread remains unresolved + +- **GIVEN** a pull request was rebased after a Codex inline finding +- **AND** the unresolved GraphQL review thread still reports `isOutdated=false` +- **AND** the thread's current commit is not the current head +- **AND** the thread's original commit is not the current head +- **AND** the thread body does not mention the current head +- **WHEN** the label synchronizer evaluates the pull request +- **THEN** that thread does not force `🤖 codex: needs work` + +#### Scenario: reanchored unresolved inline thread belongs to the current head + +- **GIVEN** an unresolved Codex inline finding thread +- **AND** the thread's current commit is the pull request head +- **AND** the thread's original commit is older than the pull request head +- **WHEN** the label synchronizer evaluates the pull request +- **THEN** that thread blocks `🤖 codex: ok` +- **AND** the synchronizer records a needs-work reason that links to the thread + +#### Scenario: unresolved inline thread belongs to the current head + +- **GIVEN** an unresolved Codex inline finding thread +- **AND** the thread's original commit is the pull request head +- **WHEN** the label synchronizer evaluates the pull request +- **THEN** that thread blocks `🤖 codex: ok` +- **AND** the synchronizer records a needs-work reason that links to the thread + +#### Scenario: unresolved inline thread mentions the current head explicitly + +- **GIVEN** an unresolved Codex inline finding thread +- **AND** the thread body mentions the current pull request head +- **WHEN** the label synchronizer evaluates the pull request +- **THEN** that thread blocks `🤖 codex: ok` +- **AND** the synchronizer records a needs-work reason that links to the thread + +#### Scenario: resolved inline thread is resynchronized by the scheduled fallback + +- **GIVEN** a pull request has a `🤖 codex: needs work` label from an unresolved Codex inline finding +- **WHEN** that review thread is resolved +- **THEN** the scheduled Codex label synchronization run resynchronizes the open pull request's labels + +### Requirement: Codex review labels use the authoritative current-head CI suite + +The Codex review label synchronizer SHALL identify the CI workflow from the +most recent `CI Required` check and SHALL treat the newest same-head run of +that workflow (ordered by the current attempt's start time, falling back to +workflow-run creation time, then check recency and run id) as the authoritative +CI suite when multiple runs of the same GitHub Actions CI workflow exist for +one pull-request head, even when that run has not yet produced its own +`CI Required` check. It MUST ignore Actions checks — +including stale required contexts — only from superseded (older) runs of that +workflow, while checks from the authoritative run, checks that cannot be +attributed to a workflow run, non-Actions status evidence, and failures from +independent workflows remain blocking evidence. + +#### Scenario: Cancelled duplicate leaves a unique failed placeholder + +- **GIVEN** an older CI workflow run for the current head was cancelled +- **AND** that run left a uniquely named non-required matrix placeholder in failure +- **AND** a newer run for the same head completed every required check including `CI Required` successfully +- **WHEN** Codex review labels are synchronized +- **THEN** the stale placeholder does not make the current head failed +- **AND** the synchronizer may request or accept current-head Codex review evidence + +#### Scenario: Authoritative CI run has an optional failure + +- **GIVEN** the newest run of the CI workflow identified by the latest `CI Required` check is the authoritative run +- **AND** another check in that same run failed +- **WHEN** Codex review labels are synchronized +- **THEN** the current head remains classified as failed + +#### Scenario: A newer run stays pending until its own CI Required completes + +- **GIVEN** an older run of the CI workflow completed `CI Required` successfully for the current head +- **AND** a newer run of the same CI workflow was created for the same head +- **AND** the newer run has started early checks but has not yet completed its own `CI Required` check +- **WHEN** Codex review labels are synchronized +- **THEN** the newer run is the authoritative CI suite and the older run's completed checks are ignored +- **AND** the current head remains classified as pending until the newer run's `CI Required` completes + +#### Scenario: An older workflow run id is manually rerun + +- **GIVEN** a newer-created CI workflow run completed successfully for the current head +- **AND** an older workflow `run_id` is manually rerun afterward +- **WHEN** the older run's new attempt has the latest `run_started_at` +- **THEN** that rerun is the authoritative CI suite +- **AND** its pending or failed checks remain blocking evidence + +#### Scenario: Independent workflow on the same head fails + +- **GIVEN** the authoritative CI workflow run is successful +- **AND** a different GitHub Actions workflow has a failed check on the same head +- **WHEN** Codex review labels are synchronized +- **THEN** the independent workflow failure remains blocking + +### Requirement: Codex label sync MUST use check-run recency evidence + +When multiple check runs have the same context name on a pull-request head, the label synchronizer MUST classify the current context from the newest run by +start or creation time. Completion time MUST NOT let an older superseded run +override a newer rerun that has already started. + +#### Scenario: older duplicate run completes after a newer rerun starts + +- **GIVEN** two check runs share the same name +- **AND** the older run started first but completes after the newer run starts +- **WHEN** the label synchronizer deduplicates check runs +- **THEN** it keeps the newer run +- **AND** a pending newer run keeps the pull request check state pending instead of failed + +### Requirement: Codex review trigger usage-limit backoff + +The Codex label synchronization script MUST NOT post a new `@codex review` comment while the comment sender's latest Codex response within the configured backoff window is a usage-limit reply. A usage-limit reply is a Codex response whose body starts (after optional leading whitespace) with the quota envelope "You have reached your Codex usage limits"; Codex reviews that merely discuss usage limits MUST NOT latch the backoff. Usage-limit evidence MUST be attributed to the sender whose request comment preceded the reply, and a newer normal Codex response for that same sender MUST lift the backoff; a clean THUMBS_UP reaction by a Codex reviewer on the sender's request comment counts as a normal response. Backoff state MUST be shared across all repositories processed in one run, so a usage limit observed in one repository suppresses the remaining review requests in the run; classified timelines from repositories without their own triggers MUST still contribute evidence. Before posting review requests in a repository, the script MUST also gather the repository's recent issue comments (within the backoff window, grouped per issue) as evidence, so quota replies on pull requests outside the current selection — including single `--pr` runs and closed pull requests — still latch the backoff; a failure to gather this evidence degrades to the classified-timeline evidence with a warning. When no quota evidence exists for the sender, the script MUST post the first `@codex review`, wait briefly, reread that pull request's timeline, and suppress the remaining review requests in the run if that probe observed a usage-limit reply; probing MUST stop once a normal Codex response has been observed, and MUST NOT run when the review request was not actually posted (for example after a tolerated write denial). Apply-loop status lines and error reports MUST reference the pull request of the decision being applied. + +The script MUST resolve the sender identity in a way that works with GitHub App installation tokens (which cannot call `GET /user`): it prefers the app slug exported by the workflow (`GH_APP_SLUG`, yielding `[bot]`) and falls back to `GET /user` for PAT-backed runs. Once the run has switched to the fallback token, the app slug no longer describes the active identity: sender resolution MUST ignore it, and review triggers MUST be suppressed with a warning because posted comments would no longer be authored by the resolved sender. The review-request POST itself MUST NOT be silently retried under the fallback token after a rate-limit response: the fallback activates for subsequent calls, but the identity-sensitive comment fails instead of posting under the wrong author. If the sender cannot be resolved, only the review-trigger path is disabled (with a warning per affected decision); label synchronization and workflow-run approvals MUST proceed. + +#### Scenario: Recent usage-limit reply latches the backoff + +- **GIVEN** the sender's `@codex review` comment was answered by a Codex usage-limit reply within the backoff window +- **AND** the sender has no newer normal Codex response +- **WHEN** the script would trigger a missing Codex review +- **THEN** it skips the `@codex review` post and surfaces a write warning naming the usage-limit evidence + +#### Scenario: Reviews that merely discuss usage limits do not latch + +- **GIVEN** a Codex review whose body discusses usage limits but does not start with the quota envelope +- **WHEN** the script classifies Codex responses for the backoff +- **THEN** the response is treated as a normal Codex response, not a usage-limit reply + +#### Scenario: Newer normal response lifts the backoff + +- **GIVEN** the sender received a Codex usage-limit reply within the backoff window +- **AND** the same sender has a newer normal Codex response +- **WHEN** the script would trigger a missing Codex review +- **THEN** it posts the `@codex review` comment + +#### Scenario: Newer clean reaction lifts the backoff + +- **GIVEN** the sender received a Codex usage-limit reply within the backoff window +- **AND** a Codex reviewer later reacted with THUMBS_UP to the sender's `@codex review` comment +- **WHEN** the script would trigger a missing Codex review +- **THEN** it posts the `@codex review` comment + +#### Scenario: Senders are attributed independently + +- **GIVEN** the sender received a Codex usage-limit reply within the backoff window +- **AND** only a different account has a newer normal Codex response +- **WHEN** the script would trigger a missing Codex review +- **THEN** it still skips the `@codex review` post for the sender + +#### Scenario: Backoff persists across repositories in one run + +- **GIVEN** a run selecting multiple repositories +- **AND** the sender's usage limit was observed while processing an earlier repository +- **WHEN** the script would trigger a missing Codex review in a later repository +- **THEN** it skips the `@codex review` post there as well + +#### Scenario: Evidence from a non-triggering repository still counts + +- **GIVEN** an earlier repository whose classified timelines contain the sender's usage-limit reply but whose decisions need no review trigger +- **WHEN** a later repository in the same run would trigger a missing Codex review +- **THEN** the earlier repository's evidence latches the backoff and the post is skipped + +#### Scenario: Quota evidence outside the selected pull requests still counts + +- **GIVEN** a single `--pr` run where the sender's usage-limit reply lives on a different (possibly closed) pull request of the repository +- **WHEN** the script would trigger a missing Codex review +- **THEN** the repository's recent issue comments provide the evidence and the post is skipped + +#### Scenario: Probe requires an actual post + +- **GIVEN** the review-request comment was not posted because the write was denied and tolerated +- **WHEN** the script would otherwise probe for a quota reply +- **THEN** it neither waits nor rereads the pull request timeline for that decision + +#### Scenario: No-data probe latches off remaining triggers + +- **GIVEN** no Codex quota evidence exists for the sender in the classified timelines +- **WHEN** the script posts the first `@codex review` of the run +- **THEN** it waits the configured probe interval, rereads that pull request's timeline, and skips the remaining review requests if the probe observed a usage-limit reply + +#### Scenario: Probing stops after a normal response + +- **GIVEN** a normal Codex response for the sender has already been observed +- **WHEN** the script posts further `@codex review` comments in the run +- **THEN** it does not wait or reread pull request timelines for those posts + +#### Scenario: Installation tokens resolve the sender from the app slug + +- **GIVEN** the run authenticates with a GitHub App installation token and the workflow exports the app slug +- **WHEN** the script resolves the `@codex review` sender +- **THEN** it derives `[bot]` without calling `GET /user` + +#### Scenario: Sender resolution failure only disables review triggers + +- **GIVEN** the sender cannot be resolved from either the app slug or `GET /user` +- **WHEN** the script applies decisions +- **THEN** label synchronization proceeds and each suppressed review trigger surfaces a warning naming the unresolved sender + +#### Scenario: Fallback token activation suppresses review triggers + +- **GIVEN** the run has switched to `GH_FALLBACK_TOKEN` after rate-limit exhaustion +- **WHEN** the script would trigger a missing Codex review +- **THEN** it skips the `@codex review` post with a warning, because the comment author would no longer match the resolved sender + +#### Scenario: The review-request POST is not retried under the fallback identity + +- **GIVEN** the review-request comment POST itself hits the primary token's rate limit +- **WHEN** the fallback token activates +- **THEN** the POST fails instead of being silently retried under the fallback identity, while later calls use the fallback token + +#### Scenario: Apply status is attributed to the applied pull request + +- **WHEN** the script applies decisions for multiple pull requests in one run +- **THEN** each status line and error report references the pull request of the decision being applied + +### Requirement: Apply-time reclassification + +Before performing writes for a classified decision (label changes, legacy label removal, workflow-run approvals, or review triggers), the Codex label synchronization script MUST reclassify the pull request and act on the fresh evidence only. If the head SHA no longer matches the SHA the decision was classified against, the decision MUST be skipped with a warning. If the head is unchanged but the evidence changed (checks, reviews, mergeability), the writes MUST follow the fresh decision, and a review trigger MUST only fire when both the original and the fresh classification want it. The freshly read timeline MUST feed the shared usage-limit backoff so quota replies that arrived after bulk classification suppress the remaining review requests. Reclassification read failures MUST honor `--tolerate-read-errors` (log and skip the decision without failing the run). Decisions without pending writes need not be reclassified. + +#### Scenario: Stale decision is skipped after a head move + +- **GIVEN** a pull request whose head changed between classification and apply +- **WHEN** the script reaches that decision in the apply loop +- **THEN** it skips all writes for the decision and warns that the head moved + +#### Scenario: Same-head evidence changes are applied fresh + +- **GIVEN** a pull request whose head is unchanged but where Codex raised a new finding after classification +- **WHEN** the script reaches that decision in the apply loop +- **THEN** the writes reflect the fresh classification instead of the superseded one + +#### Scenario: Fresh quota evidence suppresses later triggers + +- **GIVEN** a quota reply that arrived between bulk classification and apply-time reclassification of one pull request +- **WHEN** later decisions in the run would trigger missing Codex reviews +- **THEN** the reclassified timeline has latched the backoff and those posts are skipped + +#### Scenario: Reclassification honors tolerant reads + +- **GIVEN** a run with `--tolerate-read-errors` +- **WHEN** apply-time reclassification of one pull request fails with a GitHub read error +- **THEN** the decision is logged and skipped without failing the run diff --git a/openspec/changes/remove-codex-review-label-gate/tasks.md b/openspec/changes/remove-codex-review-label-gate/tasks.md new file mode 100644 index 0000000000..c6b12e3679 --- /dev/null +++ b/openspec/changes/remove-codex-review-label-gate/tasks.md @@ -0,0 +1,15 @@ +## 1. Retire label synchronization + +- [x] 1.1 Delete the Codex review label workflow, synchronization script, and dedicated unit tests. +- [x] 1.2 Discard the in-flight `label-sync-rate-limit-fallback` OpenSpec change. + +## 2. Update repository contracts and guidance + +- [x] 2.1 Replace the documented cloud Codex merge gate with the current-head CodeRabbit finding gate while retaining local Codex review as optional guidance. +- [x] 2.2 Remove stale label-sync references from CI and simplicity-budget workflow comments without changing check names or label-event triggers. +- [x] 2.3 Record removal of every label-sync requirement in the `github-automation` delta while preserving unrelated CI and simplicity-budget requirements. + +## 3. Verification + +- [x] 3.1 Validate the OpenSpec change and classify every remaining retired gate-term hit as historical, local-harness, or OpenSpec removal evidence. +- [x] 3.2 Run repository lint, type checks, and the unit test suite. diff --git a/openspec/changes/report-sqlite-long-write-holders/proposal.md b/openspec/changes/report-sqlite-long-write-holders/proposal.md new file mode 100644 index 0000000000..0847f3b6f2 --- /dev/null +++ b/openspec/changes/report-sqlite-long-write-holders/proposal.md @@ -0,0 +1,19 @@ +## Why + +Issue #1682: a single-instance SQLite deployment hit a self-sustaining ~17-minute `database is locked` stall that blocked leader re-election. The log evidence shows the lease loss was a symptom — some connection held SQLite's single writer slot past the 30-second busy timeout, starving every other writer, and recovered spontaneously when the holder finally finished. The repro is nondeterministic and the holder's identity never appears in any log, so the stall cannot currently be attributed, and any teardown/deadline fix would be guesswork until it is. + +## What Changes + +- Every SQLite engine gains a long-write-transaction watchdog: the first write statement in a transaction starts the clock (WAL takes the writer slot at the first write, not at BEGIN), and when the transaction commits or rolls back after holding longer than the busy timeout, a WARNING reports the held duration, outcome, owning task name, and the first and last write statements. +- Post-hoc by design: the stall self-recovers, so identifying the holder when it ends is sufficient for attribution and needs no sampler thread. Read-only transactions never report; fast writes never report. +- No new settings: the threshold is the existing busy timeout — a writer holding longer is precisely the one making every other writer surface `database is locked`. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `database-backends`: SQLite write-lock stalls are attributable from the log. diff --git a/openspec/changes/report-sqlite-long-write-holders/specs/database-backends/spec.md b/openspec/changes/report-sqlite-long-write-holders/specs/database-backends/spec.md new file mode 100644 index 0000000000..93522660ec --- /dev/null +++ b/openspec/changes/report-sqlite-long-write-holders/specs/database-backends/spec.md @@ -0,0 +1,40 @@ +## ADDED Requirements + +### Requirement: SQLite write-lock stalls are attributable + +When a SQLite write transaction holds the writer slot longer than the configured busy timeout, the system MUST report it at WARNING once the transaction has actually ended — the end-of-transaction call itself can be the stall, so the report MUST be deferred to the first proof the DBAPI transaction is over (the connection's next transaction, or its return to the pool) and MUST include that call in the measured hold, including the held duration, whether it committed or rolled back, the owning task where available, and the first and last write statements it executed. The window MUST be measured from the completion of the transaction's first successful write statement — including a bare `BEGIN IMMEDIATE`/`BEGIN EXCLUSIVE`, which acquires the writer slot with no DML — a statement still waiting in the busy timeout has not acquired the slot, so a victim of the stall is never reported as its holder — and read-only transactions, which never take the writer slot in WAL, are never reported. The watchdog MUST NOT raise into the query path and MUST NOT require configuration. + +#### Scenario: The starving writer is identified when it finally ends + +- **GIVEN** a write transaction that held the writer slot past the busy timeout while other writers surfaced `database is locked` +- **WHEN** it commits or rolls back +- **THEN** a warning reports its duration, outcome, task, and first/last write statements + +#### Scenario: A BEGIN IMMEDIATE holder with no DML is attributed + +- **GIVEN** a transaction that acquired the writer slot via `BEGIN IMMEDIATE` and ran only reads +- **WHEN** it holds past the busy timeout +- **THEN** it is reported like any other write holder + +#### Scenario: A failed commit is not reported as a durable commit + +- **GIVEN** a write transaction whose DBAPI commit raises and is rolled back +- **WHEN** the report fires +- **THEN** its outcome states the commit failed and rolled back + +#### Scenario: A stalled commit or rollback is inside the measured hold + +- **GIVEN** a write transaction whose commit or rollback call itself stalls past the busy timeout +- **WHEN** the connection next begins a transaction or returns to the pool +- **THEN** the report fires with the stall included in the held duration + +#### Scenario: A victim waiting out the busy timeout is not reported as the holder + +- **GIVEN** a write statement that spends the busy timeout waiting for the slot and fails with `database is locked` +- **WHEN** its transaction rolls back +- **THEN** no long-write report attributes the wait to that transaction + +#### Scenario: Healthy traffic is silent + +- **WHEN** read-only transactions and writes completing under the threshold run +- **THEN** no long-write report is produced diff --git a/openspec/changes/report-sqlite-long-write-holders/tasks.md b/openspec/changes/report-sqlite-long-write-holders/tasks.md new file mode 100644 index 0000000000..db076c0a10 --- /dev/null +++ b/openspec/changes/report-sqlite-long-write-holders/tasks.md @@ -0,0 +1,12 @@ +## 1. Watchdog + +- [x] 1.1 Track from the completion of the first successful write statement (after_cursor_execute — a statement waiting in the busy timeout has not acquired the slot) and report at commit/rollback when the hold exceeded the busy timeout, with duration, outcome, task name, and first/last write statements; the report is deferred to the next begin or pool checkin so a stalled commit/rollback is inside the measured hold +- [x] 1.2 Install it from `_configure_sqlite_engine` so the main, background, and memory engines are all covered + +## 2. Tests + +- [x] 2.1 A write transaction over the threshold is reported with its statement and outcome +- [x] 2.2 Read-only transactions and fast writes stay silent +- [x] 2.3 A write that fails while waiting for the slot does not report its victim transaction as the holder +- [x] 2.4 A stalled transaction end is included in the measured hold (fails on report-at-event code) +- [x] 2.5 A bare BEGIN IMMEDIATE holder is attributed; a DBAPI commit failure reports commit_failed_rollback diff --git a/openspec/changes/reports-api-key-filter/.openspec.yaml b/openspec/changes/reports-api-key-filter/.openspec.yaml new file mode 100644 index 0000000000..d2b2210ead --- /dev/null +++ b/openspec/changes/reports-api-key-filter/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-13 \ No newline at end of file diff --git a/openspec/changes/reports-api-key-filter/proposal.md b/openspec/changes/reports-api-key-filter/proposal.md new file mode 100644 index 0000000000..3a1a47f2cd --- /dev/null +++ b/openspec/changes/reports-api-key-filter/proposal.md @@ -0,0 +1,14 @@ +## Why + +Users need the ability to filter the Reports dashboard by specific API keys (matching the Request Logs filter) so summary, daily trends, model, and account metrics can be narrowed to specific API key traffic. + +## What Changes + +- Add optional repeatable `api_key_id` query parameter to `GET /api/reports`. +- Filter all report aggregations (summary, daily rows, model distribution, account distribution) by `RequestLog.api_key_id`. +- Add API key multi-select dropdown to the Reports dashboard filter bar. + +## Capabilities + +### Modified Capabilities +- `reports`: Add `api_key_id` filtering support across endpoint and UI. diff --git a/openspec/changes/reports-api-key-filter/specs/reports/spec.md b/openspec/changes/reports-api-key-filter/specs/reports/spec.md new file mode 100644 index 0000000000..1d58e52f16 --- /dev/null +++ b/openspec/changes/reports-api-key-filter/specs/reports/spec.md @@ -0,0 +1,18 @@ +## ADDED Requirements + +### Requirement: API Key Filtering + +The `GET /api/reports` endpoint MUST support filtering report aggregations by one or more `api_key_id` query parameters. + +#### Scenario: Single API Key filter applied +- **WHEN** `GET /api/reports` is requested with `api_key_id=key-1` +- **THEN** all summary, daily breakdown, model distribution, and account distribution metrics SHALL only include request logs generated by `key-1`. + +#### Scenario: Multiple API Key filters applied +- **WHEN** `GET /api/reports` is requested with `api_key_id=key-1&api_key_id=key-2` +- **THEN** report metrics SHALL include request logs matching either `key-1` or `key-2`. + +#### Scenario: Default all API Keys when unselected +- **WHEN** `GET /api/reports` is requested without any `api_key_id` query parameter +- **THEN** report metrics SHALL aggregate across all API keys without filtering by key ID. + diff --git a/openspec/changes/reports-api-key-filter/tasks.md b/openspec/changes/reports-api-key-filter/tasks.md new file mode 100644 index 0000000000..870f9b40b7 --- /dev/null +++ b/openspec/changes/reports-api-key-filter/tasks.md @@ -0,0 +1,9 @@ +## Tasks + +- [x] Backend: Add `api_key_id` query param to `GET /api/reports` in `app/modules/reports/api.py`. +- [x] Backend: Thread `api_key_ids` through `ReportsService` in `app/modules/reports/service.py`. +- [x] Backend: Add `RequestLog.api_key_id.in_(...)` condition to `_report_conditions` and bypass rollup when filtered in `app/modules/reports/repository.py`. +- [x] Frontend: Extend `ReportsParams` in `frontend/src/features/reports/api.ts`. +- [x] Frontend: Add API key `MultiSelectFilter` in `reports-filters.tsx` and wire options in `reports-page.tsx`. +- [x] Frontend: Forward `apiKeyId` in `useReports` hook (`use-reports.ts`). +- [x] Tests: Add backend route test for `api_key_id` filtering and frontend filter component test. diff --git a/openspec/changes/require-pending-call-resolution-after-stale-anchor/design.md b/openspec/changes/require-pending-call-resolution-after-stale-anchor/design.md deleted file mode 100644 index 9c5b47e1c4..0000000000 --- a/openspec/changes/require-pending-call-resolution-after-stale-anchor/design.md +++ /dev/null @@ -1,64 +0,0 @@ -## Context - -The durable session already owns the minimum immutable authority needed for a -safe decision: API-key scope, account owner, latest response anchor, stored -input prefix fingerprint, and pending tool-call manifest. It deliberately -does not retain conversation text. The existing full-resend verifiers compare -an incoming complete request with that authority and are the only components -allowed to authorize a one-shot unanchored same-account recovery. - -## Decision - -Add one marker to the existing `http_bridge_sessions` owner rather than a new -state machine or receipt table. The marker stores only the rejected anchor's -SHA-256 digest, the time it was observed, and at most one exact recovery wire -fingerprint. Its owner is the session's existing `(api_key_scope, account_id)` -identity. A fenced compare-and-set can bind `null -> wire fingerprint` only -while the durable row still owns the exact plaintext anchor that upstream -rejected. The same wire is idempotent; a different otherwise-valid wire fails -closed for that marker generation. - -Request admission reads the marker through the existing durable lookup. When -the digest still matches `latest_response_id`, an incoming request must satisfy -one of the existing exact owner-bound full-resend proofs. Otherwise admission -returns a stable non-retryable semantic error before session selection, -WebSocket connect, or send. The marker is not time-expired independently of -the durable row: ordinary lease expiry must not make a known bad anchor eligible -for upstream redispatch. - -When a verified recovery reaches `response.completed`, the existing atomic -anchor-registration transaction publishes the replacement response id and -clears the marker and its wire claim. Failure before terminal completion -leaves the rejected anchor, marker, and wire claim intact. - -## Constraints - -- No request body, message content, tool arguments, credentials, or raw new - identifiers are persisted by the marker. -- The marker does not authorize replay and cannot change accounts. -- It does not weaken stored-prefix, agent-message, pending-call, boundary, or - ambiguous-delivery proof requirements. -- A different account, API-key scope, anchor, or owner epoch cannot set or - clear another session's marker. - -## Failure modes - -- If the fenced marker write fails or ownership changes, the current request - remains fail-closed through the existing persistence error path. -- If the marker columns are unavailable during a rolling upgrade, schema - migration remains a deployment prerequisite; code does not emulate the - marker in process memory. -- If replacement terminal persistence fails, the marker remains set and later - delta requests remain locally blocked. - -## Example - -An automation submits a user delta while the durable row still references an -upstream anchor whose completed turn contains one undelivered tool call. -Upstream rejects that proxy-injected anchor. The gateway records only the -anchor digest and time, then returns -`previous_response_pending_call_resolution_required`. The next scheduled -delta receives the same local semantic error without opening a WebSocket. A -later exact complete-context resend proves the pending call was never accepted, -recovers once on the owning account, and clears the marker only when the new -response anchor is committed. diff --git a/openspec/changes/require-pending-call-resolution-after-stale-anchor/proposal.md b/openspec/changes/require-pending-call-resolution-after-stale-anchor/proposal.md deleted file mode 100644 index d0b5a7cc2f..0000000000 --- a/openspec/changes/require-pending-call-resolution-after-stale-anchor/proposal.md +++ /dev/null @@ -1,34 +0,0 @@ -## Why - -A long-lived scheduled Codex task can repeatedly submit delta-only follow-ups -after upstream has rejected the proxy-injected durable response anchor. The -first rejection is fail-closed, but the rejection is currently remembered only -by the process-local quarantine. After a bridge replacement, worker restart, -or quarantine expiry, the same durable anchor can be injected and rejected -again, so an automation can spend every wake-up rediscovering an already known -unrecoverable pending-call boundary. - -## What Changes - -- Persist a content-free, owner- and anchor-bound recovery-required marker on - the durable HTTP bridge session when upstream first rejects a proxy-injected - anchor and the request lacks a verified complete-context proof. -- Reject later unverified requests for that exact durable owner and anchor - locally, before any upstream connection or dispatch, with a stable semantic - error that requires pending-call resolution. -- Preserve the existing exact owner-bound full-resend and abandoned-pending - proofs as the only paths allowed to recover the anchor on the same account. -- Clear the marker atomically when a successful replacement terminal response - publishes its new durable anchor. -- Keep the rejected anchor, stored prefix fingerprint, and pending-call - manifest intact; do not persist request text or introduce another proof - verifier. - -## Impact - -- Scheduled delta wake-ups stop consuming upstream connections once the stale - anchor is already known to require complete-context recovery. -- The recovery requirement survives process replacement and ordinary bridge - lease expiry because it is durable session state. -- Existing PR #19 ambiguous-delivery exactly-once behavior is unchanged: no - request is replayed from an ambiguous receive path. diff --git a/openspec/changes/require-pending-call-resolution-after-stale-anchor/specs/responses-api-compat/spec.md b/openspec/changes/require-pending-call-resolution-after-stale-anchor/specs/responses-api-compat/spec.md deleted file mode 100644 index 57ae25c3f8..0000000000 --- a/openspec/changes/require-pending-call-resolution-after-stale-anchor/specs/responses-api-compat/spec.md +++ /dev/null @@ -1,75 +0,0 @@ -## ADDED Requirements - -### Requirement: Rejected durable anchors require local pending-call resolution - -When upstream rejects a proxy-injected `previous_response_id` before any -response event and the request does not satisfy an existing exact owner-bound -complete-context proof, the gateway MUST atomically persist a content-free -recovery-required marker on the durable session. The marker MUST bind the -session's API-key scope, owning account, and rejected anchor, MUST survive -process replacement and ordinary bridge lease expiry, and MUST NOT store -request or conversation content. - -While that marker still matches the durable owner and latest response anchor, -the gateway MUST reject every request that lacks an existing verified -owner-bound complete-context proof before selecting an upstream account, -opening a WebSocket, or dispatching `response.create`. The response MUST use a -stable non-retry semantic error code that states pending-call resolution is -required. Repeating the same scheduled delta MUST NOT create a new upstream -attempt. - -An existing exact stored-prefix full resend or abandoned-pending-call proof MAY -recover once without the rejected anchor on the same owning account. The -gateway MUST retain the marker and rejected anchor if that attempt does not -reach a replacement terminal checkpoint. Publishing the replacement response -anchor after `response.completed` MUST clear the marker in the same durable -transaction. The marker MUST NOT authorize account migration, boundary -relaxation, generic replay, or any retry after ambiguous delivery. - -#### Scenario: repeated scheduled delta is blocked before upstream dispatch - -- **GIVEN** upstream rejected a proxy-injected durable anchor with no response event -- **AND** the durable row has an unresolved pending-call manifest -- **AND** the first request lacked a verified complete-context proof -- **WHEN** another scheduled delta targets the same API-key scope, owner, and anchor -- **THEN** the gateway returns `previous_response_pending_call_resolution_required` -- **AND** does not select an upstream account, connect a WebSocket, or dispatch the request - -#### Scenario: marker survives owner process replacement and lease expiry - -- **GIVEN** a rejected-anchor marker is durable -- **WHEN** the bridge lease expires or another worker reads the session -- **THEN** the marker remains bound to the same account and anchor -- **AND** the new worker locally rejects an unverified delta - -#### Scenario: exact complete-context recovery clears the marker atomically - -- **GIVEN** a durable rejected-anchor marker and pending-call manifest -- **AND** a later request satisfies the existing exact owner-bound abandoned-pending proof -- **WHEN** the same-account recovery reaches `response.completed` -- **THEN** the new response anchor and full input fingerprint are published -- **AND** the rejected-anchor marker is cleared in that same durable transaction -- **AND** a later delta continues from the new response anchor - -#### Scenario: different valid recoveries share one durable dispatch generation - -- **GIVEN** two concurrent requests contain different wire payloads that each satisfy an exact owner-bound recovery proof -- **WHEN** both observe the durable marker before either reaches `response.create` -- **THEN** one request atomically binds its exact wire fingerprint to the marker generation and may dispatch -- **AND** the other request fails closed before an irreversible upstream effect -- **AND** exactly one replacement anchor may be published - -#### Scenario: ambiguous marker recovery is not reclaimed after process loss - -- **GIVEN** a marker-authorized recovery has an `UNKNOWN` durable attempt -- **AND** its owner process disappears or its bridge lease expires before a terminal checkpoint -- **WHEN** the same verified request reaches another worker -- **THEN** the worker fails closed before upstream connect or dispatch -- **AND** it does not claim, replay, or replace the existing attempt generation - -#### Scenario: owner, anchor, and exactly-once fences remain closed - -- **GIVEN** a marker for one API-key scope, account, and response anchor -- **WHEN** a request targets a different owner or anchor, supplies a mismatched tool output, or follows an ambiguous receive failure -- **THEN** the marker does not authorize recovery or cross-account dispatch -- **AND** all existing fail-closed and no-duplicate contracts remain in force diff --git a/openspec/changes/require-pending-call-resolution-after-stale-anchor/tasks.md b/openspec/changes/require-pending-call-resolution-after-stale-anchor/tasks.md deleted file mode 100644 index 7dcb4e5d1d..0000000000 --- a/openspec/changes/require-pending-call-resolution-after-stale-anchor/tasks.md +++ /dev/null @@ -1,19 +0,0 @@ -## 1. Durable authority - -- [x] Add an additive migration and model fields for the content-free rejected-anchor marker. -- [x] Add fenced repository/coordinator operations to set and read the marker. -- [x] Clear the marker atomically with replacement terminal anchor registration. - -## 2. Admission and failure semantics - -- [x] Persist the marker on the first unproved proxy-injected stale-anchor rejection. -- [x] Reject subsequent unproved owner/anchor-bound requests before upstream connect/send. -- [x] Preserve existing verified same-owner full-resend recovery and PR #19 no-duplicate behavior. - -## 3. Verification - -- [x] Add durable cross-process/lease-expiry, concurrent marking, owner/anchor fencing, and clearing tests. -- [x] Add an HTTP regression with stored prefix, pending manifest, developer/user messages, and mismatched tool output. -- [x] Prove the second scheduled-style delta performs zero upstream connection or dispatch. -- [x] Prove a verified replacement terminal checkpoint clears the marker and later deltas continue from the new anchor. -- [x] Run focused unit/integration, replay-safety, migration, format, lint, type, architecture, and strict OpenSpec gates. diff --git a/openspec/changes/resolve-rejected-anchor-pending-calls/design.md b/openspec/changes/resolve-rejected-anchor-pending-calls/design.md deleted file mode 100644 index 10d5facb10..0000000000 --- a/openspec/changes/resolve-rejected-anchor-pending-calls/design.md +++ /dev/null @@ -1,68 +0,0 @@ -## Context - -`http_bridge_sessions` already binds the API-key-scoped logical session, -account, latest completed response anchor, exact stored-input count and -fingerprint, pending-call manifest, and rejected-anchor marker. Recovery uses -an owner-bound immutable proof, a marker wire compare-and-set, and the -`http_bridge_recovery_attempts` journal. - -The upstream reader publishes `latest_response_id` only after -`response.completed`; an incomplete stream never becomes the durable anchor. -The observed rejection therefore concerns an earlier completed response. In -the failing complete resend, the client physically retained the matching call -and output but also retained later bounded inputs. - -## Decision - -Keep the existing recovery state machine and extend only its exact settlement -predicate. - -The predicate scans suffix prefixes and accepts the shortest self-contained -tool loop whose call and output maps both equal the durable pending manifest. -It compares the exact call-id-to-type bijection. The existing self-contained -request validator enforces each call/output ordering and rejects duplicates; -parallel call insertion order is not authority because the durable manifest is -canonically sorted. A later tool-like item, unrelated call, missing output, -unsupported status, or malformed caller cannot participate in the proof. - -Once exact settlement is complete, the tail may be empty. Otherwise it must -be either a bounded fresh user sequence, or one canonical retained -`agent_message` followed by that bounded sequence. Existing developer -interleave compatibility remains restricted to its original exact three-item -call/developer/output window and cannot authorize a later follow-up. - -All upstream admission still requires the same owner account, exact stored -prefix, exact full-input fingerprint, and durable marker. The marker path -continues to claim one exact wire fingerprint and one journal attempt. A new -terminal anchor stores the original complete-input checkpoint and clears the -marker through the existing transaction. Failures and `UNKNOWN` attempts keep -the marker and remain non-replayable. - -Rows with any rejected-anchor marker are conservatively excluded from ordinary -startup, closed-row, and abandoned-row deletion. Terminal anchor publication -already clears the marker, after which normal retention resumes. - -## Rejected alternatives - -- **Operator abandonment receipt:** unnecessary for the observed shapes and - introduces a new privilege and contract-fingerprint surface. It remains a - separate governed change if a future incident has no exact settlement proof. -- **Reconstruct deleted rows from logs:** logs do not contain durable ownership - authority and cannot safely authorize replay. -- **Retry or capacity increase:** neither repairs the deterministic proof - rejection and both risk repeated work. - -## Rollout and rollback - -This change adds no schema column or endpoint. Rollback restores the narrower -predicate and prior cleanup behavior. Existing marker/journal rows remain -compatible in either direction. - -If an explicit stale anchor arrives after its durable row was already purged, -`durable_lookup` is absent and no sealed stored-prefix or pending-manifest proof -can be constructed. The gateway continues fail closed; it MUST NOT infer -authority from logs or from a merely full-resend-shaped input. A separate -change must give that condition a stable `recovery_authority_missing` outcome, -pause scheduled retries, and define a client-owned or separately privileged -same-logical-session checkpoint import before those historical sessions can be -recovered. diff --git a/openspec/changes/resolve-rejected-anchor-pending-calls/proposal.md b/openspec/changes/resolve-rejected-anchor-pending-calls/proposal.md deleted file mode 100644 index 0ba1834d48..0000000000 --- a/openspec/changes/resolve-rejected-anchor-pending-calls/proposal.md +++ /dev/null @@ -1,43 +0,0 @@ -## Why - -The durable stale-anchor marker correctly blocks repeated sends after an -upstream owner rejects a saved response anchor. Its complete-resend verifier, -however, recognizes an exact pending call/output settlement only when that pair -is the entire suffix. Real Codex transports retain canonical response-owned -reasoning and may append a bounded user or inter-agent follow-up after the exact -call/output pair. Those requests contain physical settlement evidence but are -rejected and then remain locally blocked by the marker. - -Ordinary retention cleanup can also delete a marker-bearing session row. That -turns a recoverable, physically bound state into missing authority and defeats -the fail-closed recovery contract. - -## What Changes - -- Recognize the shortest suffix prefix that exactly and uniquely settles every - durable pending call by call id and call type. -- After that exact settlement only, accept either no tail, a bounded user - follow-up sequence, or one canonical retained `agent_message` followed by a - bounded user sequence. -- Reject missing, duplicate, call/output-order-invalid, type-drifted, - unrelated, or additional tool loops. A generic developer/user boundary is - not settlement proof. Parallel calls are compared as the durable exact - ID-to-type bijection because the persisted manifest is canonically sorted. -- When a pending manifest is non-empty, remove the broader retained-output - alternative: admission must use the exact manifest settlement path. -- Keep the existing owner/account/stored-prefix proof, marker wire CAS, and - durable recovery-attempt journal unchanged. -- Exempt marker-bearing rows from ordinary startup, closed-row, and abandoned - retention purges until a replacement terminal anchor clears the marker. - -## Impact - -- The three observed sanitized transport shapes become recoverable in place - without a new operator API, credential class, client signature, or state - machine. -- No unknown tool call is abandoned or replayed. Recovery authority comes from - the client's exact call/output settlement and the existing durable manifest. -- Historical rows already deleted by cleanup are not reconstructed from logs. - An explicit stale-anchor request with no durable lookup has no stored-prefix - or pending-manifest authority, so this change does not unanchor it. Those - sessions require a separate governed checkpoint-rehydration contract. diff --git a/openspec/changes/resolve-rejected-anchor-pending-calls/specs/responses-api-compat/spec.md b/openspec/changes/resolve-rejected-anchor-pending-calls/specs/responses-api-compat/spec.md deleted file mode 100644 index b5d670148d..0000000000 --- a/openspec/changes/resolve-rejected-anchor-pending-calls/specs/responses-api-compat/spec.md +++ /dev/null @@ -1,73 +0,0 @@ -## MODIFIED Requirements - -### Requirement: Responses stale-anchor recovery uses exact settled pending calls - -When a durable rejected-anchor marker has a non-empty pending-call manifest, -the gateway MUST admit a complete-context recovery only if the suffix contains -a self-contained call/output settlement whose call-id-to-type maps exactly -equal that durable manifest. Canonical response-owned reasoning MAY precede -the settlement because the existing replay projection removes it. After the -exact settlement, the gateway MAY accept no tail, a bounded user follow-up -sequence, or one canonical retained `agent_message` followed by such a -sequence. - -Missing, duplicate, call/output-order-invalid, type-drifted, malformed, -unrelated, or extra tool loops MUST fail closed. Parallel calls MUST match the -durable exact ID-to-type bijection; their insertion order is not authority. A -developer/user boundary or a different -call/output pair MUST NOT prove settlement. A non-empty pending manifest MUST -NOT use the broader retained-output alternative. - -Admission MUST retain the existing API-key-scoped logical session, same-account -owner, exact stored-prefix, complete-input fingerprint, marker wire -compare-and-set, and durable recovery-attempt journal checks. The gateway MUST -send at most one unanchored replacement wire. An ambiguous or failed attempt -MUST retain the marker and MUST NOT be reclaimed. A successful terminal -checkpoint MUST publish the new anchor against the original complete input and -clear the marker atomically through the existing terminal transaction. - -Ordinary startup, closed-row, and abandoned-row cleanup MUST NOT delete a row -while it carries a rejected-anchor marker. Once terminal publication clears -the marker, normal retention MAY delete the row. - -#### Scenario: exact settlement followed by bounded user input recovers once - -- **GIVEN** the durable manifest contains pending `call_A` -- **AND** the exact complete resend contains canonical reasoning, `call_A`, its - matching output, and bounded later user input -- **WHEN** stale-anchor recovery runs -- **THEN** the gateway dispatches one same-account unanchored recovery -- **AND** does not re-execute `call_A` -- **AND** terminal completion publishes the replacement checkpoint once - -#### Scenario: exact settlement followed by inter-agent output recovers once - -- **GIVEN** the exact complete resend settles every durable pending call -- **AND** one canonical retained `agent_message` and bounded user input follow -- **WHEN** stale-anchor recovery runs -- **THEN** the response-owned bookkeeping is projected safely -- **AND** the complete client context remains bound to the terminal checkpoint - -#### Scenario: unrelated or additional calls remain blocked - -- **GIVEN** the durable manifest contains pending `call_A` -- **WHEN** the suffix contains `call_B`, a missing or wrong output, a type or - call/output ordering drift, a second tool loop, or only developer/user input -- **THEN** the gateway rejects before upstream connect or send -- **AND** retains the marker and pending manifest - -#### Scenario: ambiguous recovery is not replayed - -- **GIVEN** an admitted recovery may have reached upstream and its journal is - `UNKNOWN` -- **WHEN** an identical or different wire arrives after restart or lease expiry -- **THEN** the gateway does not reclaim or redispatch it -- **AND** the historical irreversible effect count remains unchanged - -#### Scenario: cleanup preserves recovery authority - -- **GIVEN** a marker-bearing row is closed, ownerless, lease-expired, and older - than ordinary retention cutoffs -- **WHEN** startup, closed-row, or abandoned-row cleanup runs -- **THEN** the row, marker, pending manifest, and journal authority remain -- **AND** normal purge eligibility resumes only after marker clear diff --git a/openspec/changes/resolve-rejected-anchor-pending-calls/tasks.md b/openspec/changes/resolve-rejected-anchor-pending-calls/tasks.md deleted file mode 100644 index b8841a9983..0000000000 --- a/openspec/changes/resolve-rejected-anchor-pending-calls/tasks.md +++ /dev/null @@ -1,29 +0,0 @@ -## 1. Exact settlement contract - -- [x] Add a sanitized, content-free fixture for all three observed suffix - shapes and reproduce the legacy rejection. -- [x] Accept bounded follow-ups only after exact durable call-id/type settlement. -- [x] Reject unrelated output ids and a second tool loop after settlement. -- [x] Add negatives for partial, duplicate, call/output-order-invalid, - type/status drift, - malformed agent messages, and generic developer/user tails. -- [x] Require exact settlement instead of the broader retained-output path when - the durable manifest is non-empty. - -## 2. Durable retention and exactly-once behavior - -- [x] Exclude marker-bearing rows from startup, closed, and abandoned purges. -- [x] Add repository behavior coverage for all three cleanup paths, process - replacement, and normal purge after terminal marker clear. -- [x] Add HTTP integration for all three sanitized shapes proving same-account - unanchored send, one marker/journal claim under concurrency, unchanged - historical tool-effect count, and terminal checkpoint binding to the - original complete input. - -## 3. Verification - -- [x] Prove statically that incomplete streams do not publish a new durable - response anchor. -- [x] Run focused replay-safety, durable repository, streaming integration, - architecture, Ruff, Ruff format, ty, diff-check, and strict OpenSpec - validation. diff --git a/openspec/changes/route-model-sources-off-websocket/proposal.md b/openspec/changes/route-model-sources-off-websocket/proposal.md new file mode 100644 index 0000000000..2abab883a2 --- /dev/null +++ b/openspec/changes/route-model-sources-off-websocket/proposal.md @@ -0,0 +1,36 @@ +## Why + +Model sources are only consulted by the HTTP request handlers. The WebSocket +session path goes straight to subscription-account selection, so a model served +by an enabled OpenAI-compatible source is dispatched to a ChatGPT account and +rejected upstream with: + +``` +The '' model is not supported when using Codex with a ChatGPT account. +``` + +`docs/client-setup.md` documents `supports_websockets = true`, so model sources +are unusable with the documented Codex client configuration. See #1658. + +## What Changes + +- Extract the shared model-source resolution helpers into + `app/modules/model_sources/selection.py` so the HTTP and WebSocket paths agree + on which models belong to a source. +- Fail the WebSocket connect with `model_source_requires_http_transport` when the + requested model resolves to an enabled Responses-capable source, instead of + selecting a subscription account. +- Emit the failure as a `503` connect failure. Codex clients fall back to the + HTTP transport only on service-level connect failures; a `4xx` is treated as + terminal and surfaces to the user. After the fallback, the HTTP path routes to + the model source normally. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `responses-api-compat` diff --git a/openspec/changes/route-model-sources-off-websocket/specs/responses-api-compat/spec.md b/openspec/changes/route-model-sources-off-websocket/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..c035c8c1ca --- /dev/null +++ b/openspec/changes/route-model-sources-off-websocket/specs/responses-api-compat/spec.md @@ -0,0 +1,114 @@ +## ADDED Requirements + +### Requirement: Source-owned models are not served over the WebSocket transport + +Model sources are reachable only from the HTTP request path. When a WebSocket +Responses session requests a model that resolves to an enabled, +Responses-capable OpenAI-compatible model source, the system SHALL NOT dispatch +the request to a subscription account. + +The check SHALL be applied on the connect path before account selection, and +SHALL also be applied to every prepared `response.create`, so that a turn which +switches to a source-owned model on an already-open subscription upstream is +also rejected instead of being forwarded. + +Both checks SHALL evaluate the client's raw requested model, captured before +API-key enforcement normalizes model aliases (for example `gpt-5-high` to +`gpt-5`), alongside the normalized model — the same candidate list the HTTP +handlers build from `raw_source_model`, including substituting the API key's +`enforced_model` and, when fast mode is prohibited and the raw model is a +fast-mode alias, replacing the raw candidate with the normalized model. A +source that exposes an alias-named model MUST be matched on the WebSocket +transport whenever the HTTP path would route to it. + +Both checks SHALL apply only to requests that are eligible for model-source +routing on the HTTP request path, judged on the full client input before any +WebSocket-specific trimming or anchor injection. A request whose input ends +with a terminal `compaction_trigger` item, or that references uploaded files +(`input_file` / file-backed `input_image` items), is excluded from source +routing over HTTP — the former is served by the upstream compact flow on the +turn's owner account, the latter is pinned to the subscription account that +received the upload — and MUST NOT be failed by either WebSocket guard even +when its model also resolves to an enabled source. Such requests proceed to +subscription account selection and the owner-routing rules, exactly as they +would after the HTTP route skips source selection. A malformed compaction +trigger (repeated, or not the final top-level input item) SHALL keep the +guards active: the HTTP route rejects that payload with a 400, the WebSocket +path forwards it verbatim, and the exclusion changes neither. + +Both failures MUST use error code `model_source_requires_http_transport`. On the +connect path the failure MUST be emitted as a service-level connect failure +(HTTP status `503`), so that Codex clients fall back to the HTTP transport, +where source routing is applied. For a prepared `response.create` on an +established session the failure MUST be emitted as a terminal error for that +turn, and any usage reservation held for the turn MUST be released. + +When source resolution is unavailable, the WebSocket transport MUST fall back to +subscription account selection rather than failing the request. The resolution +runs after a turn's usage reservation is acquired but before it is registered +for cleanup, so a propagating failure would end the session and strand the +reservation; the degraded behaviour is the pre-change one, where the +subscription upstream rejects the model. This applies to the WebSocket transport +only — the HTTP request path MUST continue to surface resolution failures, since +silently routing source traffic to a subscription account would be worse there. + +#### Scenario: Source-owned model over WebSocket fails the connect + +- **GIVEN** an enabled OpenAI-compatible model source exposes model `m` with Responses support +- **WHEN** a client opens a WebSocket Responses session requesting model `m` +- **THEN** the system fails the connect with error code `model_source_requires_http_transport` +- **AND** no subscription account is selected for the request + +#### Scenario: Later turn switching to a source-owned model is rejected + +- **GIVEN** a WebSocket Responses session already has an open subscription-account upstream +- **AND** an enabled OpenAI-compatible model source exposes model `m` with Responses support +- **WHEN** a subsequent `response.create` requests model `m` +- **THEN** the system emits a terminal error with code `model_source_requires_http_transport` +- **AND** the frame is not forwarded to the subscription account on the open upstream +- **AND** the turn's usage reservation is released + +#### Scenario: An alias-named source model is rejected despite normalization + +- **GIVEN** an enabled OpenAI-compatible model source exposes model `gpt-5-high` with Responses support +- **AND** an API key whose `allowed_models` contains exactly `gpt-5-high` +- **WHEN** the key sends a WebSocket `response.create` for `gpt-5-high`, which enforcement normalizes to `gpt-5` +- **THEN** the source-ownership check also considers the raw `gpt-5-high` candidate +- **AND** the request is rejected with `model_source_requires_http_transport` on the connect path and on socket reuse alike + +#### Scenario: A file-referencing turn is dispatched to its pinned account, not failed + +- **GIVEN** a WebSocket Responses session already has an open subscription-account upstream +- **AND** a later `response.create` references an uploaded `input_file` pinned to that account +- **AND** the request's model is also exposed by an enabled model source +- **WHEN** the turn is prepared for the open socket +- **THEN** the reuse guard does not fail the turn with `model_source_requires_http_transport` +- **AND** the turn is forwarded to the pinned subscription account + +#### Scenario: A terminal compaction trigger is not failed by the WebSocket guards + +- **GIVEN** a `response.create` whose final top-level input item is a `compaction_trigger` +- **AND** the request's model is also exposed by an enabled model source +- **WHEN** the request reaches the connect path or an already-open subscription upstream +- **THEN** neither WebSocket guard fails the request with `model_source_requires_http_transport` +- **AND** the connect path proceeds to subscription account selection, and an open upstream receives the turn + +#### Scenario: An API key that enforces a source-owned model is rejected + +- **GIVEN** an API key whose `enforced_model` resolves to an enabled model source +- **WHEN** the key opens a WebSocket Responses session requesting any model +- **THEN** the enforced model is resolved against the model sources +- **AND** the session fails with `model_source_requires_http_transport` + +#### Scenario: Subscription models are unaffected + +- **GIVEN** a model that is not served by any enabled model source +- **WHEN** a client opens a WebSocket Responses session requesting that model +- **THEN** account selection proceeds unchanged + +#### Scenario: Source resolution failure falls back to subscription selection + +- **GIVEN** the model-source catalog cannot be read +- **WHEN** a client opens a WebSocket Responses session +- **THEN** account selection proceeds as it did before the guard existed +- **AND** the session is not terminated by the resolution failure diff --git a/openspec/changes/route-model-sources-off-websocket/tasks.md b/openspec/changes/route-model-sources-off-websocket/tasks.md new file mode 100644 index 0000000000..9dd91f6f03 --- /dev/null +++ b/openspec/changes/route-model-sources-off-websocket/tasks.md @@ -0,0 +1,36 @@ +## Tasks + +- [x] Extract `select_responses_model_source` / `allowed_source_ids_for_api_key` + into `app/modules/model_sources/selection.py`; delegate from + `app/modules/proxy/api.py`. +- [x] Add `responses_model_is_source_owned` for transport-level checks. +- [x] Guard `_select_websocket_connect_account` so source-owned models fail the + WebSocket connect instead of selecting a subscription account. +- [x] Return `503` so the Codex client falls back to the HTTP transport. +- [x] Consider the API key's `enforced_model` in the guard, matching the + candidate list the HTTP handlers build. +- [x] Add spec delta for `responses-api-compat`. +- [x] Cover the guard and the source-owned check with unit and integration + tests, including the `require_streaming` edge and the enforced-model case. +- [x] Apply the guard to every prepared `response.create` so socket reuse cannot + forward a source-owned model to the open subscription upstream (Codex P2). +- [x] Fail open to subscription selection when source resolution raises, so a + database failure cannot end the session or strand a usage reservation. +- [x] Evaluate the connect guard once per connect series instead of per failover + attempt, and judge it with the per-request api key rather than the + session key. +- [x] Gate the reuse guard on a live upstream reader so a socket that died + between turns reconnects into the 503 fallback instead of receiving a + terminal error. +- [x] Finalize the request-log row on the reuse-guard path. +- [x] Carry the client's raw model (pre alias normalization) through request + preparation and feed it to the source-ownership check on both the + connect and reuse paths, so an alias-only source behind an alias + allowlist matches like it does over HTTP (Codex P2). +- [x] Preserve the HTTP source-routing exclusions in both WebSocket guards: + extract the HTTP gate into `responses_source_route_excluded`, stamp it + on the prepared request state, and skip the guards for terminal + compaction triggers and `input_file`-referencing requests so they + dispatch to their (owner-pinned) subscription account instead of + failing with `model_source_requires_http_transport` (Codex P2). + diff --git a/openspec/changes/scope-codex-affinity-by-thread/.openspec.yaml b/openspec/changes/scope-codex-affinity-by-thread/.openspec.yaml new file mode 100644 index 0000000000..5081c98763 --- /dev/null +++ b/openspec/changes/scope-codex-affinity-by-thread/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-12 diff --git a/openspec/changes/scope-codex-affinity-by-thread/design.md b/openspec/changes/scope-codex-affinity-by-thread/design.md new file mode 100644 index 0000000000..a3125eb2d5 --- /dev/null +++ b/openspec/changes/scope-codex-affinity-by-thread/design.md @@ -0,0 +1,77 @@ +## Context + +Upstream Codex deliberately separates three roles: `session-id` identifies a +root process tree, `thread-id` identifies one root/child/fork/resumed thread, +and `prompt_cache_key` defaults to the shared process session to colocate +cache entries. The old codex-lb bridge assumption that explicit cache keys +distinguish children stopped holding when upstream adopted the shared cache +key. + +## Decisions + +1. **Use a typed thread identity.** Parse process session and `thread-id` + independently. Derive a versioned, header-inaccessible opaque key from both, + with a separately namespaced thread-only fallback when no process session + exists. Never infer identity from subagent or parent markers. +2. **Reuse bounded prompt-cache rows for soft locality.** Thread locality uses + the existing `prompt_cache` kind and configured freshness window. A missing + thread row first prefers the eligible source-separated process-session row; + successful admission persists the selected account under the thread key + without rewriting the process row. Because current Codex includes + `thread-id` on its first root request, the first admitted thread also + initializes a missing process preference with an atomic insert-if-absent. + Later thread movement can neither overwrite that first-writer default nor + mutate a sibling row. A provisional recovery-probe reservation persists + only its reversible thread row; it cannot publish the immutable process + default until a normal admission does so, because a failed probe CAS cannot + safely delete a seed that a concurrent sibling may already have observed. +3. **Keep ownership separate from locality.** Raw legacy `codex_session` rows + are looked up independently and remain hard. Exact turn state, response, + file, conversation, bridge, replay, and reattach evidence keeps its existing + precedence and conflict behavior. +4. **Use the same logical identity at transport boundaries.** Direct + WebSocket retained replay/tool state is count-bounded in memory by thread. + An HTTP bridge canonical lane is hard while live/durable, but its pre-bridge + account hint remains soft and bounded. +5. **Migrate bridge lanes only through exact aliases.** With current Codex, + legacy `(session-id, prompt_cache_key)` is shared by siblings. A request + carrying `thread-id` must not fall back to that canonical key. Existing + lanes remain recoverable through exact turn-state or previous-response + aliases and otherwise expire naturally. Authenticated forwarded affinity + keys are accepted verbatim and never derived again. +6. **Preserve upstream cache intent.** `prompt_cache_key` is forwarded + unchanged. Request-log conversation grouping continues using raw + `thread-id`. + +## Rejected Alternatives + +- Adapting the marker/TTL/schema design in #1309: thread identity is already + explicit, so marker inference, a migration, settings, and dashboard controls + add lifecycle without improving identity. +- Using subagent or parent-thread markers: they describe role/provenance and + can group siblings rather than identify the current thread. +- Using `prompt_cache_key` or `(session-id, prompt_cache_key)`: current Codex + intentionally sends the same value for root and children. +- Rewriting `prompt_cache_key` to `thread-id`: this defeats upstream's intended + tree-wide cache colocation. +- Balancing every unseen thread independently: a thread boundary contains + justified divergence; it is not a reason to discard the healthy process + preference on first placement. +- Durable per-thread Codex rows or a new sticky kind/setting: existing bounded + rows already express soft locality, while bridge/object ownership remains + durable separately. +- Falling back to the old bridge canonical key when `thread-id` exists: that + can attach a sibling's history. Only exact hard aliases are safe migration + evidence. + +## Risks / Trade-offs + +- Thread rows expire. Selection and active response completion refresh the row + so a long-lived active thread does not lose reconnect locality solely to the + freshness window. +- A health, quota, explicit restart, or proven hard-owner transition may still + move a thread. The fix removes cross-thread collisions; it does not promise + an account never changes. +- Mixed-version replicas may retain old shared bridge rows. New requests with + a thread identity ignore those rows unless an exact alias proves ownership, + allowing safe coexistence until cleanup. diff --git a/openspec/changes/scope-codex-affinity-by-thread/proposal.md b/openspec/changes/scope-codex-affinity-by-thread/proposal.md new file mode 100644 index 0000000000..0014440f6d --- /dev/null +++ b/openspec/changes/scope-codex-affinity-by-thread/proposal.md @@ -0,0 +1,38 @@ +## Why + +Current Codex sends one process `session-id` and one `prompt_cache_key` across +an entire root/subagent tree while giving each logical conversation its own +stable `thread-id`. codex-lb still keys backend account locality, direct +WebSocket replay state, and HTTP bridge lanes primarily from the shared +process/cache identities. Sibling threads therefore overwrite one another's +locality and can reuse replay or bridge history that belongs to another +thread. + +## What Changes + +- Derive one source-separated bounded locality key from the process session + and `thread-id` for backend Responses and compact requests. +- Seed a new thread from an eligible process-session preference, then persist + only the thread-local bounded row so later failover does not move siblings. +- Key direct WebSocket retained state and HTTP bridge canonical lanes by the + same logical thread identity. +- Route thread-goal operations from their payload `threadId`. +- Preserve explicit turn state, previous response, file, conversation, bridge, + replay, and legacy raw Codex rows as hard ownership; keep `prompt_cache_key` + unchanged as the upstream cache hint. + +## Capabilities + +### Modified Capabilities + +- `responses-api-compat`: Scope backend Responses, compact, direct WebSocket, + HTTP bridge, and thread-goal locality by Codex thread identity. +- `sticky-session-operations`: Keep process preference and legacy hard-owner + semantics while introducing bounded per-thread locality. + +## Impact + +The change touches proxy affinity parsing, account selection, direct +WebSocket continuity, HTTP bridge identity, thread-goal routing, and focused +tests. It adds no setting, schema, migration, dashboard surface, or upstream +payload rewrite. diff --git a/openspec/changes/scope-codex-affinity-by-thread/specs/responses-api-compat/spec.md b/openspec/changes/scope-codex-affinity-by-thread/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..e9d3df3d81 --- /dev/null +++ b/openspec/changes/scope-codex-affinity-by-thread/specs/responses-api-compat/spec.md @@ -0,0 +1,221 @@ +## MODIFIED Requirements + +### Requirement: Codex backend session_id preserves account affinity + +When a backend Codex Responses or compact request includes a nonblank +`thread-id`, the service MUST use a source-separated bounded key derived from +the independently parsed process session and thread identity for soft account +locality. If the thread has no mapping, selection MUST first prefer an eligible +source-separated process-session mapping and then persist the admitted thread +mapping. If no process-session mapping exists, the first admitted thread MUST +initialize that soft process preference atomically without overwriting a +concurrent or later first writer, unless its account is admitted only through +a recovery-probe reservation. A recovery-probe admission MUST NOT initialize +the immutable process preference; its reversible thread row MAY be persisted +independently until a normal admission establishes the process default. + +When `thread-id` is absent, a non-empty accepted process-session header MUST +retain its established account-affinity behavior. Accepted process-session +headers are `session_id`, `session-id`, `x-codex-session-id`, and +`x-codex-conversation-id`, in that priority order. A client-supplied nonblank +`x-codex-turn-state` remains a more specific hard continuity key. If the +request lacks a client-supplied `prompt_cache_key`, the service MUST derive and +attach a stable `prompt_cache_key` before upstream forwarding so account +affinity and upstream prompt-cache routing can coexist. A client-supplied +`prompt_cache_key` MUST be forwarded unchanged and MUST NOT be used as thread +identity. + +A turn state synthesized by the proxy for the current downstream WebSocket +handshake MUST NOT override client-supplied process/thread identity or a +prompt-cache key for routing or WebSocket continuity selection. The proxy MUST +seed WebSocket continuity storage under that synthesized turn state so a later +client echo can reuse the completed-turn owner. The proxy MUST continue to +forward that synthesized turn state upstream. A turn state sent by the client, +including one that the proxy generated and the client later echoed, remains a +client-supplied turn-state affinity key. + +When a WebSocket handshake has neither a client-supplied turn state nor an +accepted process/thread identity, the proxy MUST store its generated turn state +as the WebSocket continuity key. A later connection that echoes that accepted +value MUST recover the same continuity state. Direct WebSocket retained +response, input-prefix, Responses Lite, and unresolved-tool state MUST use the +derived thread identity plus API-key scope, with count-bounded storage. +Request-log conversation grouping MUST continue to use raw `thread-id`. + +#### Scenario: Backend Codex request derives prompt_cache_key before codex-session routing + +- **WHEN** `/backend-api/codex/responses` is called with `session_id` and without `thread-id` or `prompt_cache_key` +- **THEN** the routing decision retains process-session `codex_session` affinity +- **AND** the forwarded upstream payload includes a derived stable `prompt_cache_key` + +#### Scenario: backend WebSocket reconnect retains session affinity despite a generated turn state + +- **WHEN** two backend Codex Responses WebSocket connections include the same process session and `thread-id` and omit `x-codex-turn-state` +- **AND** the proxy generates a distinct turn state for each handshake +- **THEN** both account selections use the same bounded thread-local affinity key +- **AND** each generated turn state is still forwarded to the upstream + +#### Scenario: echoed generated turn state remains a client continuation key + +- **WHEN** a client reconnects with a non-empty `x-codex-turn-state` value it received from an earlier proxy handshake +- **THEN** that turn state remains the routing and WebSocket continuity key ahead of broader process/thread locality +- **AND** full-resend continuity for that echoed turn state can reuse the earlier completed response anchor + +#### Scenario: generated turn state seeds continuity without a session header + +- **WHEN** a backend Codex Responses WebSocket handshake omits process/thread identity and `x-codex-turn-state` +- **AND** the proxy generates and returns a turn state for that handshake +- **THEN** the proxy stores its WebSocket continuity state under that generated value +- **AND WHEN** a later connection sends that value in `x-codex-turn-state` +- **THEN** it recovers the stored continuity state + +#### Scenario: Root and child keep separate locality with one cache hint + +- **GIVEN** root and child requests share a process session and explicit `prompt_cache_key` +- **AND** they carry different stable `thread-id` values +- **WHEN** backend Responses or compact routes them +- **THEN** they use different bounded internal thread keys +- **AND** both upstream payloads retain the original `prompt_cache_key` + +#### Scenario: New thread inherits process preference without coupling siblings + +- **GIVEN** a process-session soft row points to eligible account A +- **AND** a previously unseen thread in that process arrives +- **WHEN** selection admits the request +- **THEN** it prefers account A and persists a bounded row for that thread +- **AND** later movement of that thread does not rewrite the process row or a sibling row + +#### Scenario: First thread initializes the process preference + +- **GIVEN** a fresh process has no process-session or thread mapping +- **WHEN** its first thread is admitted on account A +- **THEN** it initializes the process preference to A with insert-if-absent +- **AND** a later sibling prefers A without gaining authority to rewrite that process preference + +#### Scenario: Exact owner admission still initializes first-thread locality + +- **GIVEN** a fresh process has no process-session or thread mapping +- **AND** an exact response, file, or bridge owner requires account A +- **WHEN** the first thread is admitted on account A through that hard owner +- **THEN** the thread row and absent process preference are persisted atomically +- **AND** the process preference remains insert-only if another thread already initialized it + +#### Scenario: Recovery probe does not seed the process + +- **GIVEN** a fresh process has no process-session mapping +- **WHEN** a thread is selected on probing account A through a recovery reservation +- **THEN** account A is not published as the immutable process preference +- **AND** a failed reservation commit can restore the reversible thread placement + +#### Scenario: Direct WebSocket siblings do not share replay state + +- **GIVEN** sibling threads share one process session and cache key +- **WHEN** each uses direct WebSocket Responses and one reconnects +- **THEN** retained response, prefix, Lite, and pending-tool state is read only from that thread +- **AND** the reconnect cannot inject or replay its sibling's state + +#### Scenario: Unknown exact turn does not borrow broader thread replay + +- **GIVEN** a direct WebSocket thread has retained replay or tool state +- **WHEN** a request supplies a nonblank client turn state with no exact in-memory alias +- **THEN** it does not reuse or replace the broader thread state +- **AND** only a previously resolved exact alias may refresh the thread alias + +### Requirement: HTTP Responses routes preserve upstream websocket session continuity + +When serving HTTP `/v1/responses` or HTTP `/backend-api/codex/responses`, the +service MUST preserve upstream Responses websocket session continuity on a +stable per-session bridge key instead of opening a brand new upstream session +for every eligible request. For backend Codex requests carrying `thread-id`, +the canonical bridge key MUST use the same derived logical thread identity used +for account locality and compact routing. Otherwise the bridge key MUST use an +explicit session/conversation header when present, then normalized +`prompt_cache_key`, deriving a stable key from the existing cache-affinity +inputs when the client omits one. While bridged, the service MUST preserve the +external HTTP/SSE contract, continue request logging with `transport = "http"`, +and keep requests from different bridge keys isolated. + +An established live or durable thread bridge is hard continuity. The bridge +MUST retain request-scoped fork lanes for concurrent unanchored requests and +MUST preserve exact turn-state and previous-response aliases. A request with +`thread-id` MUST NOT fall back to the legacy canonical key derived from process +session and `prompt_cache_key`, because current Codex may share both across +siblings. It MAY recover an old bridge only through an exact hard alias. +Authenticated forwarded affinity kind/key values MUST remain verbatim and MUST +NOT be namespaced or hashed again. + +#### Scenario: bridge forwards hard continuity keys to the owner replica + +- **WHEN** operators configure multiple eligible bridge instance ids +- **AND** a request uses a bridge key derived from `x-codex-turn-state`, an explicit legacy session header, or `thread-id` +- **AND** that request lands on a non-owner instance +- **THEN** the service MUST forward the request internally to the owner replica +- **AND** it MUST NOT return a topology-bearing `bridge_instance_mismatch` error to the client for that owner mismatch alone + +#### Scenario: gateway-style prompt-cache bridge requests tolerate wrong-replica arrival + +- **WHEN** a request uses a bridge key derived only from `prompt_cache_key` or a derived prompt-cache key +- **AND** that request lands on a non-owner instance +- **THEN** the service MAY create or reuse a local bridge session on that instance +- **AND** it MUST treat the owner mismatch as a locality miss instead of a continuity failure + +#### Scenario: forwarded bridge requests fail closed when owner forwarding loops + +- **WHEN** a forwarded hard-continuity bridge request reaches another non-owner replica +- **THEN** the service MUST fail the request with a generic 5xx bridge-forward error +- **AND** it MUST NOT attempt another owner handoff + +#### Scenario: local restart orphan is recovered by the replacement instance + +- **WHEN** a single local bridge instance is replaced while durable hard-continuity ownership still references the old instance id +- **AND** the old owner has no distinct active forwarding endpoint from the current replacement instance +- **THEN** the replacement instance MUST treat the row as restart-orphaned and may claim durable ownership locally +- **AND** same-account takeover MUST preserve the latest persisted response anchor until a replacement response id is recorded +- **AND** normal client retries MUST NOT be stranded waiting for the old instance lease to expire + +When request aliases resolve to different durable rows for the same account, +an explicitly requested previous-response alias MUST select its row even if +that row has since advanced to a newer response id. Without an explicitly +resolved previous-response alias, recovery MUST select the freshest row that +contains a persisted response anchor rather than using alias enumeration order. + +#### Scenario: requested durable response alias survives same-account row divergence + +- **GIVEN** turn-state and previous-response aliases resolve to different durable rows for the same account +- **AND** the request names the previous-response alias whose row has since advanced to a newer response id +- **WHEN** the service resolves durable continuity +- **THEN** it selects the row resolved by the requested previous-response alias +- **AND** it preserves that row's latest persisted response anchor + +#### Scenario: Sequential siblings use distinct canonical bridges + +- **GIVEN** root and child requests share process session and `prompt_cache_key` +- **AND** they carry different `thread-id` values +- **WHEN** the child starts after the root request completes +- **THEN** the child uses a different canonical bridge identity +- **AND** another child request reuses only the child's lane + +#### Scenario: Old shared canonical lane is not a thread fallback + +- **GIVEN** an old bridge exists under `(session-id, prompt_cache_key)` +- **WHEN** a request carries a new thread identity but no exact turn-state or previous-response alias +- **THEN** it does not attach to the old shared lane +- **AND** it creates or reuses its thread-canonical lane + +## ADDED Requirements + +### Requirement: Thread-goal routing uses payload thread identity + +Thread-goal get, set, and clear operations carrying a nonblank payload +`threadId` MUST select account locality from that exact thread identity, +combined with the process session when available. An explicit client turn +state remains hard continuity and MUST retain its existing precedence or +conflict behavior. Other generic request headers MUST NOT cause one sibling +thread's goal operation to follow another sibling's locality. Existing +protocol forwarding and error behavior MUST remain unchanged. + +#### Scenario: Sibling goal operations follow their own threads + +- **GIVEN** sibling threads share a process session but have distinct `threadId` values +- **WHEN** each invokes thread-goal get, set, or clear +- **THEN** each operation uses its own bounded thread locality diff --git a/openspec/changes/scope-codex-affinity-by-thread/specs/sticky-session-operations/spec.md b/openspec/changes/scope-codex-affinity-by-thread/specs/sticky-session-operations/spec.md new file mode 100644 index 0000000000..7bb53c9577 --- /dev/null +++ b/openspec/changes/scope-codex-affinity-by-thread/specs/sticky-session-operations/spec.md @@ -0,0 +1,301 @@ +## MODIFIED Requirements + +### Requirement: Bare process-session cap spillover is non-mutating + +The system MUST parse process-session and thread headers independently. A bare +process-session mapping and a bounded thread-local mapping MUST use distinct, +header-inaccessible storage identities, so a client-supplied hard turn-state +value cannot alias either derived soft row. A current replica MUST consult a +legacy raw Codex-session key independently even when a namespaced process or +thread row exists. Any raw hit MUST take precedence as hard ownership. If a +resolved file, response, bridge, or other exact owner conflicts with that raw +legacy owner, the request MUST fail closed without creating or rewriting any +of those rows. + +A missing thread row MAY use an eligible process-session soft row as its +initial placement preference. If that process row is missing, the first +admitted thread MUST initialize it with insert-if-absent and MUST persist its +own bounded thread row. A concurrent or later thread MUST NOT overwrite that +first-writer process preference. Account-cap spillover or later thread movement +MUST NOT rewrite or delete the process-session mapping or a sibling's thread +mapping. A provisional recovery-probe reservation MUST NOT initialize a +missing process preference, because its thread mapping may still require +rollback and a probing account is not a stable process default. A later normal +admission MAY initialize the missing process preference. + +When the mapped account for a bare process-session key is locally capped and +another eligible account is selected, the spillover MUST apply only to that +request. Selection MUST NOT update or delete the stored process-session mapping +because of account-cap spillover. If the mapped account is below cap, normal +sticky selection MUST retain it. + +#### Scenario: Capped bare-session owner spills without rebinding + +- **GIVEN** a bare process-session mapping points to account A +- **AND** account A is locally capped +- **AND** account B is eligible and below cap +- **WHEN** a self-contained pre-visible request is selected +- **THEN** the request uses account B +- **AND** the stored process-session mapping still points to account A + +#### Scenario: Unsaturated bare-session owner retains locality + +- **GIVEN** a bare process-session mapping points to eligible account A below its local caps +- **WHEN** a self-contained request is selected +- **THEN** the request uses account A +- **AND** the mapping remains unchanged + +#### Scenario: Equal session and turn-state values remain isolated + +- **GIVEN** a process-session header and an explicit turn-state header have equal text values +- **WHEN** their affinity mappings are resolved +- **THEN** the process-session mapping uses a source-separated opaque key +- **AND** the explicit turn-state mapping continues to use the legacy raw key as hard ownership + +#### Scenario: Derived soft key cannot be reused as raw hard turn state + +- **GIVEN** a process-session or thread value has a derived internal storage key +- **WHEN** a client submits the visible representation of that key as a turn-state header +- **THEN** header normalization cannot reproduce the internal storage identity +- **AND** hard turn-state selection cannot read or rewrite the soft row + +#### Scenario: Legacy raw mapping remains hard + +- **GIVEN** a legacy replica persisted a raw Codex-session mapping +- **WHEN** a current replica receives the matching process or legacy thread header +- **THEN** it does not reinterpret or mutate the legacy raw row as spillable affinity +- **AND** mixed-version operation remains fail-closed for that row + +#### Scenario: Coexisting legacy and namespaced rows prefer hard ownership + +- **GIVEN** mixed-version replicas created a raw row and a namespaced process or thread row for the same request identity +- **AND** the rows point to different accounts +- **WHEN** a current replica selects the request +- **THEN** the raw row's account is treated as the hard owner +- **AND** neither row is deleted or rewritten by account-cap spillover + +#### Scenario: Legacy hard owner conflicts with resolved owner + +- **GIVEN** a raw legacy session row points to account A +- **AND** a file, previous response, or bridge resolves to account B +- **WHEN** the request is routed +- **THEN** the service fails with `continuity_owner_conflict` +- **AND** it neither bypasses nor rewrites the raw row + +#### Scenario: Process preference seeds only the new thread + +- **GIVEN** a process-session soft row points to account A +- **AND** no bounded row exists for thread T +- **WHEN** T is admitted on account A or a safely selected alternate +- **THEN** the admitted account is persisted under T's bounded key +- **AND** the process-session row remains unchanged + +#### Scenario: Missing process preference is initialized once + +- **GIVEN** no process-session mapping exists +- **WHEN** the first thread is admitted on account A and a concurrent or later thread is admitted on account B +- **THEN** insert-if-absent preserves the first persisted process owner +- **AND** each thread persists only its own bounded locality after that initialization + +#### Scenario: Provisional probe placement does not escape rollback + +- **GIVEN** neither process nor thread has a bounded mapping +- **WHEN** a probing-account placement persists provisionally and then loses its runtime commit +- **THEN** its thread mutation is restored +- **AND** no immutable process preference is left behind + +#### Scenario: Legacy raw owner wins over thread locality + +- **GIVEN** a raw legacy Codex row points to account A +- **AND** a bounded thread row points to account B +- **WHEN** the request is routed +- **THEN** the raw row remains hard ownership evidence and account A wins +- **AND** neither mapping is rewritten to reconcile the disagreement + +### Requirement: Unanchored process-session concurrency uses independent bridge lanes + +When multiple Responses requests share a process-level session header but +carry neither `previous_response_id` nor nonblank turn-state continuity, the +service MUST NOT queue an independent request behind an active response-create +gate. If the canonical bridge is still being created, reserved by another +request before submit, already has a visible request, or belongs to a different +model class, the service MUST create a server request-scoped bridge lane. The +lane identity MUST NOT depend on a client-controlled request ID. The fork MUST +leave the canonical bridge and its model metadata unchanged. + +When such requests carry nonblank `thread-id`, each thread MUST have a stable +canonical bridge identity derived from process and thread identity regardless +of `prompt_cache_key`; distinct threads MUST remain isolated even when they +execute sequentially, and repeated requests from one thread MUST retain one +identity. Requests without `thread-id` MUST retain the legacy session-header +identity, including the established explicit-prompt-cache composition. + +A pre-submit handoff reservation MUST protect its bridge from idle pruning and +capacity eviction, and any cancellation or error between lookup and visible +submission MUST release it. Owner forwarding MUST preserve whether a +session-header, thread-header, or internal-fork request was unanchored instead +of treating a proxy-generated downstream turn-state as an explicit client +anchor, but MUST NOT attach that v2-only state to prompt-cache or unrelated +affinity families. It MUST fail closed when a mixed-version hop cannot +authenticate required unanchored state. The v2 primary signature MUST bind +whether client-IP metadata was present, while the companion signature MUST bind +its value. When the canonical owner itself creates a fork for a forwarded +request, it MUST own that fork locally instead of re-hashing it into another +forwarding hop. Explicitly anchored owner forwards MUST retain the +legacy-compatible primary signature during rolling upgrades, and a receiving +instance MUST reject ambiguous delimiter-bearing legacy fields. Durable aliases +derived from the forked lane MUST retain hard owner and account continuity. If +durable ownership fencing rejects a stale owner's new alias, the stale owner +MUST remove the matching local alias without removing a newer local +generation's mapping. + +#### Scenario: sequential child agent does not reuse parent bridge history + +- **GIVEN** a parent and child Codex agent share one process session and `prompt_cache_key` +- **AND** each agent supplies its own stable `thread-id` +- **WHEN** the child starts after the parent's visible request has completed +- **THEN** the child uses a different bridge identity from the parent +- **AND** another request from that same child keeps the child's bridge identity + +#### Scenario: Background requests do not block behind a foreground turn + +- **GIVEN** a foreground request is active on a session-header or thread-header bridge +- **WHEN** two unanchored background requests arrive with the same canonical identity +- **THEN** each background request uses an independent response-create gate +- **AND** neither request waits for the foreground response to complete +- **AND** the foreground bridge's model metadata remains unchanged + +#### Scenario: Lookup-to-submit requests remain isolated + +- **GIVEN** an unanchored request has reserved an idle canonical bridge but has not yet made queued activity visible +- **WHEN** another unanchored request arrives with the same canonical identity and client request ID +- **THEN** the second request uses a distinct server-scoped bridge lane +- **AND** it does not reuse the reserved canonical bridge + +#### Scenario: Durable refresh publishes the handoff reservation + +- **GIVEN** an unanchored request reuses an idle durable canonical bridge +- **WHEN** refreshing the durable lease yields before lookup returns +- **THEN** the canonical bridge is already reserved for that request +- **AND** a concurrent unanchored request uses a distinct server-scoped lane + +#### Scenario: Cancelled pre-submit handoff does not strand a reservation + +- **GIVEN** an unanchored request is reusing an idle canonical bridge +- **WHEN** the request is cancelled after claiming the bridge but before queued activity becomes visible +- **THEN** the canonical bridge remains unreserved +- **AND** later requests are not forced onto fork lanes by the cancelled lookup + +#### Scenario: Payload preparation failure does not strand a reservation + +- **GIVEN** an unanchored request has reserved an idle canonical bridge +- **WHEN** anchor injection, trimming, or payload validation fails before submission +- **THEN** request-scope cleanup releases the reservation +- **AND** later requests may reuse the canonical bridge + +#### Scenario: Remote owner preserves unanchored concurrency + +- **GIVEN** an unanchored request is forwarded to the canonical bridge owner +- **AND** the proxy generated a downstream turn-state for response aliasing +- **WHEN** the owner receives the forwarded request while the canonical lane is active +- **THEN** the owner still treats the request as unanchored +- **AND** the request uses an independent bridge lane +- **AND** the pre-submit handoff remains reserved until submission becomes visible + +#### Scenario: Owner-side fork does not start a second forwarding hop + +- **GIVEN** an unanchored request has reached its canonical owner +- **AND** that owner creates an independent fork because the canonical lane is active +- **WHEN** rendezvous hashing the generated fork key would select another instance +- **THEN** the canonical owner creates and durably claims the fork locally +- **AND** the request is not rejected as a forwarding loop + +#### Scenario: Blank turn-state is not an anchor + +- **GIVEN** a request has process/thread identity and an empty or whitespace-only turn-state header +- **WHEN** the request is forwarded to its owner +- **THEN** the signed forwarding context marks the original request as unanchored +- **AND** the generated downstream turn-state does not collapse it onto the canonical gate + +#### Scenario: Forwarding downgrade fails closed + +- **GIVEN** an owner-forward request requires unanchored concurrency semantics +- **WHEN** the signed unanchored boolean is changed, removed, or repacked into affinity fields, or either instance only supports the legacy signature +- **THEN** the owner-forward hop fails closed +- **AND** the request is not attached to the shared canonical response-create gate + +#### Scenario: Anchored forwarding remains rolling-upgrade compatible + +- **GIVEN** an owner-forward request carries explicit previous-response or turn-state continuity +- **WHEN** the origin and owner run different bridge protocol versions +- **THEN** the primary signature remains valid under the legacy contract +- **AND** the anchored request can continue without weakening unanchored fail-closed behavior + +#### Scenario: Prompt-cache forwarding remains rolling-upgrade compatible + +- **GIVEN** an unanchored first-turn request uses a prompt-cache affinity lane +- **WHEN** that request is forwarded to its canonical owner +- **THEN** the origin does not attach session/thread-header unanchored v2 state +- **AND** an older owner may accept the legacy-compatible forwarding contract + +#### Scenario: Legacy session-header canonical lane proves its turn-state anchor + +- **GIVEN** a legacy-signed owner forward has no previous-response ID and its durable canonical key is still `session_header` +- **WHEN** its forwarded turn state is a registered durable alias for that exact canonical lane +- **THEN** the current owner accepts it as anchored continuity +- **AND** an unknown turn state or an alias for another canonical lane fails closed with `bridge_forward_upgrade_required` + +#### Scenario: Legacy proof precedes compact and bridge fallback branches + +- **GIVEN** a legacy-signed owner forward requires turn-state anchor proof +- **WHEN** the request contains a terminal compaction trigger or bypasses the websocket bridge +- **THEN** exact alias proof runs before compact, HTTP fallback, admission, or upstream work + +#### Scenario: Current origin proves a turn-state alias before legacy owner forwarding + +- **GIVEN** a current origin resolves a nonblank turn state only through a shared `session_header` durable lane +- **WHEN** that request would be forwarded to another owner with the legacy signature contract +- **THEN** the origin proves an exact turn-state alias row for that canonical lane before sending the owner request +- **AND** an unknown alias fails closed with `bridge_forward_upgrade_required` + +#### Scenario: Latest-state metadata is not proof of alias registration + +- **GIVEN** a durable session records a latest turn state but has no matching turn-state alias row +- **WHEN** that value is presented by a legacy-signed owner forward +- **THEN** the owner rejects it with `bridge_forward_upgrade_required` + +#### Scenario: Stale owners cannot register continuity aliases after takeover + +- **GIVEN** durable ownership advanced to a new owner epoch +- **WHEN** the stale owner attempts to register a turn-state or previous-response alias with its old epoch +- **THEN** alias registration writes nothing +- **AND** the stale owner removes the rejected value from its local alias index +- **AND** a newer local generation's mapping for the same value remains intact +- **AND** the stale value cannot satisfy legacy anchor proof + +#### Scenario: Ambiguous legacy signature fields fail closed + +- **GIVEN** a legacy owner-forward signature contains a delimiter in any signed header field +- **WHEN** field boundaries are repacked without changing the legacy joined byte string +- **THEN** a current owner rejects the forwarding context as invalid +- **AND** the repacked affinity kind cannot weaken hard continuity + +#### Scenario: V2 client-IP metadata cannot be removed or blanked + +- **GIVEN** an unanchored v2 owner-forward request carries signed client-IP metadata +- **WHEN** both client-IP headers are removed, the value is blanked, or the value is changed +- **THEN** the owner rejects the forwarding context as invalid +- **AND** a genuinely no-IP v2 request remains valid + +#### Scenario: Durable fork continuation remains owner-bound + +- **GIVEN** a forked lane has produced a durable turn-state or previous-response alias +- **WHEN** a later request resolves that alias on another instance +- **THEN** the request follows the hard owner-bound continuity path +- **AND** the original account binding is preserved + +#### Scenario: Explicit continuation is not split + +- **WHEN** a request carries `previous_response_id` or a turn-state header +- **THEN** the service keeps the request on the hard owner-bound continuity path +- **AND** it does not apply unanchored parallel-session isolation diff --git a/openspec/changes/scope-codex-affinity-by-thread/tasks.md b/openspec/changes/scope-codex-affinity-by-thread/tasks.md new file mode 100644 index 0000000000..0df723dd2d --- /dev/null +++ b/openspec/changes/scope-codex-affinity-by-thread/tasks.md @@ -0,0 +1,18 @@ +## 1. Identity and account locality + +- [x] 1.1 Parse process session and thread identity independently and derive source-separated opaque thread keys +- [x] 1.2 Route backend Responses and compact through bounded thread locality with process-preference seeding +- [x] 1.3 Preserve raw legacy Codex rows and all exact hard-owner precedence/conflict behavior + +## 2. Transport continuity + +- [x] 2.1 Scope direct WebSocket replay/tool continuity by thread and refresh active thread locality +- [x] 2.2 Scope HTTP bridge canonical lanes by thread while preserving exact-alias migration and forwarded-key behavior +- [x] 2.3 Route thread-goal operations from payload `threadId` + +## 3. Regression evidence + +- [x] 3.1 Cover identity parsing, source separation, process seeding, Responses/compact parity, and unchanged cache hints +- [x] 3.2 Cover sibling direct-WebSocket replay isolation and thread-goal account selection +- [x] 3.3 Cover sibling bridge isolation, exact legacy alias recovery, and no old-canonical fallback +- [x] 3.4 Run focused tests, Ruff, type checks, OpenSpec checks available in the checkout, and review the final diff diff --git a/openspec/changes/serve-h2c-upgrade-offers-as-http11/proposal.md b/openspec/changes/serve-h2c-upgrade-offers-as-http11/proposal.md new file mode 100644 index 0000000000..bc779b69fc --- /dev/null +++ b/openspec/changes/serve-h2c-upgrade-offers-as-http11/proposal.md @@ -0,0 +1,43 @@ +# Serve h2c Upgrade Offers as Plain HTTP/1.1 + +## Why + +JetBrains/Ktor clients attach opportunistic cleartext HTTP/2 upgrade headers +(`Connection: Upgrade, HTTP2-Settings` + `Upgrade: h2c` + `HTTP2-Settings`) to +ordinary HTTP/1.1 Responses API POSTs. The server's httptools-based HTTP parser +treats any such request as a protocol switch and wedges: a body coalesced with +the headers is silently dropped (the application validates an empty body and +returns 422), and a body written as a separate segment — Ktor's write pattern — +is answered with `400 Invalid HTTP request received.` before authentication +ever runs (issue #1757). RFC 9110 §7.8 allows a server to ignore an upgrade +offer and answer over HTTP/1.1, which is what upstream OpenAI endpoints do. + +## What Changes + +- Serve valid HTTP/1.1 requests that offer a non-WebSocket protocol switch + (such as `h2c`) as normal HTTP/1.1 requests, with the full body delivered to + the application for both client segmentations. +- Strip the declined offer's hop-by-hop headers (`Upgrade`, `HTTP2-Settings`, + and their `Connection` tokens) before the request reaches the application. +- Keep genuine WebSocket upgrades switching protocols exactly as today. +- No new settings, budgets, or defaults; no API or schema change. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `http-ingress-limits`: raw HTTP ingress MUST serve non-WebSocket HTTP/1.1 + upgrade offers as plain HTTP/1.1 instead of dropping the body or rejecting + the request. + +## Impact + +`app/cli.py` server bootstrap and new `app/core/http_protocol.py` / +`app/core/http_protocol_httptools.py` uvicorn protocol subclasses (the h11 +variant covers the httptools-less fallback with the same header hygiene), plus +transport-level regression coverage. No dashboard, API, schema, or +configuration change. diff --git a/openspec/changes/serve-h2c-upgrade-offers-as-http11/specs/http-ingress-limits/spec.md b/openspec/changes/serve-h2c-upgrade-offers-as-http11/specs/http-ingress-limits/spec.md new file mode 100644 index 0000000000..ec39a2123e --- /dev/null +++ b/openspec/changes/serve-h2c-upgrade-offers-as-http11/specs/http-ingress-limits/spec.md @@ -0,0 +1,57 @@ +# http-ingress-limits Delta + +## ADDED Requirements + +### Requirement: Non-WebSocket upgrade offers are served as plain HTTP/1.1 + +The server MUST serve a valid HTTP/1.1 request that offers a non-WebSocket +protocol switch (`Connection: Upgrade` with an `Upgrade` token other than +`websocket`, such as `h2c`) as a normal HTTP/1.1 request. The complete request +body MUST reach the application whether it arrives coalesced with the headers +or in later TCP segments, and the offer MUST NOT cause the request or the +connection to be rejected. The declined offer's hop-by-hop headers (`Upgrade`, +`HTTP2-Settings`, and their `Connection` tokens) MUST NOT be exposed to the +application. Genuine WebSocket upgrade requests MUST keep completing the +protocol switch. + +#### Scenario: h2c offer with the body coalesced with the headers + +- **WHEN** a client sends an HTTP/1.1 POST carrying `Connection: Upgrade, + HTTP2-Settings`, `Upgrade: h2c`, and `HTTP2-Settings` headers with the body + in the same TCP segment as the headers +- **THEN** the application receives the complete request body +- **AND** the application does not observe the `Upgrade`, `HTTP2-Settings`, or + `Connection: Upgrade` headers + +#### Scenario: h2c offer with the body in a separate segment + +- **WHEN** the same request arrives with the headers and the body written as + separate TCP segments +- **THEN** the application receives the complete request body +- **AND** the server does not answer `400 Bad Request` at the protocol layer + +#### Scenario: Repeated Connection fields do not hide the offer + +- **WHEN** the h2c offer arrives with `Connection: Upgrade, HTTP2-Settings` + followed by a second `Connection: keep-alive` field +- **THEN** the application receives the complete request body +- **AND** the surviving `Connection` tokens (such as `keep-alive`) are + preserved while the upgrade tokens are removed + +#### Scenario: Connection stays usable after a declined offer + +- **WHEN** a request with a declined h2c offer completes on a keep-alive + connection +- **THEN** a subsequent plain HTTP/1.1 request on the same connection is + served normally + +#### Scenario: Pipelined offers in one segment do not exhaust the server + +- **WHEN** a single TCP segment pipelines many upgrade-offering requests +- **THEN** every request is served as plain HTTP/1.1 without the per-offer + replay growing the call stack or aborting the connection + +#### Scenario: WebSocket upgrades still switch protocols + +- **WHEN** a client requests a WebSocket upgrade (`Upgrade: websocket`) +- **THEN** the protocol switch completes and WebSocket messages flow diff --git a/openspec/changes/serve-h2c-upgrade-offers-as-http11/tasks.md b/openspec/changes/serve-h2c-upgrade-offers-as-http11/tasks.md new file mode 100644 index 0000000000..f7b94a500f --- /dev/null +++ b/openspec/changes/serve-h2c-upgrade-offers-as-http11/tasks.md @@ -0,0 +1,31 @@ +## 1. Implementation + +- [x] 1.1 Neutralize non-WebSocket upgrade offers in the httptools HTTP + protocol: replay the request head without the declined offer's + hop-by-hop headers and serve the request as plain HTTP/1.1. +- [x] 1.2 Wire the tolerant protocol into the server bootstrap, falling back + to an h11 subclass with the same header hygiene when httptools is + unavailable (stock h11 already delivers the body but exposes the + declined offer's headers). +- [x] 1.3 Classify upgrade offers by combining repeated `Connection` fields + (RFC 9110 §5.3) so a second `Connection: keep-alive` field cannot hide + the offer and reproduce the body loss. +- [x] 1.4 Replay declined offers iteratively (loop in `data_received`) rather + than recursively, so a segment pipelining many upgrade-offering requests + cannot drive attacker-controlled recursion depth (RecursionError + escaping into the event loop) or pin per-frame byte copies. + +## 2. Validation + +- [x] 2.1 Add transport-level regressions: h2c offer with coalesced + header/body and with split segments both reach the application with the + full body and succeed; the declined offer's headers are not exposed; the + connection stays reusable. +- [x] 2.2 Add a live-server regression over real sockets using the production + protocol wiring, including a real WebSocket upgrade that must keep + completing, plus a canary pinning the stock uvicorn defect. +- [x] 2.3 Add a regression pipelining 2000 h2c offers in one segment: all are + served and the connection survives (raised RecursionError when the + replay was recursive). +- [x] 2.4 Run focused tests, lint, type checks, and strict OpenSpec + validation. diff --git a/openspec/changes/settle-compact-failover-before-health/context.md b/openspec/changes/settle-compact-failover-before-health/context.md new file mode 100644 index 0000000000..eab7912601 --- /dev/null +++ b/openspec/changes/settle-compact-failover-before-health/context.md @@ -0,0 +1,25 @@ +Compact timeout already settles the API-key reservation before +`_handle_stream_error`. The generic compact error path did the opposite: +it wrote health, then either surfaced (settle + raise) or failed over +without settling. + +`failover_next` cannot release the reservation immediately because the next +account still uses that same reservation. Health is therefore deferred until +the next settle (success, surface, timeout, or exhaustion). + +`_settle_compact_api_key_usage` still raises `usage_settlement_failed` after +a finalize failure even when its fail-safe release succeeds. That exception +must carry whether the reservation is actually released so deferred health +can flush before the 502 is surfaced. If the fail-safe release also fails, +the reservation is still held and deferred health stays queued. + +Once usage is finalized, a later deferred health-write failure is a local +persistence problem. It must not convert a billed compact success into a 500 +that clients retry. Cancellation and other non-proxy exits skip the dedicated +settle handlers, so they need an explicit settle-then-flush before the +original exception continues. The flush itself is shielded so a cancel that +arrives after queues are drained cannot drop the remaining health write. +Each deferred entry is written independently so one persistence failure does +not skip the other failed accounts. A later `_select_account_with_budget` +timeout is a `ProxyResponseError`, so it must use the same settle-and-flush +cleanup as other unsettled exits. diff --git a/openspec/changes/settle-compact-failover-before-health/proposal.md b/openspec/changes/settle-compact-failover-before-health/proposal.md new file mode 100644 index 0000000000..6cd4abd3ce --- /dev/null +++ b/openspec/changes/settle-compact-failover-before-health/proposal.md @@ -0,0 +1,28 @@ +## Why + +Compact `failover_next` writes account health through `_handle_stream_error` +while the API-key reservation is still reserved. The timeout branch already +settles first. An open reservation plus a health penalty double-charges the +request and can backoff an account that the request has not finished using. + +## What Changes + +- Classify compact failover without writing health. +- Keep the reservation across `failover_next` and defer the health write. +- Flush deferred health only after `_settle_compact_api_key_usage`. +- If finalize fails but fail-safe release succeeds, flush deferred health + before surfacing `usage_settlement_failed`. +- Surface paths settle, then write health. + +## Capabilities + +### Modified Capabilities + +- `usage-refresh-policy`: compact failover must settle the reservation + before any account-health write. + +## Impact + +Streaming SSE reservation paths and compact timeout/exhaustion terminals stay +on their current settle-then-health order. No second reservation is acquired +mid-request. diff --git a/openspec/changes/settle-compact-failover-before-health/specs/usage-refresh-policy/spec.md b/openspec/changes/settle-compact-failover-before-health/specs/usage-refresh-policy/spec.md new file mode 100644 index 0000000000..4cda6ab4d8 --- /dev/null +++ b/openspec/changes/settle-compact-failover-before-health/specs/usage-refresh-policy/spec.md @@ -0,0 +1,106 @@ +## ADDED Requirements + +### Requirement: Compact failover settles before account-health writes + +When `compact_responses` holds an API-key usage reservation, it MUST NOT write account health for a compact upstream failure until that reservation has been settled or released. A `failover_next` decision MUST keep the same reservation for the next account and MUST defer the failed account's health write until the next settlement. Timeout and exhaustion terminals MUST keep settle-then-health order. Compact MUST NOT acquire a second reservation mid-request. If usage finalization fails but the fail-safe reservation release succeeds, compact MUST flush deferred health before surfacing `usage_settlement_failed`. If the reservation remains held because that fail-safe release also fails, deferred health MUST stay unapplied. After a compact reservation is finalized, a deferred health-persistence failure MUST NOT replace the successful compact response. If compact exits through cancellation or any exception other than a `ProxyResponseError` that already settled, it MUST settle or release the reservation and flush deferred health before propagating that exception. A later account-selection budget timeout after `failover_next` MUST use that same settle-and-flush path. Deferred health flush MUST complete even if the compact request is cancelled while that flush is awaiting a health write. If one deferred health write fails, compact MUST still attempt the remaining deferred health writes. + +#### Scenario: Compact failover_next defers health until settle + +- **GIVEN** a compact request with a held API-key reservation +- **AND** the first account fails with a `failover_next` class +- **WHEN** a later account completes and settlement runs +- **THEN** `_handle_stream_error` for the failed account runs only after that settlement +- **AND** the request does not acquire another reservation + +#### Scenario: Compact timeout still settles before health + +- **GIVEN** a compact request whose upstream call times out +- **WHEN** the timeout branch records account health +- **THEN** the reservation is settled before `_handle_stream_error` + +#### Scenario: Compact HTTP 500 failover defers health until settle + +- **GIVEN** a compact request with a held API-key reservation +- **AND** the first account exhausts same-account HTTP 500 retries +- **WHEN** a later account completes and settlement runs +- **THEN** `_handle_proxy_error` and extra `record_errors` for the failed account run only after that settlement + +#### Scenario: Compact route failure after failover still applies deferred health + +- **GIVEN** a compact request that deferred health on `failover_next` +- **WHEN** the next account raises `UpstreamProxyRouteError` +- **THEN** the reservation is settled +- **AND** the deferred health write still runs + +#### Scenario: Compact refresh/connect failover defers health until settle + +- **GIVEN** a compact request with a held API-key reservation +- **AND** the first account fails a retryable freshness/connect or post-401 forced-refresh transport error +- **WHEN** a later account completes and settlement runs +- **THEN** `_handle_stream_error` for the failed account runs only after that settlement + +#### Scenario: Compact second 401 failover defers health until settle + +- **GIVEN** a compact request with a held API-key reservation +- **AND** the same account returns 401 again after a forced refresh +- **WHEN** a later account completes and settlement runs +- **THEN** `_handle_proxy_error` for the failed account runs only after that settlement + +#### Scenario: Compact permanent refresh settles before the health mark + +- **GIVEN** a compact request with a held API-key reservation +- **AND** the post-401 forced refresh raises a permanent `RefreshError` +- **WHEN** the compact request records the permanent account failure +- **THEN** the reservation is settled before `mark_permanent_failure` + +#### Scenario: Compact fallback release still flushes deferred health + +- **GIVEN** a compact request that deferred health on `failover_next` +- **AND** a later account completes but usage finalization fails +- **AND** the fail-safe reservation release succeeds +- **WHEN** settlement surfaces `usage_settlement_failed` +- **THEN** the deferred health write still runs +- **AND** it runs before the `usage_settlement_failed` error is raised + +#### Scenario: Compact unsettled reservation keeps deferred health unapplied + +- **GIVEN** a compact request that deferred health on `failover_next` +- **AND** both usage finalization and fail-safe release fail +- **WHEN** settlement surfaces `usage_settlement_failed` +- **THEN** the deferred health write does not run + +#### Scenario: Compact success survives deferred health persistence failure + +- **GIVEN** a compact request that deferred health on `failover_next` +- **AND** a later account completes and usage finalization succeeds +- **WHEN** the deferred health write raises +- **THEN** the successful compact response is still returned + +#### Scenario: Compact unexpected exit still flushes deferred health + +- **GIVEN** a compact request that deferred health on `failover_next` +- **WHEN** the next account attempt raises cancellation or another non-proxy exception +- **THEN** the reservation is settled or released +- **AND** the deferred health write still runs +- **AND** the original exception is propagated + +#### Scenario: Compact deferred health flush completes under cancellation + +- **GIVEN** a compact request that deferred health on `failover_next` +- **AND** a later account completed and settlement started flushing +- **WHEN** the request is cancelled during the deferred health write +- **THEN** the deferred health write still completes + +#### Scenario: Compact continues flushing after one deferred health write fails + +- **GIVEN** a compact request that deferred health for more than one failed account +- **WHEN** the first deferred health write raises +- **THEN** later deferred health writes are still attempted + +#### Scenario: Compact selection timeout after failover still flushes deferred health + +- **GIVEN** a compact request that deferred health on `failover_next` +- **WHEN** selecting the next account exhausts the request budget +- **THEN** the reservation is settled or released +- **AND** the deferred health write still runs +- **AND** the original budget-timeout error is propagated diff --git a/openspec/changes/settle-compact-failover-before-health/tasks.md b/openspec/changes/settle-compact-failover-before-health/tasks.md new file mode 100644 index 0000000000..16e6df8658 --- /dev/null +++ b/openspec/changes/settle-compact-failover-before-health/tasks.md @@ -0,0 +1,49 @@ +## 1. Implementation + +- [x] 1.1 Classify compact failover without writing health, and defer the + health write when a reservation is still held. +- [x] 1.2 Flush deferred health only after `_settle_compact_api_key_usage`. +- [x] 1.3 Flush deferred health when finalize fails but fail-safe release + succeeds, then surface `usage_settlement_failed`. +- [x] 1.4 Keep a finalized compact success when deferred health persistence + fails. +- [x] 1.5 Settle and flush deferred health on cancellation and other + non-proxy exits. +- [x] 1.6 Shield deferred health flush so cancellation during the write + still completes the penalty. +- [x] 1.7 Continue remaining deferred health writes if one write fails. +- [x] 1.8 Settle and flush deferred health when a later account selection + times out. + +## 2. Regression coverage + +- [x] 2.1 Assert compact `failover_next` with a held reservation settles + before `_handle_stream_error`. +- [x] 2.2 Assert exhausted HTTP 500 retries defer `_handle_proxy_error` + until settlement. +- [x] 2.3 Assert `UpstreamProxyRouteError` after failover still flushes + deferred health. +- [x] 2.4 Assert freshness/connect and post-401 refresh failovers defer + health until settlement. +- [x] 2.5 Assert a second 401 after forced refresh defers `_handle_proxy_error`. +- [x] 2.6 Assert permanent post-401 refresh settles before + `mark_permanent_failure`. +- [x] 2.7 Assert fallback-release success still flushes deferred health + before `usage_settlement_failed`. +- [x] 2.8 Assert fallback-release failure keeps deferred health unapplied. +- [x] 2.9 Assert a deferred health-persistence failure does not replace a + finalized compact success. +- [x] 2.10 Assert cancellation or another non-proxy exit still settles and + flushes deferred health. +- [x] 2.11 Assert cancellation during deferred health flush still completes + the write. +- [x] 2.12 Assert a later deferred health write still runs after an earlier + write fails. +- [x] 2.13 Assert a post-failover account-selection timeout still flushes + deferred health. + +## 3. Validation + +- [x] 3.1 Run the new compact order regression and the existing compact + timeout settle-before-health test. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/openspec/changes/settle-live-ingest-task-failures/proposal.md b/openspec/changes/settle-live-ingest-task-failures/proposal.md new file mode 100644 index 0000000000..a61149b2e0 --- /dev/null +++ b/openspec/changes/settle-live-ingest-task-failures/proposal.md @@ -0,0 +1,58 @@ +## Why + +An ingestor-owned background task (the `live-usage-ingestor` consumer or the +trailing cache-invalidation sleeper) that dies with an exception after its +owner lost track of it — for example a shutdown cancelled between clearing the +singleton and awaiting the task — surfaces only as a nondeterministic +"Task exception was never retrieved" loop warning at garbage-collection time. +In production that hides the failure until an arbitrary later moment; in the +test suite's shared session loop it poisons unrelated tests (issue #1755: +the otel lifespan-drain test and test_proxy_utils' startup-probe assertions +fail together). + +## What Changes + +- Enroll every task the live-usage ingestor creates in a weak ownership + registry and attach a done callback that settles the task at completion: + retrieve its exception, log it immediately with its traceback, and record it + in a bounded in-process failure handoff. +- Settle each task exactly once (a settled-task registry gates recording) so + the callback and any external sweep cannot double-report. +- Make the ingestor lifecycle instance-scoped: the app lifespan holds the + instance `start_live_usage_ingestor()` returned and passes it to + `stop_live_usage_ingestor(instance)`; stop only clears the module global + and publisher registration when that instance still owns them. Two live + lifespans in one process (a portal-loop `TestClient` nested inside an app + already running on the suite's session loop) previously orphaned the outer + ingestor: the nested startup overwrote the module global — the orphan's + only strong root — leaving an unreferenced cycle whose consumer task the + cyclic GC destroyed mid-await (`cannot reuse already awaited coroutine`), + and the nested shutdown cleared the global so the outer shutdown stopped + nothing. +- Keep ingestion behavior unchanged: enqueueing, coalescing, throttling, + shutdown ordering, and the fire-and-forget contract are untouched. +- Test infrastructure (out of spec scope): an autouse fence stops leaked + ingestor singletons after every test and drains the failure handoff so no + ingestor task or unretrieved exception crosses a test boundary. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `live-usage-ingestion`: unexpected ingestor-owned task deaths MUST be + settled at completion — exception retrieved, logged, and recorded in a + bounded handoff — instead of surfacing as garbage-collection-time + unobserved-task warnings. + +## Impact + +`app/modules/usage/live_ingest.py` (task enrollment, done-callback +settlement, bounded failure record, instance-scoped stop), `app/main.py` +(lifespan holds and stops its own ingestor instance), `tests/conftest.py` +(leak fence), `tests/unit/test_live_ingest_leak_fence.py` and +`tests/integration/test_live_usage_ingest.py` (regression coverage). No API, +schema, setting, or dashboard change. diff --git a/openspec/changes/settle-live-ingest-task-failures/specs/live-usage-ingestion/spec.md b/openspec/changes/settle-live-ingest-task-failures/specs/live-usage-ingestion/spec.md new file mode 100644 index 0000000000..92d4c4707c --- /dev/null +++ b/openspec/changes/settle-live-ingest-task-failures/specs/live-usage-ingestion/spec.md @@ -0,0 +1,91 @@ +# live-usage-ingestion Delta + +## ADDED Requirements + +### Requirement: Ingestor-owned task failures are settled at completion + +Every background task the live usage ingestor creates MUST be settled when it +completes: if the task ends with an exception other than cancellation, the +exception MUST be retrieved at completion time, logged immediately with its +traceback, and recorded in a bounded in-process failure record as +traceback-free metadata (task name and exception representation) so the +record cannot retain the failed task's object graph. An +ingestor-owned task failure MUST NOT surface as a garbage-collection-time +unobserved-task warning. Each task MUST be settled exactly once, including +when an external supervisor (such as test infrastructure) also observes the +task. Settlement MUST NOT extend task lifetime, change ingestion behavior, or +affect the serving path. + +#### Scenario: Detached consumer death is logged deterministically + +- **GIVEN** a consumer task whose owner lost track of it (for example a stop + cancelled between clearing the singleton and awaiting the task) +- **WHEN** the task dies with an exception +- **THEN** the exception is retrieved and logged at completion time +- **AND** it is recorded in the bounded failure record +- **AND** no unobserved-task warning fires at garbage collection + +#### Scenario: Cancelled tasks settle silently + +- **WHEN** an ingestor-owned task ends by cancellation +- **THEN** settlement records no failure and logs no error + +#### Scenario: Failure record stays bounded + +- **WHEN** ingestor-owned tasks fail repeatedly without the record being + drained +- **THEN** the failure record retains at most its fixed capacity of entries +- **AND** every failure is still logged + +### Requirement: Ingestor lifecycle is instance-scoped + +Each application lifespan MUST hold the ingestor instance its startup created +and stop exactly that instance at shutdown. Stopping an instance MUST touch +the process-wide singleton registration and the publisher hook only when the +stopped instance still owns them; when it does own them, the most recently +displaced ingestor that is still running MUST be restored as the registration +and publisher. Restoration eligibility is defined as: the candidate holds an +existing consumer task whose `done()` is false — this excludes consumers that +failed or completed, and ingestors whose `stop()` already cleared their +consumer. An instance MUST be removed from restoration tracking before its own +shutdown begins, so a stopping or stopped instance can never be restored +later. When several +lifespans are live in one process, no lifespan's startup or shutdown may +orphan another lifespan's running ingestor, leave it without a stop path, or +leave it registered-less while it still runs. + +#### Scenario: Nested lifespan cannot orphan the outer ingestor + +- **GIVEN** an app whose lifespan started ingestor A +- **WHEN** a nested lifespan starts ingestor B (taking over the singleton and + publisher) and later stops it +- **THEN** ingestor A keeps running, strongly rooted by its own lifespan +- **AND** the outer lifespan's shutdown stops ingestor A and its tasks + +#### Scenario: Nested shutdown restores the outer registration + +- **GIVEN** an app whose lifespan started ingestor A +- **AND** a nested lifespan whose startup displaced A by registering + ingestor B +- **WHEN** the nested lifespan stops ingestor B +- **THEN** ingestor A is restored as the singleton registration and publisher +- **AND** publications after the nested exit flow to ingestor A and are + ingested +- **AND** a displaced ingestor that already stopped is not restored (the + registration falls through to the next still-running displaced instance, + or is cleared) + +#### Scenario: A failed displaced ingestor is not restored + +- **GIVEN** displaced ingestor A whose consumer task has settled with an + exception (its task `done()` is true) +- **WHEN** the current registration stops +- **THEN** A is skipped by restoration (fall through to the next eligible + displaced instance, or clear the registration) + +#### Scenario: A stopping displaced ingestor is not restored + +- **GIVEN** displaced ingestor A whose `stop()` has begun (A was removed from + restoration tracking before its shutdown started) +- **WHEN** the current registration stops concurrently +- **THEN** A is never restored, even if its consumer task has not yet finished diff --git a/openspec/changes/settle-live-ingest-task-failures/tasks.md b/openspec/changes/settle-live-ingest-task-failures/tasks.md new file mode 100644 index 0000000000..01b1cdb4c6 --- /dev/null +++ b/openspec/changes/settle-live-ingest-task-failures/tasks.md @@ -0,0 +1,34 @@ +## 1. Implementation + +- [x] 1.1 Enroll ingestor-created tasks (consumer, trailing invalidation) in a + weak ownership registry with named tasks. +- [x] 1.2 Attach a done callback that settles each task exactly once: + retrieve the exception, log it with its traceback, and record it in the + bounded failure handoff. +- [x] 1.3 Add the autouse test fence that stops leaked ingestor singletons, + reaps pending ingestor-owned tasks (only those bound to the loop the + reap runs on; foreign-loop tasks are enrolled and left inert), and + drains the failure handoff after every test. +- [x] 1.4 Make the lifecycle instance-scoped: the lifespan holds the started + instance and stops exactly that instance; stop touches the module global + and publisher only when the stopped instance still owns them, so nested + lifespans cannot orphan the outer ingestor into a GC-collectable cycle. +- [x] 1.5 Restore the displaced registration after nested shutdown: a startup + that displaces a still-running instance remembers it (LIFO stack), and + stopping the current instance restores the most recent displaced + instance that still runs — never a stopped or dead one — so the outer + lifespan's ingestion resumes instead of going deaf. + +## 2. Validation + +- [x] 2.1 Order-dependent regression pair proving a leaked consumer no longer + crosses a test boundary (fails on main without the fence). +- [x] 2.2 Regressions for dead-consumer settlement, orphaned-task sweep, + detached-death recording, queued-callback settlement, and exactly-once + reporting. +- [x] 2.3 Regression for nested lifespans: a nested start/stop pair must not + orphan, kill, or unhook the outer lifespan's ingestor; the nested stop + restores the outer registration (a post-exit publication is ingested + end to end), and the outer stop reaps its own consumer. +- [x] 2.4 Run the full unit suite, live-usage integration tests, lint, type + checks, and strict OpenSpec validation. diff --git a/openspec/changes/settle-live-usage-after-account-consolidation/design.md b/openspec/changes/settle-live-usage-after-account-consolidation/design.md new file mode 100644 index 0000000000..ad42f7de6c --- /dev/null +++ b/openspec/changes/settle-live-usage-after-account-consolidation/design.md @@ -0,0 +1,177 @@ +## Context + +Live usage publication and account reconciliation run in different ownership +domains. The proxy captures a snapshot and enqueues it without waiting; the +single background consumer later opens its own database session. Meanwhile, +identity-aware account upsert can select canonical account `C`, reparent the +persisted children of duplicate `D`, and delete `D` in one transaction. + +The loss sequence is therefore deterministic: publication records only `D`; +consolidation commits `D -> C`; `_ingest` attempts to append with stale `D`; +the account foreign key rejects the write; and the serving-safe consumer logs +and drops it. Existing history reparenting cannot cover a row that did not +exist when consolidation ran. + +The relevant identity constraint is equally important: an upstream ChatGPT +account id can be shared by distinct real-email slots. Upstream identity is a +safe fallback only when it resolves to exactly one surviving local row. + +## Goals / Non-Goals + +**Goals:** + +- Preserve every already-captured live snapshot across same-slot duplicate + consolidation when one canonical owner survives. +- Keep publication non-blocking and persistence in the background consumer. +- Prefer a valid captured local owner; use upstream identity only to recover a + stale or absent local owner and only when the result is unique. +- Persist each accepted snapshot once under one owner, with all represented + windows committed atomically. +- Prove the stale-local, valid-local, and upstream-only paths without sleeps or + timing-dependent scheduling. + +**Non-Goals:** + +- Changing duplicate-account selection, shared-workspace slot preservation, or + canonical-account choice. +- Guessing between multiple local rows that share an upstream identity. +- Retrying arbitrary ingestion failures or changing the queue's drop-oldest, + throttling, or serving-path isolation behavior. +- Adding a schema migration, configuration flag, or API response field. + +## Decisions + +### D1: Queue an ownership envelope containing local and upstream identities + +Every proxy tap point that knows a local serving account and its upstream +ChatGPT account id will publish both. The queued item remains an in-memory typed +value containing `account_id`, `chatgpt_account_id`, and the snapshot; no +database or wire schema is introduced. Upstream-only callers continue to leave +the local id absent. + +Capturing the upstream id at publication time is necessary because `D` cannot +be queried after consolidation deletes it. Looking up the upstream id only +after detecting stale `D` would already have lost the recovery key. + +### D2: Select and protect the persistence owner at consume time + +Ingestion will settle ownership in this order: + +1. If the captured local id still identifies an account, select it even when + the upstream identity is absent, shared, or points at another candidate. +2. If the local id is absent or no longer exists, resolve the captured upstream + id against current account rows and select it only when exactly one row + survives. +3. If neither rule selects an owner, do not guess; retain the current logged, + serving-safe drop behavior. + +Owner selection and the atomic append of all represented usage windows belong +to one serialized write operation. SQLite acquires `BEGIN IMMEDIATE` before +lookup and keeps its database-wide writer serialization through commit. +PostgreSQL first acquires the existing transaction-scoped advisory-lock +namespace keyed by the captured upstream identity before owner lookup. It then +reads the local owner's current identity without a row lock. When that current +non-null identity is not covered, settlement rolls back to release the initial +lock, reacquires the captured/current identities through the shared canonical +sort, and reselects the owner. This rollback is required: acquiring the current +identity while retaining the captured lock could invert the account-writer lock +order. If reconciliation wins the current-identity lock and deletes the local +row, settlement uses the last observed current identity as the unique fallback. +The reselected owner is held `FOR NO KEY UPDATE` through the append. That row +lock blocks deletion and key-changing writes without blocking the `KEY SHARE` +lock taken by concurrent foreign-key inserts. One relock is allowed; a second +identity change raises a typed terminal error, and null identities add no lock +key. + +Every PostgreSQL writer that can add, replace, move, consolidate, or delete an +`Account.chatgpt_account_id` membership acquires that same upstream lock and +holds it through commit. Old and incoming non-null identities are converted to +the stable advisory keys and acquired in canonical sorted order before any +email/slot advisory locks, account row locks, fold-state lock, or writes. A +local-id writer first reads the current identity without a row lock, acquires +the sorted old/new identity locks, and then row-locks and re-reads the account; +a changed observation rolls back and repeats that lock acquisition at most +once. Upsert candidate changes use the same bounded rollback/restart before any +mutation. Membership re-reads use PostgreSQL `FOR NO KEY UPDATE`, which +stabilizes identity changes while remaining compatible with the `KEY SHARE` +locks taken by concurrent fold rollup foreign-key inserts; deletion upgrades +its lock only after acquiring the fold-state lock. The shared helper applies a +transaction-local 30-second PostgreSQL lock timeout before advisory acquisition, +so request and background transactions propagate lock contention instead of +waiting indefinitely; it performs no polling or retry. + +This ordering gives both legal interleavings the same outcome: a snapshot +committed before consolidation is included when history is reparented, while a +snapshot whose current-identity reconciliation wins first relocks and writes +directly to `C` after the local duplicate disappears. +The per-account fingerprint is evaluated against the selected current owner, +and the successful-write marker is updated only after the atomic append. One +queued item therefore cannot write once to stale `D` and again to `C`. + +### D3: Preserve account-slot ambiguity and consolidation policy + +The fallback reuses the existing unique-upstream resolution rule. Distinct +real-email slots sharing one ChatGPT workspace remain distinct and ambiguous; +the change does not merge them or choose one. Duplicate reconciliation keeps +its current email/workspace candidate filters and canonical selection. It only +runs when the incoming upstream identity is non-null, and its duplicate query +requires `Account.chatgpt_account_id == incoming_identity`; an identity-less +local row therefore cannot be selected or deleted as an identity-reconciliation +duplicate. It only needs to leave the canonical row's existing upstream +identity intact, which it already does. + +This choice rejects two alternatives: always preferring upstream identity +could cross account slots even while the serving local row is valid, and +changing consolidation to force uniqueness would violate the established +shared-workspace account-slot contract. + +### D4: Deterministic regression and authenticated surface QA + +The deterministic transaction regression captures a queued item for `D` with +the shared upstream identity and coordinates independent PostgreSQL sessions at +exact lock and commit events, with no sleep, polling delay, or retry. Database +assertions prove one row per represented window under `C`, no row under `D`, +and no duplicate snapshot in both transaction orderings. A composition test +also drives the real proxied SSE publication tap through the live hub and +background consumer after consolidation, awaiting the exact settlement event +with a bounded timeout. Separate controls prove that an existing local id wins +and that an upstream-only item still resolves uniquely. + +Manual QA will use an isolated database and authenticated backend, execute a +literal `curl -i` request to `GET /api/accounts`, and verify HTTP 200, one +canonical `C`, no `D`, and the injected primary and secondary usage values. +The database diff will independently show one canonical snapshot and no +duplicate-owned history. All QA processes, credentials, database files, ports, +and temporary artifacts will be removed after capture. + +## Risks / Trade-offs + +- **Shared upstream id remains ambiguous.** A stale item can still be dropped + when multiple real-email slots survive. This is deliberate: preserving slot + ownership is safer than attributing usage to the wrong account. +- **Captured upstream identity can be absent.** Publication preserves the valid + local id together with the nullable upstream field, so valid-local settlement + still succeeds. Identity reconciliation cannot delete that identity-less row: + reconciliation requires a non-null incoming identity and selects duplicates + by equality to it. If the local row is already stale, no upstream fallback can + be recovered; genuinely upstream-less callers retain that serving-safe drop. +- **Settlement races consolidation.** A selected-row lock protects a snapshot + when settlement wins the row, but a current-identity consolidator can win + first and delete the local owner while settlement holds only the stale + captured-identity lock. SQLite writer serialization and PostgreSQL's bounded + rollback/relock close both transaction orderings without acquiring locks out + of canonical order. +- **Atomic append changes failure granularity.** If one represented window + cannot be stored, none of that snapshot's windows commit. This is preferable + to a partial snapshot and supports exactly-once settlement. + +## Migration Plan + +Ship publication and ingestion changes atomically. There is no schema or data +migration and no backfill: only snapshots captured after deployment carry both +identities. Rollback reverts the code; existing in-memory queued items disappear +with process shutdown exactly as they do today. + +## Open Questions + +None. diff --git a/openspec/changes/settle-live-usage-after-account-consolidation/proposal.md b/openspec/changes/settle-live-usage-after-account-consolidation/proposal.md new file mode 100644 index 0000000000..b53edb3ec6 --- /dev/null +++ b/openspec/changes/settle-live-usage-after-account-consolidation/proposal.md @@ -0,0 +1,47 @@ +## Why + +A live usage snapshot can be captured for duplicate local account `D` and wait +in the fire-and-forget queue while account reconciliation consolidates `D` into +canonical account `C`. Reconciliation reparents existing history and deletes +`D`, but the delayed ingestor still trusts the captured local id. Its +usage-history insert then violates the account foreign key and the serving-safe +consumer drops the already-captured snapshot. The invariant for this change is: +**an already-captured live snapshot survives duplicate-account consolidation.** + +## What Changes + +- Queue both the serving local account id and its upstream ChatGPT account id + when both identities are available at proxy publication time. +- Settle ownership at ingestion time: prefer a still-valid local account; + otherwise resolve the captured upstream identity only when it identifies one + surviving canonical account. +- Preserve the upstream-only publication path and the existing ambiguity rule + for shared-workspace identities. +- Persist one accepted snapshot atomically under the selected owner so a stale + `D` produces exactly one primary/secondary snapshot under `C` and no history + under `D`. +- Add deterministic, no-sleep regression coverage and authenticated + `/api/accounts` QA for the externally visible canonical result. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `live-usage-ingestion`: retain both ownership identities and settle queued + snapshots against the current account rows before persistence. +- `account-identity`: keep duplicate consolidation's canonical identity usable + for delayed ownership settlement without changing which accounts consolidate. + +## Impact + +- Affected code: live-usage publication call sites and hub contract, + `app/modules/usage/live_ingest.py`, and the existing atomic usage-snapshot + persistence path. +- Affected tests: focused live-ingestion integration coverage for stale-local, + valid-local, and upstream-only ownership paths. +- No database schema migration, new setting, API schema change, or account + consolidation policy change. diff --git a/openspec/changes/settle-live-usage-after-account-consolidation/specs/account-identity/spec.md b/openspec/changes/settle-live-usage-after-account-consolidation/specs/account-identity/spec.md new file mode 100644 index 0000000000..4cbaa1fd9e --- /dev/null +++ b/openspec/changes/settle-live-usage-after-account-consolidation/specs/account-identity/spec.md @@ -0,0 +1,23 @@ +## ADDED Requirements + +### Requirement: Duplicate consolidation preserves a recoverable canonical identity + +Identity reconciliation MUST preserve the upstream ChatGPT account id on the canonical row, reparent existing account-owned usage history to that row, and remove selected duplicate rows when it consolidates duplicate local accounts under the existing email and workspace-slot policy. Reconciliation MUST NOT consolidate distinct real-email account slots solely to make an upstream identity unique. On PostgreSQL, every account insertion, replacement, token/metadata identity update, duplicate consolidation, and deletion that changes upstream-identity membership MUST acquire the same transaction-scoped upstream-identity advisory lock as live-usage settlement before row or fold-state locks and hold it through commit. An old-to-new membership move MUST acquire both stable identity lock keys in canonical sorted order. + +#### Scenario: Same-slot duplicate leaves one upstream-resolvable canonical row + +- **GIVEN** canonical account `C` and duplicate account `D` are selected for consolidation by the existing identity policy +- **AND** both rows carry the same upstream ChatGPT account id +- **WHEN** reconciliation consolidates `D` into `C` +- **THEN** `C` remains with that upstream ChatGPT account id +- **AND** existing usage history formerly owned by `D` is owned by `C` +- **AND** `D` no longer exists +- **AND** the upstream ChatGPT account id resolves uniquely to `C` + +#### Scenario: Shared-workspace sibling slots remain distinct + +- **GIVEN** two current accounts have different real email addresses +- **AND** they share the same upstream ChatGPT account id +- **WHEN** identity reconciliation evaluates the accounts +- **THEN** it preserves both local account slots +- **AND** it does not consolidate either account solely to make upstream resolution unique diff --git a/openspec/changes/settle-live-usage-after-account-consolidation/specs/live-usage-ingestion/spec.md b/openspec/changes/settle-live-usage-after-account-consolidation/specs/live-usage-ingestion/spec.md new file mode 100644 index 0000000000..0b1d75ccb9 --- /dev/null +++ b/openspec/changes/settle-live-usage-after-account-consolidation/specs/live-usage-ingestion/spec.md @@ -0,0 +1,77 @@ +## ADDED Requirements + +### Requirement: Captured live snapshots survive account consolidation + +The proxy MUST enqueue both the serving local account id and the upstream +ChatGPT account id when both are available. At consumption, live usage +ingestion MUST prefer the captured local id when it still identifies an +account. If that local id is absent or no longer exists, ingestion MUST use the +captured upstream id only when it resolves to exactly one current local account. +For a selected owner, ingestion MUST atomically persist no more than one history +row for each window represented by the queued snapshot. On PostgreSQL, +ingestion MUST acquire a transaction-scoped advisory lock keyed by the captured +upstream identity before either owner lookup and hold it through snapshot +commit. If the selected local owner's current non-null upstream identity is not +already locked, ingestion MUST roll back the initial transaction, reacquire the +captured and current identity locks in canonical sorted order, and reselect and +revalidate the owner before persistence. If that local owner was consolidated +while the current-identity lock was acquired, ingestion MUST use the last +observed current identity only when it resolves to exactly one surviving local +account. Ingestion MUST perform at most one such relock and MUST raise a typed +error if the selected owner's identity changes again; a null current identity +MUST NOT create an advisory-lock key. Every account writer that can change +membership in an upstream identity MUST acquire the same lock before row locks +or mutation and hold it through commit. Writers moving membership between two +non-null upstream identities MUST acquire both stable lock keys in canonical +sorted order. + +#### Scenario: Stale duplicate settles under the unique canonical account + +- **GIVEN** a primary/secondary live snapshot was queued for duplicate account `D` +- **AND** the queued item contains `D` and the upstream identity shared with canonical account `C` +- **AND** duplicate reconciliation reparents existing history to `C` and deletes `D` +- **WHEN** the queued snapshot is consumed +- **THEN** exactly one primary row and one secondary row are persisted under `C` +- **AND** no usage-history row is persisted under `D` +- **AND** the persisted values equal the captured snapshot + +#### Scenario: A valid local owner takes precedence + +- **GIVEN** a queued snapshot contains a local account id that still exists +- **AND** it also contains an upstream identity usable for fallback +- **WHEN** the queued snapshot is consumed +- **THEN** the snapshot is persisted under the captured local account +- **AND** ingestion does not substitute another account selected by the upstream identity + +#### Scenario: A selected owner's current identity is revalidated + +- **GIVEN** a queued snapshot contains local account `A` and captured identity `X` +- **AND** `A` currently belongs to identity `Y` +- **WHEN** settlement overlaps reconciliation of `A` into a canonical `Y` owner +- **THEN** settlement releases its initial `X` lock before acquiring the canonical sorted lock set for `X` and `Y` +- **AND** settlement reselects and revalidates the owner under that full lock set +- **AND** exactly one row per represented window survives under the canonical `Y` owner +- **AND** a second selected-owner identity change raises a typed terminal error without persisting the snapshot + +#### Scenario: Upstream-only publication still resolves + +- **GIVEN** a queued snapshot has no local account id +- **AND** its upstream identity resolves to exactly one current local account +- **WHEN** the queued snapshot is consumed +- **THEN** the snapshot is persisted once under that local account + +#### Scenario: Consolidation cannot delete a snapshot inserted after reparenting + +- **GIVEN** PostgreSQL settlement has selected duplicate `D` for a captured upstream identity +- **AND** reconciliation would reparent `D` history to `C` and then delete `D` +- **WHEN** settlement and reconciliation overlap across independent sessions +- **THEN** their shared transaction-scoped upstream-identity lock serializes the complete membership change +- **AND** the snapshot is either committed under `D` before reparenting or directly under `C` after reconciliation +- **AND** exactly one row per represented window survives under `C` + +#### Scenario: Ambiguous fallback does not guess an owner + +- **GIVEN** the captured local account id is absent or no longer exists +- **AND** the captured upstream identity matches multiple current local accounts +- **WHEN** the queued snapshot is consumed +- **THEN** no usage-history row is persisted for that snapshot diff --git a/openspec/changes/settle-live-usage-after-account-consolidation/tasks.md b/openspec/changes/settle-live-usage-after-account-consolidation/tasks.md new file mode 100644 index 0000000000..aa25b1fca3 --- /dev/null +++ b/openspec/changes/settle-live-usage-after-account-consolidation/tasks.md @@ -0,0 +1,72 @@ +## 1. Deterministic regression coverage + +- [x] 1.1 Add a no-sleep integration test that queues a primary/secondary live + snapshot for duplicate `D` with local and upstream identities, completes + same-slot reconciliation into canonical `C`, then directly consumes the + captured item. +- [x] 1.2 Assert exactly one persisted row for each represented window under + `C`, no usage row under `D`, no duplicate snapshot, and preservation of the + injected usage/reset/credits values. +- [x] 1.3 Add controls proving a still-valid local id is preferred even when an + upstream fallback exists, and an upstream-only queued item still resolves to + its unique local account. +- [x] 1.4 Capture the focused failing-first command and RED output before any + production edit; do not use sleeps, polling delays, retries, or a background + consumer timing race. +- [x] 1.5 Add a deterministic two-session PostgreSQL regression that pauses at + exact transaction events and proves consolidation cannot reparent before a + snapshot append and then cascade-delete that append. +- [x] 1.6 Add both legal PostgreSQL interleavings for queued identity `X` when + the selected local owner currently belongs to `Y`, including the causal RED + where `Y` reconciliation wins the owner row and deletes it before lookup. + +## 2. Publication ownership envelope + +- [x] 2.1 Retain both local account id and upstream ChatGPT account id in the + typed live-usage hub/queue contract. +- [x] 2.2 Update every local-account HTTP/SSE and WebSocket publication tap + point to supply the upstream identity when available, preserving the existing + upstream-only path and no-op hub behavior. + +## 3. Consume-time settlement + +- [x] 3.1 Resolve the persistence owner in the background ingestion session: + prefer an existing local row; if it is stale or absent, accept only one + current row matching the captured upstream identity. +- [x] 3.2 Protect owner resolution through persistence and write all represented + windows atomically so the item settles once under one account on SQLite and + PostgreSQL; use one shared transaction-scoped upstream-identity lock across + settlement, ordinary/slot upserts, replacement, rotation, metadata update, + consolidation, and deletion before row/fold locks. +- [x] 3.3 Keep ambiguous/missing ownership serving-safe and logged; do not alter + account consolidation policy, queue overflow, throttling, or retry behavior. +- [x] 3.4 Add no Alembic revision, model column, setting, or API schema change. +- [x] 3.5 Roll back before bounded relock of the canonical captured/current + identity set, reselect and revalidate ownership, and raise a typed terminal + error on a second identity change without fabricating a null lock key. + +## 4. Automated verification + +- [x] 4.1 Run the focused live-ingestion integration selection once to GREEN, + proving stale-local consolidation, valid-local preference, and upstream-only + resolution. +- [x] 4.2 Run diagnostics on every changed Python file and the affected backend + lint/type/test gates on both supported database paths where registered. +- [x] 4.3 Run `openspec validate settle-live-usage-after-account-consolidation --strict`. +- [x] 4.4 Run the deterministic PostgreSQL race repeatedly plus lock-routing, + identity, live-ingest, snapshot, and HTTP publication regressions after the + shared lock implementation is complete. +- [x] 4.5 Run the selected-owner identity race in both transaction orders and + the focused no-relock, one-relock, terminal-change, rollback, sorted-lock, + and null-identity unit coverage. + +## 5. Authenticated QA and cleanup + +- [x] 5.1 Start an isolated QA database/backend, reproduce `D -> C` settlement, + and execute authenticated `curl -i GET /api/accounts` with the QA bearer key. +- [x] 5.2 Capture HTTP 200 evidence showing exactly one canonical `C`, no `D`, + and the injected primary and secondary usage values; capture an independent + database diff showing one canonical row per represented window and no + duplicate-owned row. +- [x] 5.3 Stop and remove every QA process, listener, credential, database file, + and temporary artifact; record the cleanup receipt. diff --git a/openspec/changes/settle-terminal-spool-append-failure/.openspec.yaml b/openspec/changes/settle-terminal-spool-append-failure/.openspec.yaml new file mode 100644 index 0000000000..f161d5cc47 --- /dev/null +++ b/openspec/changes/settle-terminal-spool-append-failure/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-16 diff --git a/openspec/changes/settle-terminal-spool-append-failure/design.md b/openspec/changes/settle-terminal-spool-append-failure/design.md new file mode 100644 index 0000000000..59f11dd605 --- /dev/null +++ b/openspec/changes/settle-terminal-spool-append-failure/design.md @@ -0,0 +1,38 @@ +## Context + +The terminal HTTP-bridge event path intentionally skips a separate operation-state update because `append_terminal_operation_event` normally stores the terminal event and authoritative state atomically. If that repository call raises, the batcher currently logs and returns `False`; the durable row can therefore remain `acknowledged` with an incomplete spool after the terminal event was delivered downstream. + +The durable bridge exposes operation settlement under the same operation, session, instance, and owner-epoch fence. Fallback additionally needs to identify the acknowledged terminal attempt so a delayed write cannot overwrite a newer retry admitted under the same owner epoch. + +## Goals / Non-Goals + +**Goals:** + +- Preserve an authoritative terminal operation state after a terminal append exception. +- Apply the existing session and owner-epoch fence plus an acknowledged-state and response-identity comparison to fallback settlement. +- Keep the incomplete spool ineligible for transcript replay. +- Prove the result through the production repository/coordinator seam. + +**Non-Goals:** + +- Drain queued events during graceful shutdown. +- Change successful terminal event persistence or replay eligibility. +- Change warmup, upstream delivery, retry policy, or public response shapes. + +## Decisions + +- On `append_terminal_operation_event` exception, return an incomplete append result that explicitly requires fallback settlement. The relay queues the selected terminal SSE block and end-of-stream marker before awaiting a dedicated conditional settlement with the same operation ID, session ID, instance ID, owner epoch, immutable recovery-dispatch generation, intended terminal state, persisted upstream response IDs, and client-visible response ID. The repository accepts only that acknowledged attempt or its already-committed terminal result, without keeping the terminal event behind a stalled fallback write. +- Keep append and fallback settlement structured in the relay task instead of detaching them. This bounds settlement concurrency to active relay operations, and the relay defers cancellation until the append and any required settlement finish before preserving the cancellation outcome. +- Force `event_spool_complete=false` in the same conditional fallback update. This keeps replay disabled even when terminal append committed but its commit acknowledgement was lost before the caller observed success. +- Log a rejected fence or fallback exception inside the batcher's settlement method and do not re-raise. The terminal event has already been queued for downstream delivery, so bookkeeping failure must not replace or delay that event. +- Do not invoke fallback for ordinary `False` returns. The repository's bounded-spool overflow path already settles terminal state atomically, while a false owner fence must not be bypassed. + +## Risks / Trade-offs + +- [A transient database failure can affect both append and fallback update] -> Queue the terminal event and end-of-stream marker before awaiting structured fallback settlement and emit a warning for operator diagnosis. +- [Relay cancellation can interrupt terminal append or the delivery-authority claim] -> Defer cancellation through append, delivery ownership, and any required fallback; mark completed delivery authoritative before preserving cancellation. +- [A grouped terminal fan-out can release owner authority too early or stall on its first fallback] -> Start every owner-fenced append concurrently and await all append outcomes before exposing any sibling queue, then queue every sibling terminal event and end-of-stream marker before fallback settlement, settle every sibling before finalization, continue later finalizers after one fails, and preserve cancellation as the final outcome. +- [A stale owner could attempt to settle another owner's operation] -> Pass the unchanged session/instance/epoch fence and treat rejection as non-settlement. +- [A delayed append or fallback could overwrite a newer retry under the same owner] -> Require the prior attempt's immutable recovery-dispatch generation plus acknowledged/terminal state and response identity in both persistence predicates. +- [Replay aliases can differ from the upstream response identity persisted at acknowledgement] -> Carry both the active upstream identity and retained replay identity as possible CAS values separately from the client-visible terminal identity, covering a failed replacement-acknowledgement write, and preserve the known identity when no new client-visible identity is supplied. +- [A failed terminal append leaves no replayable terminal event] -> Keep `event_spool_complete` false and report `persisted=false`; authoritative state and transcript completeness remain separate facts. diff --git a/openspec/changes/settle-terminal-spool-append-failure/proposal.md b/openspec/changes/settle-terminal-spool-append-failure/proposal.md new file mode 100644 index 0000000000..b9693c33ee --- /dev/null +++ b/openspec/changes/settle-terminal-spool-append-failure/proposal.md @@ -0,0 +1,20 @@ +## Why + +A durable HTTP-bridge operation can remain `acknowledged` after its terminal event has already been delivered downstream when terminal transcript persistence raises. Reconnect and recovery must observe an authoritative terminal outcome rather than treating that operation as incomplete work. + +## What Changes + +- Settle the operation to its intended terminal state when atomic terminal-event append raises. +- Preserve the existing owner/session/epoch fence, reject settlement after a newer same-owner retry, and leave the event spool explicitly incomplete. +- Keep successful terminal append behavior unchanged. +- Add deterministic unit and production-repository recovery coverage for the failure path. + +## Capabilities + +### Modified Capabilities + +- `responses-api-compat`: terminal HTTP-bridge operation settlement remains authoritative when terminal transcript persistence fails. + +## Impact + +The HTTP-bridge event batcher, focused durable bridge tests, and reconnect/recovery semantics are affected. Public request and response shapes, graceful-shutdown draining, and warmup behavior are unchanged. diff --git a/openspec/changes/settle-terminal-spool-append-failure/specs/responses-api-compat/spec.md b/openspec/changes/settle-terminal-spool-append-failure/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..bd0158a3f0 --- /dev/null +++ b/openspec/changes/settle-terminal-spool-append-failure/specs/responses-api-compat/spec.md @@ -0,0 +1,62 @@ +## ADDED Requirements + +### Requirement: Terminal append failure preserves authoritative settlement + +When durable append of a terminal HTTP-bridge event raises after the operation was acknowledged, the proxy MUST attempt to persist the intended terminal operation state through the same operation, session, instance, and owner-epoch fence. Cancellation MUST be deferred through the append and any required fallback settlement. The event spool MUST remain incomplete, and the persistence failure MUST NOT replace or block the terminal event and end-of-stream marker already selected for downstream delivery. A rejected or failed fallback settlement MUST be logged and MUST NOT bypass the owner fence or overwrite a newer operation attempt admitted under the same owner epoch. + +#### Scenario: Terminal append exception settles the current owner operation + +- **GIVEN** an acknowledged HTTP-bridge operation owned by the current session epoch +- **WHEN** durable terminal-event append raises +- **THEN** the operation is persisted in the intended terminal state +- **AND** its event spool remains incomplete +- **AND** the terminal event and end-of-stream marker are queued before fallback settlement can stall +- **AND** reconnect or recovery does not observe the operation as acknowledged work + +#### Scenario: Grouped failures deliver every sibling before settlement + +- **GIVEN** one upstream error selects terminal failures for multiple pending operations +- **WHEN** the first operation's fallback settlement stalls +- **THEN** every selected operation attempts its owner-fenced terminal append before any terminal queue is exposed +- **AND** every selected operation then receives its terminal event and end-of-stream marker before fallback settlement +- **AND** sibling delivery does not wait for the first fallback settlement +- **AND** cancellation is preserved as the final outcome only after every pre-delivered sibling finishes settlement and finalization +- **AND** one sibling's finalization failure does not prevent later siblings from settling or replace pending cancellation + +#### Scenario: Cancellation preserves terminal delivery authority + +- **GIVEN** terminal append finishes while relay cancellation is deferred +- **WHEN** the append result becomes available +- **THEN** the terminal event and end-of-stream marker are queued +- **AND** a completed-delivery scope is marked authoritative before cleanup can deactivate it +- **AND** cancellation during that delivery-authority claim does not skip required fallback settlement +- **AND** cancellation is preserved only after delivery and required settlement + +#### Scenario: Stale owner cannot settle after terminal append exception + +- **GIVEN** an HTTP-bridge operation whose owner epoch has advanced +- **WHEN** the stale batcher encounters a terminal-event append exception +- **THEN** fallback settlement is rejected by the durable owner fence +- **AND** the stale batcher does not mutate the operation state + +#### Scenario: Newer retry rejects delayed fallback settlement + +- **GIVEN** terminal append committed its operation state before reporting an exception +- **AND** a retry under the same owner epoch has since reset the operation to submitted +- **WHEN** fallback settlement for the prior attempt runs +- **THEN** the fallback is rejected by an immutable recovery-attempt generation plus operation-state and persisted upstream-response identity fence +- **AND** the newer submitted attempt remains unchanged + +#### Scenario: Replay alias preserves the acknowledged-attempt fence + +- **GIVEN** a replay whose client-visible response alias differs from its persisted upstream response ID or whose active upstream response ID was reset before a replacement response was created +- **WHEN** durable terminal-event append raises +- **THEN** fallback settlement compares the acknowledged or already terminal operation against every response identity that may remain persisted when a replacement acknowledgement update fails +- **AND** persists the intended client-visible terminal response ID when present +- **AND** otherwise preserves the known upstream response ID + +#### Scenario: Successful terminal append remains atomic and replayable + +- **WHEN** durable terminal-event append succeeds +- **THEN** the terminal event and intended operation state are persisted atomically +- **AND** the completed event spool remains eligible for replay diff --git a/openspec/changes/settle-terminal-spool-append-failure/tasks.md b/openspec/changes/settle-terminal-spool-append-failure/tasks.md new file mode 100644 index 0000000000..231106f0d3 --- /dev/null +++ b/openspec/changes/settle-terminal-spool-append-failure/tasks.md @@ -0,0 +1,15 @@ +## 1. Regression + +- [x] 1.1 Add deterministic regressions proving a terminal append exception settles through the unchanged owner fence without overwriting a newer retry, blocking terminal EOF, or starving grouped siblings. +- [x] 1.2 Capture the focused regression failing before production code changes. + +## 2. Implementation + +- [x] 2.1 Add the minimum fenced fallback settlement for terminal append exceptions without claiming spool completeness. +- [x] 2.2 Add a production-repository process/recovery proof that a reconnect cannot observe the operation as acknowledged and delayed settlement cannot overwrite its retry. + +## 3. Verification + +- [x] 3.1 Capture focused GREEN and run adjacent HTTP-bridge unit and integration tests. +- [x] 3.2 Run changed-file formatting, lint, type diagnostics, strict OpenSpec validation, and package/build checks required by the repository. +- [x] 3.3 Review the committed diff independently and address every in-scope finding. diff --git a/openspec/changes/show-cancelled-request-logs/design.md b/openspec/changes/show-cancelled-request-logs/design.md new file mode 100644 index 0000000000..dc68d41a50 --- /dev/null +++ b/openspec/changes/show-cancelled-request-logs/design.md @@ -0,0 +1,85 @@ +## Context + +Cancellation persistence and aggregate accounting are already correct: +downstream disconnects are stored with `status='cancelled'`, and shared usage +logic classifies them as non-errors. The defect begins in the Request Logs +read path. Its filter model has booleans for success and error rows only, so +the raw query, filter facets, count cache, and demand-rollup count all omit +cancelled rows. If a cancelled row were allowed through independently, the +mapper would currently expose it as `error`. + +The frontend already transports arbitrary repeated status query values and +builds filter options from the server response. It needs only a localized, +visually distinct cancelled presentation once the backend exposes the status. + +## Goals / Non-Goals + +**Goals:** + +- Keep the unfiltered list, filtered list, status facet, and displayed total + aligned for cancelled rows. +- Preserve the invariant that cancelled and error are separate terminal + statuses. +- Reuse the existing Request Logs filter and badge primitives. + +**Non-Goals:** + +- Change cancellation persistence or proxy disconnect handling. +- Change error/cancellation aggregate metrics, Reports, or live usage. +- Add a migration, setting, navigation item, or new dashboard primitive. +- Backfill or rewrite historical request logs. + +## Decisions + +### Thread an explicit cancelled inclusion flag through the existing read path + +Add `include_cancelled` beside the existing success/error filter fields. The +default and `all` paths enable it; the explicit `cancelled` filter enables +only it; the explicit `error` filter continues to match persisted error rows +excluding rate-limit and quota codes. + +The flag also participates in the count-cache key and +`_DemandCountParams`. The demand-rollup count adds the persisted +`cancelled` status when enabled. This keeps pagination totals equal to the +rows that can actually be listed across folded and raw windows. + +Alternative considered: classify cancelled rows under the existing error +branch. That contradicts the canonical non-error contract and would make the +error filter semantically wrong. + +### Preserve additive frontend compatibility + +Keep the existing string-based API and URL status schemas. They intentionally +permit a mixed-version frontend to transport an unfamiliar additive status. +Add the known `cancelled` label, locale entries, and badge class without +turning the boundary into a closed enum. + +Alternative considered: replace status strings with a closed Zod enum. That +would reject future additive statuses and break the existing stale-filter +recovery behavior. + +### Reuse the current badge primitive + +Use the existing outline badge and its status-class map with a neutral sky +treatment distinct from success, rate-limit, quota, and error. No design token +or reusable component is introduced. + +## Risks / Trade-offs + +- [Risk] Raw rows become visible while folded totals remain too small + → Include cancelled in the demand-rollup status filter and count-cache + signature; cover total and filter behavior through the public API. +- [Risk] Cancelled leaks into the error filter + → Seed both cancellation and genuine error controls and assert each explicit + filter independently. +- [Risk] Mixed-version status values stop parsing + → Retain open string schemas and add presentation only for the known value. + +## Migration Plan + +Ship as an additive read-path and presentation change. Rollback restores the +previous omission without changing stored data or schema. + +## Open Questions + +None. diff --git a/openspec/changes/show-cancelled-request-logs/proposal.md b/openspec/changes/show-cancelled-request-logs/proposal.md new file mode 100644 index 0000000000..ed9c0bd6d3 --- /dev/null +++ b/openspec/changes/show-cancelled-request-logs/proposal.md @@ -0,0 +1,34 @@ +## Why + +Request logs persist downstream disconnects as `status='cancelled'`, but the +Request Logs read path only includes persisted `success` and `error` rows. +Cancelled requests therefore disappear from the unfiltered operator list and +cannot be selected through the status filter even though cancellation metrics +remain visible elsewhere. + +## What Changes + +- Include persisted cancelled rows in the default Request Logs listing and its + rollup-backed total. +- Expose cancelled rows as the distinct public status `cancelled`. +- Add a `cancelled` status facet whose filter remains separate from genuine + errors. +- Render and localize a distinct cancelled badge in the dashboard. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `usage-error-metrics`: Persisted cancellations remain visible and + distinguishable from errors on the Request Logs operator surface. + +## Impact + +Request-log repository filtering/counting, service status mapping, dashboard +status labels, and focused backend/frontend regression tests. No database +migration, request producer, metric calculation, setting, navigation item, or +deployment change. diff --git a/openspec/changes/show-cancelled-request-logs/specs/usage-error-metrics/spec.md b/openspec/changes/show-cancelled-request-logs/specs/usage-error-metrics/spec.md new file mode 100644 index 0000000000..e13509a24e --- /dev/null +++ b/openspec/changes/show-cancelled-request-logs/specs/usage-error-metrics/spec.md @@ -0,0 +1,33 @@ +## ADDED Requirements + +### Requirement: Cancelled request logs remain visible and distinct + +The Request Logs operator surface MUST include persisted +`status='cancelled'` rows in its unfiltered listing and total. Such rows MUST +be exposed with public status `cancelled`, MUST be available through a +`cancelled` status option and filter, and MUST NOT be returned by the `error` +status filter. The dashboard MUST render the status with localized cancelled +copy and a visual treatment distinct from error. + +#### Scenario: Unfiltered Request Logs include cancellations + +- **GIVEN** one persisted cancelled request and one persisted genuine error +- **WHEN** an operator requests the unfiltered Request Logs list +- **THEN** both requests are returned and included in the total +- **AND** the cancelled request exposes public status `cancelled` + +#### Scenario: Cancelled and error filters remain separate + +- **GIVEN** one persisted cancelled request and one persisted genuine error +- **WHEN** an operator filters Request Logs by `cancelled` +- **THEN** only the cancelled request is returned +- **AND WHEN** the operator filters Request Logs by `error` +- **THEN** only the genuine error is returned + +#### Scenario: Dashboard presents a cancelled status + +- **GIVEN** Request Logs contain a persisted cancelled request +- **WHEN** the dashboard loads status options and renders the request +- **THEN** the status filter includes a localized Cancelled option +- **AND** the row and request details use the localized cancelled label +- **AND** the cancelled badge is visually distinct from the error badge diff --git a/openspec/changes/show-cancelled-request-logs/tasks.md b/openspec/changes/show-cancelled-request-logs/tasks.md new file mode 100644 index 0000000000..243a2c4a96 --- /dev/null +++ b/openspec/changes/show-cancelled-request-logs/tasks.md @@ -0,0 +1,21 @@ +## 1. Backend Request Logs contract + +- [x] 1.1 Add failing public API regressions for unfiltered, cancelled-filter, + error-filter, and status-option behavior +- [x] 1.2 Include cancelled rows in raw listing predicates, rollup-backed + totals, and cache signatures +- [x] 1.3 Expose persisted cancelled rows as public status `cancelled` + +## 2. Frontend presentation + +- [x] 2.1 Add a failing rendered-table regression for the localized, + non-error cancelled badge +- [x] 2.2 Add cancelled status fallback text, locale strings, and a distinct + existing-badge treatment + +## 3. Validation + +- [x] 3.1 Run focused backend and frontend regressions +- [x] 3.2 Run affected lint, type, build, and OpenSpec validation +- [x] 3.3 Exercise unfiltered, cancelled-filter, and error-filter behavior + through the running dashboard and capture before/after evidence diff --git a/openspec/changes/skip-empty-usage-reservations/design.md b/openspec/changes/skip-empty-usage-reservations/design.md new file mode 100644 index 0000000000..14f0aba9bc --- /dev/null +++ b/openspec/changes/skip-empty-usage-reservations/design.md @@ -0,0 +1,80 @@ +# Design + +## Where the skip lives + +`ApiKeysService._enforce_limits_for_request_once` builds +`reservation_items` by iterating the key's limits and appending one item +per **applicable** limit (including zero-delta items — the +"Zero-reservation limits still settle actual usage" requirement depends on +those items existing). `reservation_items` is therefore empty **iff** no +limit applies to the request. In that case the function returns `None` +before `create_usage_reservation` + `commit`. + +Safety of skipping the commit: with zero applicable limits no +`try_reserve_usage` CAS ran (and a zero-delta call is read-only), and the +lazy expired-limit reset commits inside `reset_limit` itself, so there is +no pending write to lose when admission returns early. The early return +still issues a `commit()` to close the implicit transaction opened by +the admission SELECTs: proxy call sites use short-lived background +sessions, but the quota-planner warmup service holds one long-lived +session, and leaving the read transaction open would pin an +idle-in-transaction window across the warmup probe's upstream round-trip. + +Why `commit()` and not `rollback()`: `AsyncSession.rollback()` expires +every tracked ORM instance **regardless of** `expire_on_commit=False`. +The warmup service shares its long-lived session with this repository and +already tracks `account` and `decision` rows; expiring them makes the +subsequent `_send_warmup_probe` access to `account.access_token_encrypted` +(and the error path's `decision.id`) raise `MissingGreenlet` — limit-free +warmups would never execute. `commit()` with `expire_on_commit=False` +(both session factories in `app/db/session.py`) leaves tracked state +loaded. It is semantically equivalent to a rollback at this point because +the open transaction holds only the admission SELECTs, and no unrelated +dirty state can be flushed by it: the proxy call sites dedicate a +fresh/scoped session to admission (`get_background_session`, +`_repo_factory`), and the quota-planner repositories commit every prior +write inside their own methods (`log_decision`, `update_decision_status`, +`claim_warmup_decision` — the latter even commits at the start of its own +transaction). Regression coverage drives the real `ApiKeysService` through +a shared session and asserts the probe's attributes stay readable. + +Settlement is not a pure no-op for the ledger only: `_settle_usage_reservation` +is also the production writer of the key's last-used touch (write-behind +coalescer → `api_keys.last_used_at`). Without a reservation settlement never +runs, so the limit-free admission path records the coalescer touch itself +before returning `None` — `last_used_at` keeps advancing for limit-free keys +exactly once per admitted request, at admission time instead of stream end. +The record is in-memory (no extra commit) and sits outside +`sqlite_writer_section()` for the same reason as settlement's: the shutdown +write-through flush takes the writer section itself. + +## Consumer audit (verified in code before implementation) + +| Consumer | Behavior on missing reservation | +| --- | --- | +| `_settle_stream_api_key_usage` (`api_key_usage.py`) | `api_key_reservation is None` → returns `True` (settled no-op) | +| `_settle_compact_api_key_usage` | `api_key_reservation is None` → returns | +| `_release_reservation` / `_release_reservation_best_effort` / `_finalize_image_reservation` / `_settle_source_reservation` (`proxy/api.py`) | `reservation is None` → return / `True` | +| `_release_websocket_reservation` / heartbeat `_maybe_touch_api_key_reservation` / heartbeat task start | `None` → no-op | +| HTTP bridge forwarding | reservation headers only added when non-`None`; `_reservation_from_headers` returns `None` when absent | +| Bridge retry re-reservation (`http_bridge/streaming.py`) | guarded by `api_key_reservation is not None`; `begin_bridge_lifecycle` accepts `None`. With a `None` reservation, `same_reservation` (`previous is reservation`) is `True` across submit retries, so deferred account error backoffs carry over instead of resetting per re-reservation. This is the **pre-existing** lifecycle semantic for every keyless request (`api_key=None` account-direct traffic exercises `begin_bridge_lifecycle(None)` on each retry today); limit-free keyed requests now intentionally join that class. Drain-once is preserved (the dict is carried by reference and popped on drain), and the wrapper-finally early-release branch not firing for both-`None` loses nothing: releasing a `None` reservation is a no-op and non-empty `pending_backoffs` still triggers the branch via the `or`. | +| Stale-release scheduler | operates on reservation rows; limit-free admissions simply produce none | +| Quota-planner warmup | **adapted**: `reservation_id` becomes `None` when admission returns no reservation; finalize/fail calls already guard on `reservation_id is not None` | + +## Interaction with `has_applicable_limits` + +`ApiKeyUsageReservationData.has_applicable_limits` stays (the bridge +header round-trip and `_reservation_requires_usage` read it), but a +returned reservation now always has it `True`; `None` replaces the former +"reservation exists but has no applicable limits" state. The +`_reservation_requires_usage(reservation)` predicate is unchanged and +degenerates to `reservation is not None`. + +## Rejected alternatives + +- Keeping the empty INSERT with relaxed durability: still pays the + round trips and the stream-end settlement transaction; #1665 pinned + reservation-ledger writes to full durability, so relaxing is off-limits. +- Returning a sentinel reservation without persisting it: every consumer + would need to learn the sentinel; `None` already has a fully audited + no-op path (the `api_key is None` case exercises it today). diff --git a/openspec/changes/skip-empty-usage-reservations/proposal.md b/openspec/changes/skip-empty-usage-reservations/proposal.md new file mode 100644 index 0000000000..f466969082 --- /dev/null +++ b/openspec/changes/skip-empty-usage-reservations/proposal.md @@ -0,0 +1,51 @@ +## Why + +Every keyed request pays the reservation ledger even when the key has no +applicable limits: admission INSERTs an empty `api_key_usage_reservations` +row (zero items) and runs a full-durability commit, and stream end runs the +whole settlement transaction just to flip that empty row to `finalized`. +For unlimited keys this is pure hot-path CPU and write amplification with +no enforcement value — there is nothing to reserve and nothing to settle. + +## What Changes + +- API-key admission returns no reservation when no configured limit applies + to the request (key has no limits, or none match the request model). The + reservation INSERT and its full-durability commit are skipped entirely. +- Downstream reservation consumers (stream/compact settlement, release + paths, heartbeat touch, quota-planner warmup finalize) already no-op on a + missing reservation; the quota-planner warmup executor is adapted to + tolerate admission returning no reservation. +- Because settlement is also the production writer of the key's last-used + touch, the limit-free admission path records the write-behind coalescer + touch itself, so dashboard-visible `last_used_at` keeps advancing for + limit-free keys (per admitted request, at admission time instead of + stream end). +- Keys with at least one applicable limit are unaffected: reservation + creation, full commit durability (#1665), exactly-once settlement, and + stale-reservation reclamation are unchanged for them. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `api-keys`: Add a reservation-ledger requirement that limit-free + admissions skip reservation creation and that downstream settlement, + release, and heartbeat paths no-op without a reservation. + +## Impact + +- `app/modules/api_keys/service.py`: `enforce_limits_for_request` (and the + single-attempt worker) return `None` when no reservation items exist. +- `app/modules/quota_planner/warmup.py`: warmup executor handles a `None` + reservation (probes without finalizing). +- Per-request effect for unlimited keys: one INSERT + one synchronous + full-durability commit removed from admission, and the entire stream-end + settlement transaction removed. No API, setting, dependency, migration, + or dashboard change. Operator-visible effect: keys without applicable + limits no longer produce reservation rows, so stale-reservation + reclamation counts no longer include them. diff --git a/openspec/changes/skip-empty-usage-reservations/specs/api-keys/spec.md b/openspec/changes/skip-empty-usage-reservations/specs/api-keys/spec.md new file mode 100644 index 0000000000..eb63781dcc --- /dev/null +++ b/openspec/changes/skip-empty-usage-reservations/specs/api-keys/spec.md @@ -0,0 +1,51 @@ +## ADDED Requirements + +### Requirement: Limit-free admissions skip the reservation ledger + +When API-key admission finds no applicable limit for a request (the key has no configured limits, or none of its limits apply to the request model), the system MUST NOT create a usage reservation row and MUST NOT run the reservation commit for that request. Admission MUST report that no reservation exists, and every downstream reservation consumer (stream and compact settlement, release paths, heartbeat touch, quota-planner warmup finalization) MUST treat the missing reservation as "nothing to settle" and no-op without error. Admission-time validity checks (key active, key not expired, lazy expired-limit reset) MUST still run unchanged. Because settlement — which records the key's last-used touch for reserved requests — never runs without a reservation, admission MUST record the last-used touch itself on the limit-free path so `last_used_at` continues to advance for these keys. Admission MUST also close the read transaction it opened before returning without a reservation, and MUST do so without expiring ORM state tracked by a caller-shared session (callers such as the quota-planner warmup service hold already-loaded rows on the same session and access them after admission). Keys with at least one applicable limit MUST continue to create reservations with per-limit items (including zero-delta items) and full commit durability. + +#### Scenario: Key without limits creates no reservation + +- **WHEN** admission runs for an API key with no configured limits +- **THEN** no usage reservation row is inserted and no reservation write is committed (the only commit issued closes the read-only admission transaction) +- **AND** the request is admitted without a reservation + +#### Scenario: Key whose limits do not apply to the request model creates no reservation + +- **WHEN** admission runs for a key whose limits all carry a `model_filter` that does not match the request model +- **THEN** no usage reservation row is inserted +- **AND** the non-matching limits' `current_value` values are unchanged + +#### Scenario: Limit-free admissions still advance last-used + +- **WHEN** admission runs for a key with no applicable limits +- **THEN** the key's last-used touch is recorded at admission via the write-behind coalescer +- **AND** the dashboard-visible `last_used_at` continues to advance for the key + +#### Scenario: Settlement, release, and heartbeat no-op without a reservation + +- **WHEN** a request admitted without a reservation finishes (success or failure) +- **THEN** settlement, release, and heartbeat-touch paths skip without error +- **AND** no settlement transaction runs for that request + +#### Scenario: Quota-planner warmup probes without a reservation + +- **WHEN** the quota-planner warmup executor admits its probe with a key that has no applicable limits +- **THEN** the warmup probe executes +- **AND** no reservation finalization is attempted + +#### Scenario: Limit-free admission preserves shared-session ORM state + +- **WHEN** a caller that holds already-loaded ORM rows on the same session (the quota-planner warmup service tracks the target account and decision) admits a request with a limit-free key +- **THEN** the admission read transaction is closed before admission returns +- **AND** the caller's tracked rows remain readable afterwards without reload errors, so the warmup probe executes + +#### Scenario: Stale-reservation reclamation sees no rows for limit-free admissions + +- **WHEN** stale usage-reservation reclamation runs after admissions for keys without applicable limits +- **THEN** those admissions contribute no reservations to reclaim + +#### Scenario: Limited keys are unaffected + +- **WHEN** admission runs for a key with an applicable limit +- **THEN** a reservation with per-limit items is created and committed exactly as before admission returned reservations unconditionally diff --git a/openspec/changes/skip-empty-usage-reservations/tasks.md b/openspec/changes/skip-empty-usage-reservations/tasks.md new file mode 100644 index 0000000000..242f323bb7 --- /dev/null +++ b/openspec/changes/skip-empty-usage-reservations/tasks.md @@ -0,0 +1,50 @@ +# Tasks + +## 1. Admission skip + +- [x] 1.1 Return `None` from `ApiKeysService._enforce_limits_for_request_once` + when `reservation_items` is empty (no applicable limits), skipping the + reservation INSERT and its commit; widen `enforce_limits_for_request` + return type to `ApiKeyUsageReservationData | None`. + +## 2. Consumer audit / adaptation + +- [x] 2.1 Verify settlement (`_settle_stream_api_key_usage`, + `_settle_compact_api_key_usage`), release paths + (`_release_reservation*`, `_release_websocket_reservation`), + heartbeat (`_maybe_touch_api_key_reservation`), bridge forwarding + (`_reservation_from_headers`), and retry re-reservation guards all + no-op on a `None` reservation. +- [x] 2.2 Adapt the quota-planner warmup executor to a `None` reservation + (probe without finalize). +- [x] 2.3 Record the key's last-used coalescer touch at admission on the + limit-free path (settlement, the production `last_used_at` writer, + never runs without a reservation). +- [x] 2.4 Roll back the admission read transaction before the limit-free + early return (long-lived sessions must not idle in transaction + across upstream round-trips). + +## 3. Tests + +- [x] 3.1 Unit: key without limits → admission returns `None`, no + reservation INSERT; the only commit is the read-only transaction close. +- [x] 3.2 Unit: key whose limits do not match the request model → `None`, + limits untouched. +- [x] 3.2b Unit: limit-free admission records the last-used coalescer touch + and closes the read transaction via commit — never rollback, which + would expire shared-session ORM state (quota-planner warmup). +- [x] 3.3 Unit: settlement/release/heartbeat with `reservation=None` no-op. +- [x] 3.4 Integration: quota-planner warmup executes with a limit-free key + (no finalize call, probe succeeds). +- [x] 3.4b Integration (regression): warmup through the REAL ApiKeysService + on the shared session — the probe's `account.access_token_encrypted` + access stays readable after the limit-free early return (no + MissingGreenlet from expired shared state). +- [x] 3.5 Integration: stale-release reclamation finds no rows after + limit-free admissions; limited keys keep creating reservations + (regression). + +## 4. Spec + +- [x] 4.1 Delta to `openspec/specs/api-keys/spec.md` reservation-ledger + requirements; validate with `openspec validate --specs`. diff --git a/openspec/changes/source-model-reasoning-metadata/proposal.md b/openspec/changes/source-model-reasoning-metadata/proposal.md new file mode 100644 index 0000000000..b5a7367afc --- /dev/null +++ b/openspec/changes/source-model-reasoning-metadata/proposal.md @@ -0,0 +1,80 @@ +## Why + +Source-model Codex catalog entries hardcode `supported_reasoning_levels=()`, +`default_reasoning_level=None`, and `supports_reasoning_summaries=False`. Every +other client-capability field on those entries is an operator-overridable +`raw_metadata_json` default, so a reasoning-capable backend has no way to +advertise its efforts and Codex clients show no reasoning-effort options for +model-source models. + +The efforts themselves already reach the source: the Responses path forwards +`reasoning` unchanged, so an operator who hardcodes `model_reasoning_effort` in +`config.toml` gets working reasoning today. Only the advertisement is missing, +which makes the capability undiscoverable in the client UI. + +Backends differ in the efforts they accept — for example Alibaba Model Studio +exposes `none`/`minimal`/`low`/`medium`/`high`/`xhigh`/`max`, while DeepSeek and +Kimi expose `low`/`high`/`max` — so the advertised set has to be operator +declared rather than inferred, and validated by shape rather than against a +fixed enum. That includes `none`, which the #1660 backend survey shows is a +real value (GLM `max`/`high`/`none`), and which is already first-class for +API-key enforced efforts in `app/modules/api_keys/service.py`; filtering it out +of source catalogs alone would have left the two vocabularies disagreeing. + +## What Changes + +- Read `supported_reasoning_levels`, `default_reasoning_level`, and + `supports_reasoning_summaries` for source-model catalog entries from + `raw_metadata_json` instead of hardcoding them. +- Accept both effort slugs (`["low", "high"]`) and objects + (`[{"effort": "low", "description": "..."}]`), ignoring malformed entries. +- Gate all of it on the existing `"supports_reasoning"` switch, the only + reasoning control the dashboard exposes, so a model with it off advertises + nothing and keeps the existing no-reasoning behavior. +- Normalize and deduplicate declared efforts, validating shape rather than + membership of a fixed vocabulary. +- Undo the unsupported-effort rewrite for requests that are actually routed to + a model source and that declared the effort, instead of inferring the route + from registry membership. This covers only the `minimal` workaround; the + `ultra` -> `max` wire alias mirrors the reference client and stays applied on + every surface. + +## Relationship to `supports_reasoning` + +`raw_metadata_json` carries reasoning keys with different jobs: + +- `supports_reasoning` is the **switch**. It is written by the dashboard's + single `Reasoning` checkbox and gates `sanitize_source_chat_payload`, which + strips `reasoning`, `reasoning_effort` and related toggles on the Chat + Completions path. +- `supported_reasoning_levels` / `default_reasoning_level` / + `supports_reasoning_summaries` are the **detail**: which efforts an opted-in + backend accepts. They are set through the API today; the dashboard UI for + them is the UI-only rebase of #1675 on this parser. + +Detail is gated on the switch. An earlier revision of this change instead made +declared levels imply the switch, which inverted the relationship: it turned a +description of *which* efforts into permission for reasoning at all, and left +the dashboard checkbox reading `false` for a model the backend treated as +opted in. Gating the other way keeps every surface consistent — `/v1/models` +derives `supports_reasoning` from the levels and the summary flag before +consulting the raw key, so a model advertising levels while the sanitizer +strips its chat requests would be visible and inert at once. With the gate, that +state is unreachable. + +The Responses path forwards `reasoning` regardless, as it does today: it is a +first-class field of the Responses schema that a source opts into with +`supports_responses`, unlike the chat path where three of the stripped keys are +vendor extensions that only survive because the request model allows extra +fields. Making the Responses path strip as well would reverse that existing +design decision and is out of scope here. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `model-catalog-compat` diff --git a/openspec/changes/source-model-reasoning-metadata/specs/model-catalog-compat/spec.md b/openspec/changes/source-model-reasoning-metadata/specs/model-catalog-compat/spec.md new file mode 100644 index 0000000000..71af684608 --- /dev/null +++ b/openspec/changes/source-model-reasoning-metadata/specs/model-catalog-compat/spec.md @@ -0,0 +1,193 @@ +## ADDED Requirements + +### Requirement: Source-model catalog entries advertise operator-declared reasoning efforts + +Codex catalog entries built for OpenAI-compatible source models MUST derive +`supported_reasoning_levels`, `default_reasoning_level`, and +`supports_reasoning_summaries` from the source model's `raw_metadata_json` +rather than reporting a fixed no-reasoning capability. + +Derivation MUST be gated on `"supports_reasoning": true`. That flag is the only +reasoning control the dashboard exposes, so a model whose operator left it off +MUST advertise no efforts, no default, and no summary support regardless of what +else the metadata declares. Levels say *which* efforts an opted-in backend +accepts, not *whether* reasoning is permitted, and gating them on the same flag +that gates the chat-completions sanitizer is what keeps the Codex catalog, +`/v1/models` and the dashboard checkbox in agreement. + +`supported_reasoning_levels` MUST accept a list of effort slugs and a list of +`{"effort", "description"}` objects. Entries that are neither a string nor a +mapping with a string `effort`, and duplicate efforts, MUST be ignored. A +non-list value MUST yield no advertised efforts. `default_reasoning_level` MUST +be reported only when it matches one of the advertised efforts. A source model +without reasoning metadata MUST continue to advertise no efforts, no default, +and no summary support. + +Declared efforts MUST be normalized (trimmed and lowercased) and +deduplicated. They MUST NOT be filtered against a fixed vocabulary: backends +disagree on which efforts exist -- `none` is real on GLM and Alibaba Model +Studio, while others stop at `low`/`high`/`max` -- so an enum would drop +efforts a provider genuinely accepts. Only shape is validated; an entry that +is not a string, a mapping without a string `effort`, or an empty slug MUST be +dropped. + +#### Scenario: Effort slugs are advertised in declaration order + +- **GIVEN** a source model whose `raw_metadata_json` sets + `"supported_reasoning_levels": ["low", "medium", "high", "xhigh"]` and + `"default_reasoning_level": "high"` +- **WHEN** a client fetches the Codex model catalog +- **THEN** the entry advertises efforts `low`, `medium`, `high`, `xhigh` in that order +- **AND** `default_reasoning_level` is `high` + +#### Scenario: Effort objects carry operator descriptions and summary support + +- **GIVEN** a source model whose `raw_metadata_json` sets + `"supported_reasoning_levels": [{"effort": "low", "description": "Low effort"}]` + and `"supports_reasoning_summaries": true` +- **WHEN** a client fetches the Codex model catalog +- **THEN** the `low` effort is advertised with description `Low effort` +- **AND** `supports_reasoning_summaries` is `true` + +#### Scenario: Malformed entries and out-of-range defaults are dropped + +- **GIVEN** a source model whose `raw_metadata_json` sets + `"supported_reasoning_levels": ["low", "low", {"description": "x"}, 7, {"effort": "high"}]` + and `"default_reasoning_level": "ultra"` +- **WHEN** a client fetches the Codex model catalog +- **THEN** the entry advertises exactly `low` and `high` +- **AND** `default_reasoning_level` is absent + +#### Scenario: Casing variants are normalized, unknown efforts are kept + +- **GIVEN** a source model whose `raw_metadata_json` sets + `"supported_reasoning_levels": [" Low ", "HIGH", "provider-specific"]` and + `"default_reasoning_level": " HIGH "` +- **WHEN** a client fetches the Codex model catalog +- **THEN** the entry advertises `low`, `high`, and `provider-specific` +- **AND** `default_reasoning_level` is `high` + +#### Scenario: An operator-declared `none` survives + +- **GIVEN** a source model whose `raw_metadata_json` sets + `"supported_reasoning_levels": ["none", "high", "max"]` and + `"default_reasoning_level": "none"` +- **WHEN** a client fetches the Codex model catalog +- **THEN** the entry advertises `none`, `high`, and `max` +- **AND** `default_reasoning_level` is `none` + +#### Scenario: Models without reasoning metadata keep the previous behavior + +- **GIVEN** a source model with no `raw_metadata_json` +- **WHEN** a client fetches the Codex model catalog +- **THEN** the entry advertises no reasoning efforts, no default effort, and no + reasoning-summary support + +### Requirement: The reasoning switch is the single opt-in across every surface + +`"supports_reasoning": true` MUST remain the only reasoning opt-in for a source +model. Declared levels or `supports_reasoning_summaries` MUST NOT imply it. + +Because catalog derivation is gated on the same flag, the surfaces cannot +disagree: with the switch off the model advertises no efforts, `/v1/models` +reports `supports_reasoning: false`, the chat-completions sanitizer strips the +client's reasoning fields, and the unsupported-effort restore has no declared +effort to act on. With it on, the operator's declared efforts reach all of them. + +#### Scenario: The switch is off + +- **GIVEN** a source model that declares `supported_reasoning_levels` and + `supports_reasoning_summaries` but not `"supports_reasoning": true` +- **WHEN** its catalog entry is built and a chat-completions request for it + carries reasoning fields +- **THEN** the entry advertises no efforts, no default, and no summary support +- **AND** `/v1/models` reports `supports_reasoning: false` +- **AND** the request's reasoning fields are stripped + +#### Scenario: The switch is on + +- **GIVEN** the same source model with `"supports_reasoning": true` added +- **WHEN** its catalog entry is built and a chat-completions request for it + carries reasoning fields +- **THEN** the entry advertises the declared efforts and summary support +- **AND** the request's reasoning fields are forwarded + +#### Scenario: The switch alone still opts in + +- **GIVEN** a source model that sets only `"supports_reasoning": true` +- **WHEN** a chat-completions request for that model carries reasoning fields +- **THEN** the fields are forwarded, and the entry advertises no specific efforts + +### Requirement: The unsupported-effort rewrite is undone for source-routed requests + +The `minimal` normalization works around a ChatGPT/Codex backend that drops the +value, hanging the stream. Model sources do not have that defect, so a request +served by one MUST NOT be downgraded by it. + +Whether a request is served by a model source is known only after source +selection, which runs after enforcement. The rewrite MUST therefore be applied +unconditionally at enforcement time, and the replaced effort MUST be reported to +the caller so it can be restored once a source has actually been selected. +Restoration MUST occur only when a source was selected and the replaced effort +is among the efforts that source declares for the model. Declared efforts are +read through the same `"supports_reasoning"` gate as the catalog, so a model +whose switch is off has none and is never restored. The reported effort +MUST be the post-enforcement value, so restoring it cannot resurrect an effort +an API key overrode, and MUST be the normalized (trimmed, lowercased) form, so +restoration cannot reintroduce a casing variant the normalizer removed. + +Restoration MUST apply only to efforts replaced by the unsupported-effort +fallback. The `ultra` -> `max` rewrite is a wire alias rather than a workaround: +it mirrors the reference client and is required on every upstream surface, so it +MUST remain applied to source-routed payloads even when the source declares +`ultra`. + +Registry membership MUST NOT be used to decide this. A populated snapshot can +omit a genuine subscription model — a partial refresh, an account unavailable +during refresh, or an operator-mapped slug outside the bootstrap set — and those +requests still reach the ChatGPT backend, where skipping the rewrite restores +the hang. Conversely a source model whose slug shadows a subscription slug is +present in the snapshot yet source-routed. + +#### Scenario: A source that declared the effort receives it unchanged + +- **GIVEN** a source model declaring `["minimal", "low", "high"]` +- **AND** a request for that model with `reasoning.effort` of `minimal` +- **WHEN** the request is routed to the source +- **THEN** the source receives `minimal` + +#### Scenario: A source that did not declare the effort keeps the safe value + +- **GIVEN** a source model declaring `["low", "high"]` +- **AND** a request for that model with `reasoning.effort` of `minimal` +- **WHEN** the request is routed to the source +- **THEN** the source receives the rewritten effort + +#### Scenario: A source declaring ultra still receives the max alias + +- **GIVEN** a source model declaring `["ultra", "max"]` +- **AND** a request for that model with `reasoning.effort` of `ultra` +- **WHEN** the request is routed to the source +- **THEN** the source receives `max` + +#### Scenario: Subscription requests keep the workaround + +- **GIVEN** a request with `reasoning.effort` of `minimal` that is not routed to + a model source, including one whose model is absent from a populated registry + snapshot +- **WHEN** the request is forwarded +- **THEN** the effort is rewritten to the model's lowest supported effort + +#### Scenario: WebSocket requests keep the workaround + +- **GIVEN** a WebSocket Responses request with `reasoning.effort` of `minimal` +- **WHEN** the request is forwarded +- **THEN** the effort is rewritten, because the WebSocket transport never + reaches a model source + +#### Scenario: An enforced effort is not resurrected by restoration + +- **GIVEN** an API key that enforces a reasoning effort +- **AND** a request for a source model that declares the client's original effort +- **WHEN** the request is routed to the source +- **THEN** the source receives the enforced effort diff --git a/openspec/changes/source-model-reasoning-metadata/tasks.md b/openspec/changes/source-model-reasoning-metadata/tasks.md new file mode 100644 index 0000000000..b54332b000 --- /dev/null +++ b/openspec/changes/source-model-reasoning-metadata/tasks.md @@ -0,0 +1,43 @@ +## 1. Catalog metadata + +- [x] 1.1 Derive source-model reasoning levels, default level, and summary + support from `raw_metadata_json`. +- [x] 1.2 Restrict the declared default to one of the advertised efforts. + +## 2. Effort delivery + +- [x] 2.1 Apply the unsupported-effort rewrite unconditionally at enforcement + time and restore it at the source-routing branch, gated on the effort + being declared for that source model. Route membership is not inferred + from the model registry. +- [x] 2.2 Report the replaced effort from the normalizer and thread it through + enforcement. Paths whose enforced Responses payload only ever reaches a + subscription (WebSocket, stream, collect, compact, and chat -- whose own + source branch forwards the untouched original chat payload) discard it, so + the workaround still applies there. +- [x] 2.3 Report only fallback rewrites, so the `ultra` -> `max` wire alias + survives source routing, and restore the normalized effort form. +- [x] 2.4 Normalize and deduplicate declared efforts, validating shape rather + than membership of a fixed vocabulary, so operator-declared `none` and + other provider-specific efforts survive. +- [x] 2.5 Gate catalog derivation, the declared-levels accessor and the + chat-path opt-in on the `supports_reasoning` switch, so the Codex + catalog, `/v1/models`, the chat sanitizer and the restore agree. + +## 3. Verification + +- [x] 3.1 Unit coverage for slug lists, object lists, malformed entries, an + out-of-range default, and the no-metadata default. +- [x] 3.2 Manual end-to-end check that `/backend-api/codex/models` advertises the + declared efforts and that forwarding behavior is unchanged. +- [x] 3.3 Unit coverage for the restore matrix (declared, undeclared, enforced), + the never-restored `ultra` alias, and the normalized restored form. +- [x] 3.4 Integration coverage that a source declaring `minimal` receives it, + via both `/v1/responses` and the codex-native `/backend-api/codex/responses` + route. Mutation-checked per call site: dropping the restore call, or the + threading at either route, fails the corresponding test. One test per route + is required -- the codex-native threading is invisible to the `/v1` test. +- [ ] 3.5 The WebSocket scenario is verified by inspection only: the WebSocket + service tree contains no model-source references, so there is no restore to + suppress. Left unchecked rather than claimed as tested. + diff --git a/openspec/changes/stream-idle-timeout-account-neutral/context.md b/openspec/changes/stream-idle-timeout-account-neutral/context.md new file mode 100644 index 0000000000..41033fdd0d --- /dev/null +++ b/openspec/changes/stream-idle-timeout-account-neutral/context.md @@ -0,0 +1,13 @@ +Idle stream silence is a transport/read timeout, not an account fault. +`_stream_once` already failovers via `_RetryableStreamError(..., exclude_account=True)`. +The missing piece was `_handle_stream_error`: `stream_idle_timeout` was not in +the account-neutral set, so classification fell through to `record_error`. + +Adding the code to `_is_account_neutral_error_code` is the same seam used by +process-network, `proxy_unavailable`, and compact input-too-large. This request +still excludes the idle account. Later independent requests may select it again +while it remains healthy. + +Example: account A emits `response.failed` / `stream_idle_timeout` as the first +SSE event. The request moves to account B and succeeds. A's request log stays +`stream_idle_timeout`. A's `error_count` stays 0. diff --git a/openspec/changes/stream-idle-timeout-account-neutral/proposal.md b/openspec/changes/stream-idle-timeout-account-neutral/proposal.md new file mode 100644 index 0000000000..e2bc2ef732 --- /dev/null +++ b/openspec/changes/stream-idle-timeout-account-neutral/proposal.md @@ -0,0 +1,27 @@ +# Why + +An HTTP SSE Responses stream whose first upstream event is `response.failed` +with `stream_idle_timeout` still failovers, but `_handle_stream_error` treats +the code as a transient account fault and calls `record_error`. Idle silence +is not evidence that the account is unhealthy. The penalty can backoff a +healthy account and send later requests into probe/drain. + +# What Changes + +- Treat `stream_idle_timeout` as an account-neutral error code. +- Keep this-request exclude/failover so the idle attempt does not retry the + same account. +- Do not write `record_error`, rate-limit, quota, or permanent-failure health + for that idle timeout. + +# Capabilities + +### Modified Capabilities + +- `responses-api-compat`: HTTP SSE first-event `stream_idle_timeout` must stay + account-neutral for health writes. + +# Impact + +Existing failover and request-log behavior stay the same. Websocket liveness +and keepalive timeouts already have their own account-neutral rules. diff --git a/openspec/changes/stream-idle-timeout-account-neutral/specs/responses-api-compat/spec.md b/openspec/changes/stream-idle-timeout-account-neutral/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..af2b6f0fd1 --- /dev/null +++ b/openspec/changes/stream-idle-timeout-account-neutral/specs/responses-api-compat/spec.md @@ -0,0 +1,15 @@ +## ADDED Requirements + +### Requirement: HTTP SSE stream idle timeouts remain account-neutral + +When an HTTP SSE Responses stream's first upstream event is `response.failed` with `code=stream_idle_timeout`, the proxy MUST exclude that account from the remainder of the same request and MAY fail over to another account. It MUST NOT write account error-health (`record_error`, rate-limit, quota, or permanent failure) for that idle timeout. Request logs MUST still record `stream_idle_timeout` on the idle attempt. + +#### Scenario: First-event stream idle timeout failovers without health penalty + +- **GIVEN** an HTTP SSE Responses stream whose first upstream event is `response.failed` with `code=stream_idle_timeout` +- **AND** another healthy account is available +- **WHEN** the proxy retries the request +- **THEN** the idle account is excluded from the remainder of this request +- **AND** the idle account receives no error-health write +- **AND** the client receives the later account's successful stream +- **AND** the idle attempt's request log still uses `error_code=stream_idle_timeout` diff --git a/openspec/changes/stream-idle-timeout-account-neutral/tasks.md b/openspec/changes/stream-idle-timeout-account-neutral/tasks.md new file mode 100644 index 0000000000..39f94db73f --- /dev/null +++ b/openspec/changes/stream-idle-timeout-account-neutral/tasks.md @@ -0,0 +1,18 @@ +## 1. Implementation + +- [x] 1.1 Classify `stream_idle_timeout` as account-neutral in + `_is_account_neutral_error_code`. +- [x] 1.2 Keep this-request exclude/failover for the idle account. + +## 2. Regression coverage + +- [x] 2.1 Assert `_handle_stream_error` does not call `record_error` for + `stream_idle_timeout`. +- [x] 2.2 Assert the existing first-event idle-timeout failover still succeeds + and the idle account's error_count stays 0. + +## 3. Validation + +- [x] 3.1 Run the new helper regression and the existing idle-timeout failover + integration test. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/openspec/changes/surface-reasoning-token-usage/design.md b/openspec/changes/surface-reasoning-token-usage/design.md new file mode 100644 index 0000000000..959782e813 --- /dev/null +++ b/openspec/changes/surface-reasoning-token-usage/design.md @@ -0,0 +1,38 @@ +## Context + +The Responses API reports exact reasoning usage at `usage.output_tokens_details.reasoning_tokens`. codex-lb already parses that terminal usage object into `request_logs.reasoning_tokens` for direct Codex subscription traffic over HTTP and WebSocket and exposes it as `reasoningTokens` from the request-log API. The request table and reports page omit the value, while reports aggregate only input, cached-input, and inclusive output totals. + +## Goals / Non-Goals + +**Goals:** + +- Make per-request reasoning usage visible without a database query. +- Aggregate reported reasoning usage over the same date and account/model/user-agent filters as the existing reports totals, with summary coverage for requests whose count is known. +- State the subset relationship in the UI contract so reasoning is never added to output a second time. + +**Non-Goals:** + +- Estimate reasoning usage from reasoning summaries or visible text. +- Change token limits, pricing, cost calculation, or API-key quota enforcement. +- Backfill responses whose upstream terminal event did not provide usage. +- Extend reasoning-detail capture for custom OpenAI-compatible model sources; their forwarding parser is a separate protocol change. + +## Decisions + +- Use the upstream-provided count already stored in `request_logs.reasoning_tokens`; no tokenizer or heuristic is introduced. +- Keep `outputTokens` inclusive of reasoning tokens. `reasoningTokens` is an additive breakdown field, not another component of total tokens. +- Add nullable `reasoningTokens` to each reports daily row, plus `totalReasoningTokens` and `reasoningUsageKnownRequests` to the reports summary. An all-unknown day remains null, while a known zero remains zero. The previous-window token comparison remains input plus inclusive output, using the existing reasoning-only fallback when an older row lacks an output total. +- Render the reasoning subset as secondary request-row metadata and as an explicit request-detail field. Reports render it in the token summary, daily table, and CSV. + +## Risks / Trade-offs + +- [Risk] Operators add reasoning to output and overstate usage. The spec and UI copy identify reasoning as included in output, and total-token calculations continue to use input plus inclusive output without adding reasoning again. +- [Risk] Legacy, cancelled, or interrupted rows may have no terminal reasoning count. Request history leaves the value unknown, an all-unknown daily aggregate remains null, reports label the aggregate as reported reasoning, and summary coverage states how many requests supplied a count. Missing values are excluded rather than inferred as known zero. + +## Migration Plan + +Ship the additive API and dashboard fields together. Rollback removes the new fields and rendering; the existing `request_logs.reasoning_tokens` data remains intact. + +## Open Questions + +None. diff --git a/openspec/changes/surface-reasoning-token-usage/proposal.md b/openspec/changes/surface-reasoning-token-usage/proposal.md new file mode 100644 index 0000000000..0099bad190 --- /dev/null +++ b/openspec/changes/surface-reasoning-token-usage/proposal.md @@ -0,0 +1,24 @@ +## Why + +codex-lb already persists the upstream reasoning-token count reported for completed direct Codex subscription responses, but the dashboard only renders total, cached-input, and output token totals. Operators cannot see the reasoning subset in request history or aggregate it over a report window without querying the database directly. + +## What Changes + +- Show the persisted reasoning-token count in request-log rows and request details. +- Add reported reasoning-token totals and summary coverage to the reports response. +- Render reported reasoning totals in the reports summary, daily breakdown, and CSV export. +- Keep reasoning tokens as a subset of output tokens so existing total-token and cost calculations do not double-count them. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `frontend-architecture`: Request history and reports expose the reasoning-token subset already recorded by the proxy. + +## Impact + +Reports API aggregation, dashboard schemas and components, localized labels, user-facing usage-reporting documentation, and focused backend/frontend tests. No database migration, configuration, routing, pricing, or proxy-protocol change. diff --git a/openspec/changes/surface-reasoning-token-usage/specs/frontend-architecture/spec.md b/openspec/changes/surface-reasoning-token-usage/specs/frontend-architecture/spec.md new file mode 100644 index 0000000000..82d221ca47 --- /dev/null +++ b/openspec/changes/surface-reasoning-token-usage/specs/frontend-architecture/spec.md @@ -0,0 +1,64 @@ +## ADDED Requirements + +### Requirement: Request logs surface reasoning-token usage + +The dashboard request-log API and UI MUST preserve and render the upstream-provided reasoning-token count separately from the inclusive output-token count. The UI MUST treat reasoning tokens as a subset of output tokens and MUST NOT add them to total-token or cost calculations. + +#### Scenario: Request row shows an available reasoning count + +- **GIVEN** a request-log row has `outputTokens=200` and `reasoningTokens=80` +- **WHEN** the dashboard renders the recent-requests table +- **THEN** the token cell shows 80 reasoning tokens as secondary metadata +- **AND** the row's total-token value remains input tokens plus 200 output tokens + +#### Scenario: Request detail identifies the reasoning subset + +- **GIVEN** a request-log row has a persisted reasoning-token count +- **WHEN** the operator opens `View Details` +- **THEN** the dialog renders the exact reasoning-token count +- **AND** its label identifies the count as included in output tokens + +#### Scenario: Missing reasoning usage is not estimated + +- **GIVEN** a request-log row has `reasoningTokens=null` +- **WHEN** the dashboard renders the row and its details +- **THEN** the dashboard does not derive a reasoning count from output text, reasoning summaries, or total output tokens + +### Requirement: Reports expose reasoning-token totals + +`GET /api/reports` MUST expose the sum of reported reasoning counts as `totalReasoningTokens` in its summary and nullable `reasoningTokens` in each daily row, using the same date, account, model, and user-agent filters as the existing token totals. The summary MUST expose `reasoningUsageKnownRequests`, counting rows whose upstream reasoning count is known, including known zeroes. The reports UI MUST label the aggregate as reported reasoning, show its known-request coverage, render it in the daily breakdown, and export it in the daily CSV. A daily row with requests but no reported reasoning counts MUST preserve `reasoningTokens=null`; known zero MUST remain zero. Existing total-token comparisons MUST remain input tokens plus inclusive output tokens, with a stored reasoning count serving as the output fallback when an older row has no output total. + +#### Scenario: Reports aggregate reasoning usage + +- **GIVEN** eligible request logs in a report window contain reasoning-token counts of 30 and 70 and one request with an unknown count +- **WHEN** an operator requests that report window +- **THEN** `summary.totalReasoningTokens` is 100 +- **AND** `summary.reasoningUsageKnownRequests` is 2 +- **AND** each daily row's `reasoningTokens` is the sum for that local-calendar day + +#### Scenario: Reports identify reasoning as an output subset + +- **GIVEN** a report summary has 1,000 input tokens, 400 output tokens, and 250 reasoning tokens +- **WHEN** the dashboard renders the token summary +- **THEN** the total remains 1,400 tokens +- **AND** the summary identifies 250 reasoning tokens as included in the 400 output tokens + +#### Scenario: Daily CSV exports reasoning tokens + +- **WHEN** an operator exports the reports daily breakdown +- **THEN** the CSV contains a Reported Reasoning Tokens column +- **AND** every row contains that day's reasoning-token aggregate + +#### Scenario: A daily aggregate has no reported reasoning usage + +- **GIVEN** a report day contains requests whose reasoning-token counts are all unknown +- **WHEN** the reports API and dashboard render that day +- **THEN** the daily `reasoningTokens` value remains null +- **AND** the table and CSV do not present it as a known zero + +#### Scenario: A legacy row has reasoning usage but no output total + +- **GIVEN** an eligible request log has `outputTokens=null` and `reasoningTokens=40` +- **WHEN** the report aggregates inclusive output tokens +- **THEN** the row contributes 40 output tokens and 40 reported reasoning tokens +- **AND** reasoning is not added to that output total a second time diff --git a/openspec/changes/surface-reasoning-token-usage/tasks.md b/openspec/changes/surface-reasoning-token-usage/tasks.md new file mode 100644 index 0000000000..13b6dbf16d --- /dev/null +++ b/openspec/changes/surface-reasoning-token-usage/tasks.md @@ -0,0 +1,21 @@ +## 1. Reports contract + +- [x] 1.1 Aggregate reasoning tokens in reports summary and daily rows +- [x] 1.2 Expose `totalReasoningTokens`, `reasoningUsageKnownRequests`, and daily `reasoningTokens` from the reports API +- [x] 1.3 Add backend regression coverage for filtered and unfiltered report windows +- [x] 1.4 Preserve reasoning-only output fallback and nullable all-unknown daily reasoning aggregates + +## 2. Dashboard presentation + +- [x] 2.1 Render reasoning usage in request rows and request details +- [x] 2.2 Render reported reasoning totals and summary coverage in reports +- [x] 2.3 Include reasoning tokens in the daily CSV export +- [x] 2.4 Add English, Korean, and Simplified Chinese labels +- [x] 2.5 Render and export all-unknown daily reasoning usage distinctly from known zero +- [x] 2.6 Document token-bucket semantics, dashboard surfaces, and missing-usage behavior in the docs site + +## 3. Validation + +- [x] 3.1 Run focused backend and frontend tests +- [x] 3.2 Run backend lint/type checks and frontend typecheck/build +- [x] 3.3 Validate OpenSpec diff --git a/openspec/changes/surface-reports-cancellation-totals/design.md b/openspec/changes/surface-reports-cancellation-totals/design.md new file mode 100644 index 0000000000..fc3279b81c --- /dev/null +++ b/openspec/changes/surface-reports-cancellation-totals/design.md @@ -0,0 +1,54 @@ +# Design: Surface reports cancellation totals + +## Context + +The raw Reports backend models define cancellation data as `ReportSummary.total_cancelled` and `DailyReportRow.cancelled_count`. Dashboard API serialization exposes those fields as `summary.totalCancelled` and `daily[].cancelledCount`, which are also the names consumed by the frontend. The frontend's strict response schemas omit both camelCase properties, so parsing strips the values before the report model reaches rendering and export. The date-range completion path also creates synthetic daily rows without a cancellation field. As a result, cancellation data is absent from the summary, daily table, and downloaded CSV despite being available at the system boundary. + +This change is limited to the Reports frontend. The existing `usage-error-metrics` specification remains the owner of request terminal classification and cancellation accounting. + +## Goals and Non-goals + +### Goals + +- Preserve the backend cancellation fields through frontend parsing. +- Treat a synthesized no-activity day as having zero cancellations. +- Present cancellations beside requests and errors in the visible summary, table, and CSV. +- Keep labels localized, including English, Korean, and Simplified Chinese. +- Prove existing request and error values do not regress. + +### Non-goals + +- Changing backend aggregation, response casing, storage, or terminal classification. +- Recomputing cancellations in the browser. +- Redesigning the Reports page or introducing a new visual primitive. + +## Decisions + +### Preserve cancellation values in the typed report model + +The response schemas will explicitly parse `summary.totalCancelled` and each `daily[].cancelledCount`. The UI and export paths will consume these parsed fields rather than deriving cancellation counts from requests and errors. Derivation would be incorrect because requests may include successful, cancelled, and genuinely failed terminals, and future terminal classes may exist. + +### Zero-fill only synthesized empty days + +The date-range completion path will assign `cancelledCount: 0` to synthetic rows, matching the existing zero-fill semantics for other count metrics. A cancellation value returned by the API will be preserved, including an explicit zero. + +### Extend existing Reports presentation patterns + +The cancellation summary item and daily-table column will compose the page's existing summary and table primitives, spacing, typography, responsive behavior, and semantic design tokens. The CSV will add a localized cancellation header in the same column order used by the visible daily table. No one-off visual values or separate desktop/mobile markup will be introduced. + +### Use the existing localization boundary + +Visible labels and the CSV header will use the Reports translation namespace. English, Korean, and Simplified Chinese resources will receive equivalent cancellation labels; CSV generation will use the active locale just as existing headers do. + +## Failure Modes and Mitigations + +- **Schema strips valid backend data:** parser tests assert both cancellation fields survive parsing. +- **Synthetic rows expose `undefined` or a blank CSV cell:** zero-fill tests assert a numeric `0` in the model, table, and export. +- **Cancellation is accidentally folded into errors:** regression fixtures retain distinct requests, cancellations, and errors and assert all three independently. +- **A label is readable in one locale only:** locale coverage and real zh-CN browser evidence verify the visible label and CSV header. +- **The added column overflows or hides key values:** desktop and 390px mobile browser evidence verifies the existing responsive table behavior with the added column. +- **QA leaves local state behind:** the browser QA task records teardown of servers, ports, browser sessions, downloads, fixtures, and temporary data. + +## Example + +Given a parsed frontend report response with `totalRequests: 4`, `totalCancelled: 2`, and `totalErrors: 1`, plus a daily row with `requests: 4`, `cancelledCount: 2`, and `errorCount: 1`, parsing preserves all six values. The summary visibly shows Requests 4, Cancelled 2, and Errors 1; the daily table shows the same breakdown; and the localized CSV contains a cancellation column with value `2`. A missing date synthesized into the selected range displays and exports cancellation value `0`. diff --git a/openspec/changes/surface-reports-cancellation-totals/proposal.md b/openspec/changes/surface-reports-cancellation-totals/proposal.md new file mode 100644 index 0000000000..43d478aba6 --- /dev/null +++ b/openspec/changes/surface-reports-cancellation-totals/proposal.md @@ -0,0 +1,25 @@ +# Change: Surface reports cancellation totals + +## Why + +The reports API already returns cancellation totals, but the frontend parser drops them and the reports summary, daily table, and CSV export omit them. Operators therefore cannot distinguish cancelled requests from genuine errors on the Reports surface even though the owning metric contract requires cancellation counts alongside errors. + +## What Changes + +- Preserve `summary.totalCancelled` and `daily[].cancelledCount` when parsing reports responses. +- Zero-fill missing daily cancellation counts as `0` when constructing a complete date range. +- Show localized cancellation totals in the reports summary and daily table. +- Include a localized cancellation column and values in reports CSV exports. +- Add deterministic parser, rendering, export, localization, responsive-layout, and regression evidence while preserving existing request and error totals. + +## Capabilities + +### Modified Capabilities + +- `usage-error-metrics`: require the Reports frontend to preserve and visibly surface the cancellation fields already supplied by the reports API. + +## Impact + +- Affected area: Reports frontend response parsing, date-range zero-fill, summary cards, daily detail table, CSV export, and report translations. +- Compatibility: additive presentation only; existing requests and errors values and CSV semantics remain unchanged apart from the new cancellation column. +- No backend, database, or API contract changes are required. diff --git a/openspec/changes/surface-reports-cancellation-totals/specs/usage-error-metrics/spec.md b/openspec/changes/surface-reports-cancellation-totals/specs/usage-error-metrics/spec.md new file mode 100644 index 0000000000..a649c9e7e7 --- /dev/null +++ b/openspec/changes/surface-reports-cancellation-totals/specs/usage-error-metrics/spec.md @@ -0,0 +1,69 @@ +## MODIFIED Requirements + +### Requirement: Cancelled counts surface alongside error counts + +Metric surfaces that expose an error count MUST also expose the window's +cancelled count as an additive field: the dashboard overview metrics +(`cancelledCount`), the usage summary metrics (`cancelled7d`), the raw Reports +backend daily rows (`cancelled_count`) and summary (`total_cancelled`), and the +fleet pressure metrics (`cancelledCount`). The dashboard overview cancelled total +MUST be sourced from the demand quarter rollup (status grain) for the folded +segment plus the raw tail, so it stays accurate across history already folded +without the hourly `cancelled_count` measure. The dashboard frontend MUST +preserve `cancelledCount` when parsing the overview response. + +The Reports dashboard API MUST serialize the raw backend `total_cancelled` and +`cancelled_count` fields as `summary.totalCancelled` and +`daily[].cancelledCount`, respectively. The Reports frontend MUST preserve +those parsed camelCase values from the reports response. A daily row +synthesized to fill a missing date in the selected range MUST set +`cancelledCount` to `0`. The Reports summary and daily table MUST visibly show +the cancellation values with localized labels, and the Reports CSV export MUST +include a localized cancellation header and each daily row's cancellation +value. Adding cancellation presentation MUST NOT change the parsed, visible, or +exported request and error values. + +#### Scenario: Dashboard overview reports the status breakdown + +- **GIVEN** a window containing 1 successful, 2 cancelled, and 1 error rows + that are partially folded into the rollups +- **WHEN** the dashboard overview metrics are computed +- **THEN** the metrics expose `requests=4`, `errorCount=1`, and + `cancelledCount=2` + +#### Scenario: Dashboard overview preserves the status breakdown + +- **GIVEN** the dashboard overview API returns `requests=4`, `errorCount=1`, + and `cancelledCount=2` +- **WHEN** the frontend parses the overview response +- **THEN** the parsed metrics expose all three values unchanged + +#### Scenario: Reports preserve and display cancellation values + +- **GIVEN** a reports response whose summary has `totalRequests=4`, + `totalCancelled=2`, and `totalErrors=1` and whose frontend daily row has + `requests=4`, `cancelledCount=2`, and `errorCount=1` +- **WHEN** the Reports frontend parses and displays the response +- **THEN** the parsed summary has `totalCancelled=2` and the parsed daily row + has `cancelledCount=2` +- **AND** the localized summary visibly shows requests `4`, cancellations `2`, + and errors `1` +- **AND** the localized daily table visibly shows requests `4`, cancellations + `2`, and errors `1` + +#### Scenario: Reports zero-fill cancellations for a missing date + +- **GIVEN** a selected report range containing a date absent from the reports + response +- **WHEN** the Reports frontend synthesizes the daily row for that date +- **THEN** the synthesized row has `cancelledCount=0` +- **AND** the daily table visibly shows cancellation value `0` for that row + +#### Scenario: Reports CSV exports localized cancellation values + +- **GIVEN** parsed report rows with cancellation values `2` and `0` +- **WHEN** a user exports the report while a supported locale is active +- **THEN** the CSV contains the locale's cancellation header and the values `2` + and `0` in that column +- **AND** the exported request and error headers and values remain present and + unchanged diff --git a/openspec/changes/surface-reports-cancellation-totals/tasks.md b/openspec/changes/surface-reports-cancellation-totals/tasks.md new file mode 100644 index 0000000000..dc9a70985b --- /dev/null +++ b/openspec/changes/surface-reports-cancellation-totals/tasks.md @@ -0,0 +1,25 @@ +# Tasks + +## 1. Deterministic regression coverage + +- [x] 1.1 Add parser tests proving `summary.totalCancelled` and `daily[].cancelledCount` survive reports response parsing. +- [x] 1.2 Add date-range completion coverage proving a synthesized daily row has `cancelledCount: 0`. +- [x] 1.3 Add summary and daily-table tests proving Requests 4, Cancelled 2, and Errors 1 remain distinct and visible. +- [x] 1.4 Add CSV coverage proving the localized cancellation header, cancellation value `2`, zero-filled value `0`, and existing request/error values. +- [x] 1.5 Run the focused parser/table/export tests before implementation and record the expected cancellation-specific failures. + +## 2. Reports cancellation presentation + +- [x] 2.1 Extend the typed Reports schemas and zero-fill model to preserve cancellation values. +- [x] 2.2 Add a cancellation item to the existing Reports summary composition. +- [x] 2.3 Add a cancellation column to the existing daily detail table and CSV export. +- [x] 2.4 Add equivalent Reports cancellation labels for every supported locale, including English, Korean, and Simplified Chinese. +- [x] 2.5 Re-run the focused tests and frontend diagnostics, then run the affected frontend test, type-check, lint, and build gates. + +## 3. User-visible verification and cleanup + +- [x] 3.1 Exercise a real Reports fixture with Requests 4, Cancelled 2, and Errors 1 in Chromium and verify the visible summary, daily table, and downloaded CSV. +- [x] 3.2 Capture desktop, 390px mobile, and zh-CN evidence showing cancellation alongside unchanged request and error values. +- [x] 3.3 Verify a zero-filled date visibly shows and exports cancellation value `0`. +- [x] 3.4 Tear down browser sessions, frontend/backend processes, ports, fixture databases, downloads, and temporary QA artifacts, and record the cleanup receipt. +- [x] 3.5 Run `openspec validate surface-reports-cancellation-totals --strict` and retain the exact successful output. diff --git a/openspec/changes/surface-upstream-route-metadata/design.md b/openspec/changes/surface-upstream-route-metadata/design.md new file mode 100644 index 0000000000..176af320a5 --- /dev/null +++ b/openspec/changes/surface-upstream-route-metadata/design.md @@ -0,0 +1,54 @@ +## Context + +The proxy writer and database model already preserve five credential-safe +route diagnostics. The request-log read model stops at a narrower Pydantic +schema, and the frontend has a second Zod boundary before the existing details +dialog. Operators need the same persisted values across both boundaries. + +## Goals / Non-Goals + +**Goals:** + +- Carry the five existing route fields through the API and frontend unchanged. +- Show populated values in the existing request details dialog. +- Keep the presentation compact and omit absent metadata. + +**Non-Goals:** + +- Change route selection, persistence, or database schema. +- Expose proxy URLs, usernames, passwords, or headers. +- Add table columns, filters, settings, or navigation. + +## Decisions + +- Extend the existing `RequestLogEntry` and `RequestLogSchema` instead of + adding a second endpoint or nested routing object. This matches the flat + persisted model and existing request-log response. +- Render metadata only in the details dialog. Routing diagnostics are useful + during investigation but too sparse for permanent table columns. +- Reuse the details grid's existing label/value rows and localized strings. + Boolean fallback state is rendered as localized yes/no text. + +Alternative considered: expose only the fail-closed reason. That would leave +successful fallback and endpoint selection unobservable, so all five +credential-safe fields move together. + +## Risks / Trade-offs + +- [Risk] Internal identifiers add visual noise → Render only non-null values in + the opt-in details dialog. +- [Risk] Future fields accidentally expose secrets → Explicitly whitelist the + five existing credential-safe columns rather than serializing route objects. +- [Risk] Mixed-version deployments omit fields → Keep every added field + nullable and optional in the frontend parser. + +## Migration Plan + +The API change is additive and reads existing columns, so no migration or +backfill is needed. Older frontends ignore the new keys; newer frontends accept +responses from older backends. Rollback removes the response/UI fields without +changing stored data. + +## Open Questions + +None. diff --git a/openspec/changes/surface-upstream-route-metadata/proposal.md b/openspec/changes/surface-upstream-route-metadata/proposal.md new file mode 100644 index 0000000000..9835eb039e --- /dev/null +++ b/openspec/changes/surface-upstream-route-metadata/proposal.md @@ -0,0 +1,30 @@ +## Why + +Request logs persist the upstream route mode, pool, endpoint, fallback use, +and fail-closed reason, but the request-log API read model drops every field. +The dashboard therefore cannot explain which configured route served or +blocked a request even though the diagnostic data already exists. + +## What Changes + +- Preserve credential-safe upstream routing metadata through the request-log + API response and frontend parser. +- Present the metadata in the existing request details dialog. +- Add API and frontend contract regressions for the five fields. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `upstream-proxy-routing`: Persisted route metadata is exposed through the + operator request-log surface without proxy credentials. + +## Impact + +Request-log response schemas/mapping, dashboard parsing and request details, +focused API/frontend tests, and localized labels. Database persistence and +proxy routing behavior do not change. diff --git a/openspec/changes/surface-upstream-route-metadata/specs/upstream-proxy-routing/spec.md b/openspec/changes/surface-upstream-route-metadata/specs/upstream-proxy-routing/spec.md new file mode 100644 index 0000000000..3ddb7b7bc8 --- /dev/null +++ b/openspec/changes/surface-upstream-route-metadata/specs/upstream-proxy-routing/spec.md @@ -0,0 +1,25 @@ +## MODIFIED Requirements + +### Requirement: Route metadata must be persisted for migrated upstream calls + +Request logs for migrated upstream calls MUST record route mode, proxy pool +id, proxy endpoint id, same-pool fallback use, and fail-closed reason where +applicable. The request-log API and dashboard request details MUST expose +those credential-safe values to operators and MUST NOT expose proxy +credentials. + +#### Scenario: Fail-closed route is diagnosable from request details + +- **GIVEN** route resolution fails closed before network open +- **AND** the request log records the route mode and fail-closed reason +- **WHEN** an operator opens that request in the dashboard +- **THEN** the request details show the recorded route mode and fail-closed + reason +- **AND** no proxy credentials are included + +#### Scenario: Successful routed request exposes its selected route + +- **GIVEN** a request log records a proxy pool id, proxy endpoint id, and + same-pool fallback use +- **WHEN** the request-log API returns that row +- **THEN** all three values are present unchanged diff --git a/openspec/changes/surface-upstream-route-metadata/tasks.md b/openspec/changes/surface-upstream-route-metadata/tasks.md new file mode 100644 index 0000000000..bd7a8f9994 --- /dev/null +++ b/openspec/changes/surface-upstream-route-metadata/tasks.md @@ -0,0 +1,18 @@ +## 1. API contract + +- [x] 1.1 Add a failing request-log API regression for all five route fields +- [x] 1.2 Add the route fields to `RequestLogEntry` and its mapper + +## 2. Dashboard contract + +- [x] 2.1 Add a failing frontend schema regression for all five route fields +- [x] 2.2 Add the route fields to the request-log Zod schema +- [x] 2.3 Show credential-safe route metadata in request details with + localized labels + +## 3. Validation + +- [x] 3.1 Run focused request-log API and dashboard schema/component tests +- [x] 3.2 Run Python diagnostics, frontend typecheck, and frontend build +- [x] 3.3 Validate OpenSpec +- [x] 3.4 Browser-verify request details with deterministic route metadata diff --git a/openspec/changes/sweep-idle-bridge-sessions-off-request-path/proposal.md b/openspec/changes/sweep-idle-bridge-sessions-off-request-path/proposal.md new file mode 100644 index 0000000000..ec7e732aac --- /dev/null +++ b/openspec/changes/sweep-idle-bridge-sessions-off-request-path/proposal.md @@ -0,0 +1,21 @@ +## Why + +The HTTP-bridge idle sweep (`_prune_http_bridge_sessions_locked`) is reached from exactly one place: `_get_or_create_http_bridge_session`. It is therefore request-driven, so a replica that stops receiving bridge requests never evicts its idle sessions and holds their upstream WebSockets, registry entries, and durable claims until the process restarts. + +This is the residual leg of issue #1354. Fix (a) (#1476) already made account stream leases turn-scoped, so an idle session releases its cap slot as soon as its last turn detaches — the cap-exhaustion symptom is addressed there. What remains is resource hygiene on a quiet replica: in a multi-replica deployment, traffic moving off one replica leaves its warm sessions pinned indefinitely. + +## What Changes + +- Expose the existing sweep as `prune_idle_http_bridge_sessions()`: take the bridge lock, run the same `_prune_http_bridge_sessions_locked` selection the request path uses, and schedule the pruned sessions' closes through the existing bounded close scheduler. +- Drive it from the per-replica ring heartbeat, alongside the durable-ownership reconcile that already runs there, so the sweep happens on every replica regardless of traffic, leadership, or durable-row cleanup. +- No new selection logic and no new timing: eligibility, the idle TTL that protects a session freshly handed to a request, and the close path are unchanged. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `sticky-session-operations`: idle bridge-session eviction no longer depends on request traffic reaching the replica. diff --git a/openspec/changes/sweep-idle-bridge-sessions-off-request-path/specs/sticky-session-operations/spec.md b/openspec/changes/sweep-idle-bridge-sessions-off-request-path/specs/sticky-session-operations/spec.md new file mode 100644 index 0000000000..99eec97f93 --- /dev/null +++ b/openspec/changes/sweep-idle-bridge-sessions-off-request-path/specs/sticky-session-operations/spec.md @@ -0,0 +1,28 @@ +## ADDED Requirements + +### Requirement: Idle bridge sessions are swept without request traffic + +The system MUST evict idle HTTP-bridge sessions on every replica independently of whether that replica is receiving bridge requests. The sweep MUST reuse the same eligibility the request path applies — a session with pending or queued work, an admission waiter, a handoff in progress, or an unanchored reservation, and a session still inside its idle TTL, MUST NOT be evicted — and MUST close evicted sessions through the existing bounded close path so a slow upstream-reader cancellation cannot block the caller. A sweep failure MUST NOT interrupt the loop that drives it, and MUST NOT prevent the other per-replica bridge upkeep that shares that loop from running. + +#### Scenario: A replica with no bridge traffic still evicts idle sessions + +- **GIVEN** a replica holds an idle bridge session past its idle TTL and receives no further bridge requests +- **WHEN** the sweep runs +- **THEN** the session is detached from the registry and closed, releasing its upstream WebSocket + +#### Scenario: Sweep eligibility matches the request path + +- **GIVEN** a session with pending work whose idle TTL has elapsed, and a session used moments ago +- **WHEN** the sweep runs +- **THEN** neither session is evicted + +#### Scenario: One failing upkeep pass does not skip the other + +- **GIVEN** the durable-ownership reconcile raises on a heartbeat tick +- **WHEN** that tick runs +- **THEN** the idle sweep still runs and the heartbeat loop continues + +#### Scenario: Sweeping an empty registry does nothing + +- **WHEN** the sweep runs with no registered bridge sessions +- **THEN** no session is closed and no cleanup work is scheduled diff --git a/openspec/changes/sweep-idle-bridge-sessions-off-request-path/tasks.md b/openspec/changes/sweep-idle-bridge-sessions-off-request-path/tasks.md new file mode 100644 index 0000000000..1f6968b9ad --- /dev/null +++ b/openspec/changes/sweep-idle-bridge-sessions-off-request-path/tasks.md @@ -0,0 +1,18 @@ +## 1. Sweep entry point + +- [x] 1.1 Add `prune_idle_http_bridge_sessions()` to the bridge session-registry mixin (mixin.py is at its architecture line ratchet): take the bridge lock, reuse `_prune_http_bridge_sessions_locked`, and schedule closes via `_schedule_http_bridge_session_closes` with reason `idle_sweep` + +## 2. Heartbeat wiring + +- [x] 2.1 Extract the heartbeat's bridge upkeep into `run_http_bridge_heartbeat_maintenance()` in `app/main.py` and call the sweep there beside the durable-ownership reconcile, isolating each pass so one failing cannot skip the other or stop the heartbeat + +## 3. Tests + +- [x] 3.1 Idle session is evicted with no request traffic; a freshly-used session is spared +- [x] 3.2 A session with pending work is spared even past its idle TTL +- [x] 3.3 Empty registry is a no-op and schedules no cleanup task +- [x] 3.4 Heartbeat maintenance runs both passes, isolates a failing one, and tolerates a missing service — so removing the wiring fails a test rather than silently restoring the leak + +## 4. Spec + +- [x] 4.1 Record that idle eviction does not depend on request traffic reaching the replica diff --git a/openspec/changes/validate-stream-lifecycle-events-only/design.md b/openspec/changes/validate-stream-lifecycle-events-only/design.md new file mode 100644 index 0000000000..05c8e5b006 --- /dev/null +++ b/openspec/changes/validate-stream-lifecycle-events-only/design.md @@ -0,0 +1,91 @@ +# Design — validate-stream-lifecycle-events-only + +## Context + +py-spy GIL profiles attribute ~3% of proxy CPU to `validate_python` on the +stream hot path, plus a second full validation per websocket frame inside the +parallel-tool-call rewrite and redundant JSON extract+parse in the core client +and websocket relay. Every consumer of the validated `OpenAIEvent` model reads +it only on lifecycle frames; delta frames need only the `type` string. + +## Goals / Non-Goals + +**Goals:** pydantic validation only on lifecycle frames; one `json.loads` per +frame per owning layer; byte-identical SSE output; unchanged settlement, +error-classification, and terminal-detection semantics. + +**Non-Goals:** verbatim relay of unmodified SSE delta frames (the +`format_sse_event` canonical re-encode in the streaming mixin stays — that is +the separate `relay-unmodified-sse-frames-verbatim` follow-up); touching the +chat/completions bridge (genuine cross-dialect translation keeps full +parsing); the `/v1` public normalizer (independent per-chunk consumer); the +cold rewrite/prewarm paths (`limit_warmup`, `quota_planner`, +`http_bridge/helpers`, `websocket/helpers` rewrite builders), which run once +per stream or per rewrite. + +## Decisions + +- **Lifecycle set = {response.created, response.completed, + response.incomplete, response.failed, error}.** Terminal frames carry the + usage and error fields settlement needs; `response.created` is included + because the websocket path assigns the upstream response id from the + validated model there. +- **Classify-then-validate.** `classify_event_type(payload)` mirrors the dict + branch of `_event_type_from_payload` exactly (string `type` wins; a typeless + dict `error` classifies as `"error"`). Ordering classification before + validation is equivalence-preserving: when validation succeeds, + `event.type == payload["type"]`; when it fails (e.g. typeless error + payloads, or a lifecycle `type` with a non-dict `response` — `OpenAIEvent` + has no before-validator on `response`), today's code already fell back to + the same dict branch with `event=None`. +- **`event=None` for non-lifecycle frames compiles against existing + structure.** Every downstream read of `event.response`/`event.error` in the + streaming mixin, websocket finalization, and bridge settlement is gated on a + terminal `event_type`, so non-lifecycle `None` never reaches them. +- **Core-client deletion is a pure no-op.** At the two SSE loops the + `elif normalized_event_type` branch performed the identical terminal check + from the same payload the deleted `parse_sse_event` re-parsed; whenever the + model validated, its `type` equalled `normalized_event_type`. The websocket + receive loops now use the payload `type` string directly, matching the + dict semantics the SSE loops already had (a malformed lifecycle frame that + failed whole-event validation now counts for terminal detection there, as + it already did on the SSE loops). +- **Rewrite helpers no longer validate on the unchanged path.** The + `event is None` fallback in `rewrite_parallel_tool_call_text/_sse_line` was + the second per-frame validation on the websocket path; callers now own + lifecycle-gated validation and the helpers classify from the dict. The + changed path (actual dedupe rewrite of `response.output_item.done`) still + validates the rewritten payload as before. +- **Websocket identity skip.** `_rewrite_websocket_downstream_response_id` + returns the same object when no replay rewrite applies; only then is the + per-frame `json.dumps` skipped and the upstream text relayed unchanged. + Frames without a matched request state were already relayed verbatim, so + downstream clients already accept upstream-encoded frames on this surface. + +## Risks / Trade-offs + +- [Whitespace-padded response ids] Non-lifecycle frames that carry a + `response` object (`response.in_progress`) now resolve their response id via + the dict fallback, which strips whitespace, where the model path did not. + Upstream ids are never whitespace-padded; `response.created` (the id + assignment point) keeps the model path. +- [Usage on delta frames] If upstream ever emitted `usage` on non-lifecycle + frames it would no longer be validated — accepted limitation; today usage + appears only on `response.completed`/`response.incomplete`, and no consumer + reads usage outside terminal branches. +- [Websocket byte relaxation] Identity frames relay upstream JSON text + (raw UTF-8, upstream spacing) instead of the canonical + `ensure_ascii` re-encode. JSON-semantically identical, and consistent with + the existing unmatched-frame behavior on the same socket. +- [First-frame response-id capture] A stream whose first frame is a + non-lifecycle `response`-carrying frame (never observed; upstream always + opens with `response.created`) no longer captures `settlement.response_id` + from that frame; it is still captured at the terminal frame. + +## Migration Plan + +Code-only; rollback = revert. + +## Open Questions + +None. diff --git a/openspec/changes/validate-stream-lifecycle-events-only/proposal.md b/openspec/changes/validate-stream-lifecycle-events-only/proposal.md new file mode 100644 index 0000000000..3fe9be28dd --- /dev/null +++ b/openspec/changes/validate-stream-lifecycle-events-only/proposal.md @@ -0,0 +1,76 @@ +# Validate Stream Lifecycle Events Only + +## Why + +Every streamed SSE/websocket event still pays a full pydantic `OpenAIEvent` +validation per layer, even though every consumer of the validated model reads +it only on stream lifecycle frames: usage/token settlement +(`response.completed`/`response.incomplete`), error normalization and +retry/health classification (`response.failed`/`error`), and websocket +response-id assignment (`response.created`). Delta frames — the dominant +traffic by two to three orders of magnitude — only ever need their `type` +string, which the already-parsed payload dict provides. On top of that, the +core client validated each chunk a second time solely to duplicate a terminal +check it had already computed from the dict, the websocket relay extracted and +re-parsed each frame's JSON twice, built a canonical SSE re-encode per frame +just to populate a discarded argument, re-validated each frame inside the +parallel-tool-call rewrite, and re-encoded `json.dumps` on every matched frame +even when the response-id rewrite changed nothing. This is the follow-up the +`2026-07-13-optimize-sse-single-parse` design doc deferred ("threading a +parsed-event struct across layer boundaries"). + +## What Changes + +- `app/core/openai/parsing.py` gains the shared `_LIFECYCLE_EVENT_TYPES` + frozenset (`response.created`, `response.completed`, `response.incomplete`, + `response.failed`, `error`) and `classify_event_type(payload)`, the dict-only + classifier that `_event_type_from_payload` and + `tool_call_dedupe.event_type_from_payload` now delegate to. +- Streaming mixin, websocket relay, and HTTP-bridge upstream reader classify + each frame from the parsed dict first and run `parse_sse_event_payload` + only for lifecycle frames; all other frames flow with `event=None`, which + every downstream branch already guards for. +- Core client: the redundant per-chunk `parse_sse_event` terminal checks are + deleted (the `normalized_event_type` dict branch is the same check from the + same payload); the websocket receive loops detect terminal frames from + `parse_sse_data_json` + the payload `type` string. +- Websocket relay: single `json.loads` per frame (no synthetic-block re-parse), + the caller's `event` is passed into `rewrite_parallel_tool_call_text` and the + rewrite helpers no longer re-validate on the unchanged path, the discarded + `format_sse_event` argument is no longer built, and the downstream + response-id re-encode is skipped when the rewrite returned the payload + unchanged (identity), relaying the upstream frame bytes as-is. +- No output-byte change on the SSE paths: canonical `format_sse_event` + serialization, usage settlement, error rewriting, and terminal detection are + unchanged. Existing streaming/websocket suites pass unmodified. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `responses-api-compat`: the single-parse streaming requirement is extended — + schema validation of parsed stream payloads MUST run only for lifecycle + frames, with all other frames classified from the parsed payload dict and + all downstream semantics (framing, settlement, error normalization) + unchanged; identity websocket relay frames are forwarded without a canonical + re-encode. + +## Impact + +- **Code**: `app/core/openai/parsing.py`, `app/core/clients/proxy.py`, + `app/modules/proxy/tool_call_dedupe.py`, + `app/modules/proxy/_service/support.py`, + `app/modules/proxy/_service/streaming/mixin.py`, + `app/modules/proxy/_service/websocket/mixin.py`, + `app/modules/proxy/_service/http_bridge/upstream_events.py`. +- **Behavior**: none on SSE surfaces. Websocket relay frames whose matched + response-id rewrite is an identity are now forwarded with the upstream JSON + text instead of a canonical `json.dumps` re-encode (JSON-equivalent; frames + without a matched request state were already forwarded verbatim). +- **Performance**: removes 1–2 pydantic validations and 1–2 redundant JSON + parses per delta frame per layer, plus one wasted `format_sse_event` and one + wasted `json.dumps` per websocket frame. diff --git a/openspec/changes/validate-stream-lifecycle-events-only/specs/responses-api-compat/spec.md b/openspec/changes/validate-stream-lifecycle-events-only/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..f9fe0a7b95 --- /dev/null +++ b/openspec/changes/validate-stream-lifecycle-events-only/specs/responses-api-compat/spec.md @@ -0,0 +1,36 @@ +# responses-api-compat Delta + +## MODIFIED Requirements + +### Requirement: Streaming events are parsed once and re-serialized only when modified + +Within each streaming layer (core client consumer, streaming mixin, bridge upstream reader, websocket relay, /v1 normalizers), an SSE event's JSON payload MUST be parsed at most once and reused by that layer's consumers, and an event that no consumer modified MUST NOT be re-serialized by the /v1 normalizers. Schema validation of the parsed payload MUST run only for stream lifecycle frames (`response.created`, `response.completed`, `response.incomplete`, `response.failed`, `error`); all other frames MUST be classified from the parsed payload's `type` field (with a typeless payload carrying an `error` object classifying as `error`). Event framing, payload contents, dedupe/rewrite semantics, usage settlement, and error normalization MUST be unchanged. + +#### Scenario: Unmodified events pass through the /v1 normalizer verbatim + +- **GIVEN** a canonical stream event that no normalizer branch rewrites +- **WHEN** the /v1 response normalizer processes it +- **THEN** the original block is yielded byte-identically without re-serialization + +#### Scenario: Tool-call rewrite reuses the parsed event on the no-change path + +- **GIVEN** an event without duplicate parallel tool calls +- **WHEN** the rewrite step runs with the caller's parsed event +- **THEN** it returns the original line, payload, and event without re-parsing or re-validating + +#### Scenario: Rewritten events stay consistent + +- **WHEN** the rewrite step removes duplicate tool calls +- **THEN** the returned line, payload, and validated event all reflect the rewritten content + +#### Scenario: Delta frames skip schema validation + +- **GIVEN** a stream of `response.output_text.delta` frames between `response.created` and `response.completed` +- **WHEN** the streaming mixin, websocket relay, or bridge upstream reader processes the stream +- **THEN** only the lifecycle frames are schema-validated, the delta frames are classified from the parsed payload dict, and downstream output, usage settlement, and error normalization are unchanged + +#### Scenario: Identity websocket relay frames are forwarded without re-encoding + +- **GIVEN** a websocket frame matched to a request whose downstream response-id rewrite does not apply +- **WHEN** the relay forwards the frame downstream +- **THEN** the upstream frame text is forwarded as-is instead of a canonical JSON re-encode diff --git a/openspec/changes/validate-stream-lifecycle-events-only/tasks.md b/openspec/changes/validate-stream-lifecycle-events-only/tasks.md new file mode 100644 index 0000000000..7fddb00bc0 --- /dev/null +++ b/openspec/changes/validate-stream-lifecycle-events-only/tasks.md @@ -0,0 +1,29 @@ +# Tasks — validate-stream-lifecycle-events-only + +## 1. Implementation + +- [x] 1.1 `_LIFECYCLE_EVENT_TYPES` + `classify_event_type` in + `app/core/openai/parsing.py`; `_event_type_from_payload` and + `tool_call_dedupe.event_type_from_payload` delegate to it +- [x] 1.2 Streaming mixin (first-event block + loop): classify from dict, + validate lifecycle frames only +- [x] 1.3 Core client: delete redundant per-chunk `parse_sse_event` terminal + checks (SSE loops keep the `normalized_event_type` branch); websocket + receive loops detect terminal via `parse_sse_data_json` + `type` +- [x] 1.4 Websocket relay: single `json.loads`, lifecycle-only validation, + pass `event=` into `rewrite_parallel_tool_call_text`, stop building the + discarded `format_sse_event` argument, skip `json.dumps` on identity + response-id rewrite +- [x] 1.5 `tool_call_dedupe` rewrite helpers: no re-validation on the + unchanged path +- [x] 1.6 HTTP-bridge upstream reader: same lifecycle gating + +## 2. Validation + +- [x] 2.1 Existing streaming/websocket/bridge unit + integration suites pass + unmodified (byte-level SSE assertions act as the parity oracle) +- [x] 2.2 Regression tests: lifecycle-only validation counts with interleaved + `event=None` deltas on the SSE mixin and websocket relay (usage + settlement, error rewrite, response-id assignment), dedupe helpers do + not re-validate unchanged frames, `classify_event_type` unit coverage +- [x] 2.3 `uvx ruff format --check`, `uv run ruff check` on changed files diff --git a/openspec/changes/warm-free-plan-transition/design.md b/openspec/changes/warm-free-plan-transition/design.md new file mode 100644 index 0000000000..77c51f1cb2 --- /dev/null +++ b/openspec/changes/warm-free-plan-transition/design.md @@ -0,0 +1,76 @@ +## Context + +See `proposal.md` for motivation. The usage updater mutates and synchronizes the +selected account only after its existing paid-to-Free confirmation policy is +satisfied. The warm-up service currently sees only the post-refresh account and +requires matching canonical before/after windows, so it cannot distinguish a +confirmed plan transition from an account that was already Free. + +The existing `usage_reset_confirmed` guard protects ordinary reset detection +from cross-window comparisons and timestamp drift. The transition path must not +weaken that guard. + +## Goals / Non-Goals + +**Goals:** + +- Carry enough refresh-scoped evidence to identify a confirmed paid-to-Free + transition without introducing new persistent state. +- Require the monthly candidate to have been written by the same refresh and + to pass the existing availability, account, and global opt-in gates. +- Reuse the existing monthly warm-up identity and atomic claim. + +**Non-Goals:** + +- Changing paid-to-Free confirmation or ordinary same-window reset detection. +- Adding settings, schema, migrations, retry queues, or periodic backfill. +- Sending warm-up traffic to inactive or non-opted-in accounts. + +## Decisions + +### Snapshot the selected account plan before refresh + +The scheduler will preserve the selected account's normalized pre-refresh plan +and pass it to warm-up evaluation after reloading the account. A transition is +eligible only when the snapshot is a recognized paid plan and the persisted +post-refresh plan is `free`. + +Alternative considered: infer a transition from `secondary` to `monthly` usage +rows. That would incorrectly classify already-Free accounts whose first monthly +sample arrives after stale secondary history. + +### Require a monthly sample written during the same refresh + +The fallback candidate will accept only the selected long-window row when its +canonical window is `monthly`, it has a reset deadline, and its `recorded_at` is +at or after the refresh start. It will apply the existing minimum-availability +gate before returning a candidate. + +Alternative considered: use the latest persisted monthly row regardless of +age. That could warm stale quota after an unrelated plan metadata update. + +### Keep the transition as a fallback to normal reset detection + +The service will first evaluate the existing same-window reset candidate. Only +when that returns no candidate for the configured long window will it evaluate +the paid-to-Free transition. The resulting candidate uses `window="monthly"` +and the monthly `reset_at`, so the existing atomic attempt claim provides +deduplication. + +Alternative considered: alter `usage_reset_confirmed` to allow cross-window +transitions. That would weaken a safety guard used by status recovery and +ordinary warm-up paths. + +## Risks / Trade-offs + +- [A process exits after persisting plan and usage but before warm-up] → The + transition can be missed, matching the current event-triggered reset path; + avoid new persistence until stronger delivery semantics are required. +- [A future updater mutates plan before confirmation] → Keep regression coverage + at scheduler/service boundaries and rely on the updater's existing durable + two-observation confirmation contract. + +## Migration Plan + +No data migration is required. Deploy the code normally; rollback restores the +previous behavior without changing stored warm-up attempts or usage history. diff --git a/openspec/changes/warm-free-plan-transition/proposal.md b/openspec/changes/warm-free-plan-transition/proposal.md new file mode 100644 index 0000000000..807336756f --- /dev/null +++ b/openspec/changes/warm-free-plan-transition/proposal.md @@ -0,0 +1,35 @@ +## Why + +A confirmed paid-to-Free plan change can replace the account's prior paid quota +window with a newly available monthly window. The existing same-window safety +guard correctly rejects arbitrary cross-window comparisons, but it also skips +the opted-in warm-up for this confirmed plan transition. + +## What Changes + +- Preserve the selected account's plan type across one background refresh. +- Treat a confirmed paid-to-Free transition that writes a fresh available + monthly sample as a long-window warm-up candidate. +- Keep ordinary reset detection restricted to matching canonical windows and + keep single, unconfirmed Free observations ineligible. +- Add regressions for the consumer-visible warm-up attempt and the safety + boundaries around unchanged plans and stale monthly history. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `usage-refresh-policy`: Allow an opted-in long-window warm-up after a + confirmed paid-to-Free transition opens a fresh monthly quota window. + +## Impact + +- Affected code: `app/core/usage/refresh_scheduler.py` and + `app/modules/limit_warmup/service.py`. +- Affected tests: focused scheduler and limit warm-up tests. +- No API, schema, migration, setting, dependency, dashboard, or deployment + change. diff --git a/openspec/changes/warm-free-plan-transition/specs/usage-refresh-policy/spec.md b/openspec/changes/warm-free-plan-transition/specs/usage-refresh-policy/spec.md new file mode 100644 index 0000000000..73167cfcd8 --- /dev/null +++ b/openspec/changes/warm-free-plan-transition/specs/usage-refresh-policy/spec.md @@ -0,0 +1,56 @@ +## ADDED Requirements + +### Requirement: Confirmed paid-to-Free transitions warm the new monthly window + +When background usage refresh confirms that an opted-in active account changed +from a recognized paid plan to `free`, and that confirming refresh writes a +fresh monthly usage sample with a reset deadline and enough available quota for +the configured warm-up threshold, the system SHALL attempt one long-window +warm-up for that monthly quota window. Eligibility MUST NOT depend on the usage +percentage reported before the plan change. + +The plan-transition exception SHALL apply only to an actual paid-to-Free change +confirmed by the refresh that wrote the monthly sample. It MUST NOT apply to a +single unconfirmed Free observation, an account that was already Free, or a +monthly sample left over from an earlier refresh. Ordinary same-window reset +detection MUST remain unchanged. The durable warm-up identity SHALL remain the +account, canonical `monthly` window, and monthly reset deadline. The confirming +monthly sample MUST report `used_percent < 100`; the configured minimum- +available threshold MAY impose a stricter lower usage limit. + +#### Scenario: Confirmed paid-to-Free transition warms fresh monthly quota + +- **GIVEN** an active opted-in account whose stored plan is a recognized paid plan +- **WHEN** background usage refresh confirms its transition to `free` +- **AND** that confirming refresh writes a monthly sample with a reset deadline and enough available quota +- **THEN** the system attempts one warm-up identified by the account, `monthly` window, and monthly reset deadline + +#### Scenario: Previous usage percentage does not gate plan-transition warm-up + +- **GIVEN** an active opted-in paid account whose previous selected quota sample was not exhausted +- **WHEN** background usage refresh confirms its transition to `free` and writes an eligible fresh monthly sample +- **THEN** the system attempts the monthly warm-up regardless of the previous usage percentage + +#### Scenario: One unconfirmed Free observation does not warm + +- **GIVEN** an active opted-in account whose stored plan is a recognized paid plan +- **WHEN** one background usage refresh reports `free` without satisfying downgrade confirmation +- **THEN** no plan-transition warm-up is attempted + +#### Scenario: Already-Free account does not use the plan-transition exception + +- **GIVEN** an active opted-in account whose stored plan was already `free` +- **WHEN** background usage refresh writes its first monthly sample without confirming a plan change +- **THEN** no plan-transition warm-up is attempted + +#### Scenario: Stale monthly history does not warm after a plan change + +- **GIVEN** an active opted-in account whose transition from a paid plan to `free` is confirmed +- **WHEN** the latest monthly sample predates the confirming refresh +- **THEN** no plan-transition warm-up is attempted + +#### Scenario: Existing durable identity deduplicates the transition warm-up + +- **GIVEN** a warm-up attempt already exists for an account, `monthly` window, and monthly reset deadline +- **WHEN** the same confirmed paid-to-Free transition is evaluated again +- **THEN** no second warm-up request is sent for that durable identity diff --git a/openspec/changes/warm-free-plan-transition/tasks.md b/openspec/changes/warm-free-plan-transition/tasks.md new file mode 100644 index 0000000000..6aac835ede --- /dev/null +++ b/openspec/changes/warm-free-plan-transition/tasks.md @@ -0,0 +1,19 @@ +## 1. Refresh-scoped transition evidence + +- [x] 1.1 Snapshot the selected account's plan before background usage refresh. +- [x] 1.2 Pass the pre-refresh plan map and refresh timestamp into long-window warm-up evaluation. + +## 2. Monthly transition candidate + +- [x] 2.1 Add a paid-to-Free fallback candidate that requires a fresh available monthly sample. +- [x] 2.2 Preserve ordinary same-window reset detection and the existing durable monthly claim. + +## 3. Regression coverage + +- [x] 3.1 Prove a confirmed paid-to-Free scheduler refresh sends one monthly warm-up regardless of prior usage. +- [x] 3.2 Cover unconfirmed or unchanged Free plans, stale monthly history, availability gating, and deduplication. + +## 4. Validation + +- [x] 4.1 Run focused scheduler and limit warm-up tests. +- [x] 4.2 Run Ruff format/check, Ty, strict OpenSpec validation, and diff hygiene checks. diff --git a/openspec/changes/websocket-scope-cleanup-budget/.openspec.yaml b/openspec/changes/websocket-scope-cleanup-budget/.openspec.yaml new file mode 100644 index 0000000000..4af864176c --- /dev/null +++ b/openspec/changes/websocket-scope-cleanup-budget/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-14 diff --git a/openspec/changes/websocket-scope-cleanup-budget/design.md b/openspec/changes/websocket-scope-cleanup-budget/design.md new file mode 100644 index 0000000000..eb96d1d45b --- /dev/null +++ b/openspec/changes/websocket-scope-cleanup-budget/design.md @@ -0,0 +1,68 @@ +## Context + +The WebSocket handler publishes one tracked `finalize_websocket_scope()` task +from its `finally` block. The task must preserve cancellation while completing +the terminal request cleanup sequence. During process drain, +`shutdown_state.remaining_drain_timeout_seconds()` provides the shared absolute +deadline. Outside drain it returns `None`, so the current code falls back to +the generic one-second `_TASK_CANCEL_TIMEOUT_SECONDS` value. + +The generic timeout is used across HTTP bridge and proxy child-task cancellation +paths. Increasing it globally would slow unrelated cancellation and would +change the semantics of a helper that is intentionally a short cancellation +observation bound. The scope finalizer needs a different bounded allowance +because its work is a sequence of request-state and lease finalization steps. + +## Goals / Non-Goals + +**Goals:** + +- Give normal direct WebSocket scope cleanup enough bounded time for the + existing request finalization and lease-release sequence. +- Preserve the existing tracked-task ownership and cancellation behavior when + the bound is reached. +- Keep shutdown cleanup governed by the one shared remaining drain deadline. +- Prove the behavior through the real WebSocket route finalizer. + +**Non-Goals:** + +- Changing the `response.created` watchdog or any upstream request budget. +- Retrying, replaying, or moving an interrupted request to another account. +- Increasing the generic `_TASK_CANCEL_TIMEOUT_SECONDS` value. +- Adding an operator setting, environment variable, database state, or a new + background cleanup registry. + +## Decisions + +1. **Use one internal five-second scope budget.** The value is deliberately + fixed and bounded because this is a lifecycle safety allowance, not an + operator tuning surface. Five seconds is long enough to absorb ordinary + persistence/lease scheduling variance while still returning promptly when + teardown is stuck. + +2. **Prefer the active drain deadline.** When shutdown drain is active, the + finalizer continues to use the remaining shared deadline exactly as today. + The normal-operation budget is only the fallback for the no-drain case and + cannot extend process shutdown. + +3. **Keep child cancellation semantics separate.** Individual task waits keep + the one-second generic cancellation bound during normal operation. During + drain they remain capped by the shared remaining deadline. Only the outer + scope-finalization wait receives the five-second normal-operation allowance. + +4. **Retain tracked cleanup after the bound.** `asyncio.wait()` continues to + observe the finalizer without cancelling it at the scope budget. The + existing `_background_cleanup_tasks` registry and persistence drain remain + the owner of unfinished cleanup, so a timeout is honest and does not cause + lease or request finalization to be abandoned. + +## Verification Strategy + +- Run the focused WebSocket terminal-cancellation tests, including a regression + that lowers the generic task timeout and delays finalization beyond it while + allowing completion within the separate scope budget. +- Run Ruff check/format on changed Python files, the proxy architecture check, + and the applicable type/test targets. +- Validate the OpenSpec delta if the CLI is available; otherwise record the + unavailable local CLI as a handoff limitation and keep the artifacts in the + repository for CI validation. diff --git a/openspec/changes/websocket-scope-cleanup-budget/proposal.md b/openspec/changes/websocket-scope-cleanup-budget/proposal.md new file mode 100644 index 0000000000..6666fe1962 --- /dev/null +++ b/openspec/changes/websocket-scope-cleanup-budget/proposal.md @@ -0,0 +1,41 @@ +## Why + +Direct Responses WebSocket scope teardown currently reuses the generic +`_TASK_CANCEL_TIMEOUT_SECONDS` value as its entire normal-operation cleanup +budget. That value is intentionally one second for individual task +cancellation, but scope teardown can also have to finalize request logs, +release response-create ownership, and release the account connection lease. +Under ordinary load those operations can exceed one second, producing +`Websocket scope cleanup exceeded its remaining drain budget` even when the +server is not draining. The cleanup task remains tracked, but the warning and +unfinished teardown increase the chance of follow-up reconnect churn. + +## What Changes + +- Give normal-operation WebSocket scope teardown its own fixed five-second + bounded budget. +- Keep the existing one-second generic task-cancellation timeout for ordinary + child-task waits. +- Continue using the remaining shared shutdown deadline whenever process drain + is active; the new budget must not extend shutdown. +- Add a route-level cancellation regression proving that cleanup which takes + longer than the generic task timeout can still finish within the scope budget + and does not leave an orphaned cleanup task. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: Direct Responses WebSocket scope cleanup has a + separate bounded normal-operation budget while preserving the shared + shutdown deadline and task ownership guarantees. + +## Impact + +The change is limited to the direct Responses WebSocket finalizer, its focused +unit coverage, and the OpenSpec contract. It adds no setting, dependency, +database migration, API shape, upstream watchdog change, or retry policy. diff --git a/openspec/changes/websocket-scope-cleanup-budget/specs/responses-api-compat/spec.md b/openspec/changes/websocket-scope-cleanup-budget/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..35fb23b2db --- /dev/null +++ b/openspec/changes/websocket-scope-cleanup-budget/specs/responses-api-compat/spec.md @@ -0,0 +1,39 @@ +# responses-api-compat Delta + +## ADDED Requirements + +### Requirement: Direct WebSocket scope cleanup has a bounded normal-operation budget + +When a direct Responses WebSocket scope exits while the process is not using an +active shutdown drain deadline, the proxy MUST allow its existing scope +finalization task a fixed five-second bounded observation budget, separate from +the one-second generic child-task cancellation timeout. The finalizer MUST +continue to own request finalization and lease cleanup through that budget. If +the budget expires, the proxy MUST preserve the existing cancellation result, +leave unfinished cleanup tracked by the existing cleanup-task registry, and +MUST NOT cancel or silently abandon that cleanup solely because the observation +budget expired. + +When an active shutdown drain deadline exists, the proxy MUST use the remaining +shared drain deadline instead of the normal-operation budget, so normal cleanup +allowance MUST NOT extend process shutdown. + +#### Scenario: normal scope cleanup outlives generic child cancellation + +- **GIVEN** a direct Responses WebSocket scope is cancelled while its existing + request finalization takes longer than the generic one-second child-task + cancellation timeout +- **AND** the finalization completes within the five-second normal-operation + scope budget +- **WHEN** scope cleanup runs +- **THEN** the finalizer completes and request/lease ownership is released +- **AND** the scope preserves its cancellation result +- **AND** no cleanup task remains orphaned after the finalizer completes + +#### Scenario: shutdown drain remains the upper bound + +- **GIVEN** a direct Responses WebSocket scope is cancelled while an active + shutdown drain deadline has less than five seconds remaining +- **WHEN** scope cleanup runs +- **THEN** the remaining shared drain deadline remains the upper bound +- **AND** the normal-operation five-second budget does not extend shutdown diff --git a/openspec/changes/websocket-scope-cleanup-budget/tasks.md b/openspec/changes/websocket-scope-cleanup-budget/tasks.md new file mode 100644 index 0000000000..1d68bdb7e5 --- /dev/null +++ b/openspec/changes/websocket-scope-cleanup-budget/tasks.md @@ -0,0 +1,23 @@ +## 1. Regression Coverage + +- [x] 1.1 Add a real direct Responses WebSocket cancellation regression that + distinguishes the generic task timeout from the scope cleanup budget. +- [x] 1.2 Confirm the regression fails against the baseline implementation and + passes with the scoped budget. + +## 2. Scope Cleanup Budget + +- [x] 2.1 Add the fixed normal-operation WebSocket scope cleanup budget. +- [x] 2.2 Preserve the active shared shutdown deadline and one-second generic + child-task cancellation behavior. +- [x] 2.3 Keep unfinished cleanup tracked and prevent cancellation/lease + ownership regressions when the bound expires. + +## 3. Verification + +- [x] 3.1 Run focused WebSocket terminal-cancellation tests. +- [x] 3.2 Run changed-file Ruff check/format, proxy architecture checks, and + applicable type checks. +- [x] 3.3 Validate the OpenSpec delta and inspect the final diff/status. +- [x] 3.4 Open a Draft PR targeting upstream `main` and add the live 1.23.0 + evidence to issue #1711. diff --git a/openspec/specs/account-import/spec.md b/openspec/specs/account-import/spec.md new file mode 100644 index 0000000000..72e8cfad7c --- /dev/null +++ b/openspec/specs/account-import/spec.md @@ -0,0 +1,65 @@ +# account-import Specification + +## Purpose +Authorization and resource bounds for importing account auth material (auth.json uploads and equivalent flows) into the pool. +## Requirements +### Requirement: Account auth imports are authorized and bounded + +`POST /api/accounts/import` MUST authenticate the dashboard session and require dashboard write access before reading any request-body bytes. It MUST accept exactly one file part named `auth_json`, no text parts, a file size no greater than 1 MiB (1,048,576 bytes), and a complete multipart body no greater than 2 MiB (2,097,152 bytes). + +The service MUST enforce the body limit against both a usable declared `Content-Length` and the actual streamed bytes. It MUST enforce the file limit before retaining bytes beyond the limit, close every multipart spool before account persistence or import-time network work begins, and add no new runtime setting. + +This route-owned policy MUST take precedence over the generic raw HTTP body budget for `POST /api/accounts/import`. Its exact-path content-encoding gate MUST run outside the generic raw and decompression guards regardless of the declared media type. Requests handled by that gate, and unencoded requests declared as multipart, MUST NOT be rejected by the generic guards before dashboard authorization or the dedicated parser applies this capability's body limit. An unencoded request that does not declare multipart remains under generic admission and MAY be rejected there before authorization. This exception MUST NOT change generic ingress behavior for any other operation. + +Byte-limit failures MUST return HTTP 413 with dashboard error `code = payload_too_large`. Missing or non-file `auth_json` input MUST retain the dashboard validation envelope, while malformed multipart syntax or additional parts MUST return a dashboard-compatible HTTP 400 without invoking account import logic. + +#### Scenario: Unauthorized import does not consume the body + +- **WHEN** a request without a valid dashboard session or write permission targets account import +- **THEN** the existing authentication or permission response is returned before the ASGI request body is consumed +- **AND** no multipart temporary file is created + +#### Scenario: Valid bounded auth file imports normally + +- **WHEN** an authorized operator uploads exactly one valid `auth_json` file and both file and multipart body are within their limits +- **THEN** the existing account identity, persistence, usage-refresh, cache-invalidation, and audit behavior continues +- **AND** the multipart spool is closed before persistence or network work begins + +#### Scenario: Declared or streamed account-import body exceeds its limit + +- **WHEN** a usable `Content-Length` exceeds 2 MiB or actual streamed multipart bytes cross 2 MiB +- **THEN** the service returns HTTP 413 with dashboard error `code = payload_too_large` +- **AND** it does not parse credentials, mutate an account, refresh usage, invalidate caches, or write a success audit event + +#### Scenario: Auth file exceeds its limit + +- **WHEN** the `auth_json` file part exceeds 1 MiB while the multipart body is otherwise valid +- **THEN** the service returns HTTP 413 with dashboard error `code = payload_too_large` +- **AND** bytes beyond the file limit are not retained in a multipart spool or handler buffer + +#### Scenario: Account import has an invalid multipart shape + +- **WHEN** an import omits a file-valued `auth_json` part or includes duplicate, additional file, or text parts +- **THEN** the service returns the established dashboard validation or bad-request envelope +- **AND** account import logic is not invoked + +#### Scenario: Compressed account import is rejected without prebuffering + +- **GIVEN** account import has passed dashboard session and write authorization +- **WHEN** it declares a non-identity `Content-Encoding` +- **THEN** the service returns HTTP 400 with dashboard error `code = invalid_request` before reading the request body +- **AND** a no-op `identity` encoding is handled as an ordinary multipart request governed by the 2 MiB dedicated body limit + +#### Scenario: Generic ingress does not preempt encoded account-import authorization + +- **GIVEN** an account-import request fails dashboard session or write authorization +- **WHEN** it declares a non-identity `Content-Encoding` and a `Content-Length` greater than the generic raw HTTP budget +- **THEN** the existing authentication or permission response is returned instead of a generic HTTP 413 or encoded-body HTTP 400 +- **AND** the request body is not consumed + +#### Scenario: Disconnect and cancellation clean up parsing + +- **WHEN** the client disconnects or request processing is cancelled during account multipart parsing +- **THEN** every created spool is closed +- **AND** the disconnect or cancellation propagates without being converted to HTTP 413 + diff --git a/openspec/specs/account-routing/spec.md b/openspec/specs/account-routing/spec.md index 7b5f73980b..3fcca9d59b 100644 --- a/openspec/specs/account-routing/spec.md +++ b/openspec/specs/account-routing/spec.md @@ -292,26 +292,44 @@ unit. When the hint contains no recognizable unit token, the system SHALL fall back to the error-count backoff schedule. A rate-limited account SHALL NOT be re-selected before its cooldown elapses. -When the upstream rate-limit error carries no explicit reset metadata -(`resets_at`/`resets_in_seconds`), the resolved cooldown deadline SHALL be -persisted on the account row (`reset_at`) so the cooldown survives process -restarts and is visible to all replicas sharing the database: a parsed -Retry-After hint deadline SHALL be persisted rounded up to the next whole -second (persistence stores `reset_at` as an integer, so a short or fractional -hint MUST NOT truncate down to an already-elapsed deadline), and when the -cooldown comes from the error-count backoff fallback the persisted deadline -SHALL be at least `RATE_LIMITED_MIN_COOLDOWN_SECONDS` (30 seconds) in the -future. Explicit upstream reset metadata, when present, SHALL continue to be -persisted as-is. -The marking replica's in-process cooldown MAY remain shorter than the -persisted deadline so its existing fresh-usage recovery gate is unchanged. +Explicit upstream reset metadata SHALL be accepted only when it resolves to a +finite deadline strictly later than the current time and no more than +`RATE_LIMIT_RESET_MAX_HORIZON_SECONDS` (366 days) in the future. `resets_at` +SHALL be interpreted as an absolute Unix timestamp and `resets_in_seconds` +SHALL be interpreted as a relative duration. When `resets_at` is invalid but +`resets_in_seconds` is valid, the relative duration SHALL be used. An accepted +fractional deadline SHALL be rounded up to the next whole second before +persistence. A persisted integer deadline produced by that rounding MAY be +less than one second beyond the raw 366-day horizon and MUST remain valid when +selection reconstructs it. When neither field is valid, the error SHALL be +treated as carrying no explicit reset metadata. + +When the upstream rate-limit error carries no valid explicit reset metadata, +the resolved cooldown deadline SHALL be persisted on the account row +(`reset_at`) so the cooldown survives process restarts and is visible to all +replicas sharing the database: a parsed Retry-After hint deadline SHALL be +persisted rounded up to the next whole second (persistence stores `reset_at` +as an integer, so a short or fractional hint MUST NOT truncate down to an +already-elapsed deadline), and when the cooldown comes from the error-count +backoff fallback the persisted deadline SHALL be at least +`RATE_LIMITED_MIN_COOLDOWN_SECONDS` (30 seconds) in the future. The marking +replica's in-process cooldown MAY remain shorter than the persisted deadline +so its existing fresh-usage recovery gate is unchanged. + +An already-persisted `rate_limited` reset deadline beyond the same plausibility +horizon SHALL be treated as missing metadata rather than as an unexpired +cooldown. A row carrying `blocked_at` SHALL still honor the existing 30-second +minimum floor and SHALL require recent usage evidence recorded after that block +before selection-time recovery may clear it. A row without `blocked_at` SHALL +require recent available usage evidence. In both cases, every applicable +derived quota window MUST report below `100%` usage before recovery. #### Scenario: Compound minute-and-second hint sets the full cooldown - **GIVEN** an upstream 429 whose message says "try again in 6m0s" - **WHEN** the balancer records the rate limit for the account - **THEN** the account cooldown lasts 360 seconds -- **AND** the account is not re-selected until that cooldown elapses +- **AND** the account is not re-selected until its cooldown elapses #### Scenario: Minutes-only hint is honored @@ -352,6 +370,58 @@ persisted deadline so its existing fresh-usage recovery gate is unchanged. - **THEN** the persisted integer `reset_at` deadline is strictly in the future - **AND** peer replicas honor the hinted cooldown instead of reselecting the account immediately +#### Scenario: Plausible explicit reset metadata remains authoritative + +- **GIVEN** an OpenAI service 429 carrying a finite `resets_at` deadline 30 days in the future +- **WHEN** the balancer records the rate limit for the account +- **THEN** the accepted explicit deadline is persisted +- **AND** the Retry-After/backoff fallback does not replace it + +#### Scenario: Implausible explicit reset metadata uses the bounded fallback + +- **GIVEN** an OpenAI service 429 carrying `resets_at=15023672358` while the current Unix time is approximately `1784146959` +- **AND** the error carries no valid `resets_in_seconds` or parseable duration +- **WHEN** the balancer records the rate limit for the account +- **THEN** the implausible absolute deadline is rejected +- **AND** the persisted deadline uses the minimum bounded backoff instead + +#### Scenario: Valid relative metadata survives an invalid absolute value + +- **GIVEN** an OpenAI service 429 whose `resets_at` is implausibly far in the future +- **AND** whose `resets_in_seconds` is a finite positive duration within 366 days +- **WHEN** the balancer records the rate limit for the account +- **THEN** the relative duration determines the persisted deadline + +#### Scenario: Horizon-edge rounding remains stable + +- **GIVEN** valid absolute or relative reset metadata resolves exactly 366 days after a fractional current timestamp +- **WHEN** the balancer rounds and persists the deadline to a whole second +- **THEN** persisted-state reconstruction continues to accept that deadline +- **AND** does not clear the cooldown solely because rounding crossed the raw horizon by less than one second + +#### Scenario: Existing implausible deadline does not pin selection indefinitely + +- **GIVEN** a persisted `rate_limited` account whose `reset_at` is more than 366 days in the future +- **AND** whose `blocked_at` minimum floor has elapsed +- **WHEN** selection reconstructs the account from fresh available usage evidence +- **THEN** the implausible deadline is treated as missing metadata +- **AND** normal compare-and-set recovery may restore the account to `active` + +#### Scenario: Exhausted long-window quota prevents poisoned-row recovery + +- **GIVEN** a persisted `rate_limited` account whose reset deadline is implausible +- **AND** a fresh primary window reports available quota +- **AND** an applicable weekly or monthly window reports `100%` usage +- **WHEN** selection reconstructs the account +- **THEN** the account remains `rate_limited` + +#### Scenario: Implausible legacy deadline without a block marker recovers + +- **GIVEN** a persisted `rate_limited` account whose reset deadline is implausible +- **AND** the row has no `blocked_at` marker +- **WHEN** selection reconstructs the account from recent available usage in every applicable window +- **THEN** normal compare-and-set recovery may restore the account to `active` + ### Requirement: Re-authentication-required accounts are not selectable When an account credential/session is invalidated but the upstream account is not known to be disabled, the system MUST mark the account `reauth_required`. The selector MUST remove `reauth_required` accounts from every routing strategy and hard-affinity fallback until the account is re-authenticated. Operator pickers that configure single-account routing or account-scoped routing MUST only offer accounts that are not hard-blocked by paused, reauth-required, or deactivated status. @@ -641,3 +711,36 @@ Recovery admission MUST occur only after all ordinary account eligibility, coold - **WHEN** selection finalizes the stable local account-cap error - **THEN** any provisional delete or rebind decision is discarded - **AND** the existing hard-sticky owner mapping remains unchanged + +### Requirement: Trusted cyber intent narrows the existing account pool + +Account routing MUST constrain an authenticated direct Responses WebSocket +turn requiring `trusted_cyber` by passing +`require_security_work_authorized=True` to the canonical selector before the +first upstream attempt and every later retry. The selector MUST apply the +constraint only to accounts already permitted by API-key, account, model, +service-tier, ownership, health, quota, affinity, concurrency, and failover +rules. Routing MUST NOT add an account, change the configured strategy, rebind +an owner, or fall back to an ordinary account. + +#### Scenario: First attempt uses the capable pool +- **WHEN** an authenticated direct WebSocket turn establishes `trusted_cyber` +- **THEN** its first account-selection call requires a + security-work-authorized account +- **AND** no ordinary account receives an upstream attempt + +#### Scenario: Empty capable pool fails closed +- **WHEN** a required turn has no eligible security-work-authorized account +- **THEN** selection returns the existing typed + `no_security_work_authorized_accounts` error +- **AND** its advisory states that no ordinary-account fallback occurred +- **AND** an earlier reactive or account/model error cannot replace that typed + capability-routing result +- **AND** ordinary routing is not attempted + +#### Scenario: Ordinary routing is unchanged +- **WHEN** an authenticated direct WebSocket turn has neither a trusted signal + nor required lineage +- **THEN** selection receives the same scope, strategy, ownership, admission, + and retry inputs as before this change + diff --git a/openspec/specs/api-keys/spec.md b/openspec/specs/api-keys/spec.md index ac3311d678..274490f613 100644 --- a/openspec/specs/api-keys/spec.md +++ b/openspec/specs/api-keys/spec.md @@ -272,7 +272,13 @@ The system SHALL keep the existing lazy on-read reset strategy for API key usage ### Requirement: RequestLog API key reference -The system SHALL record the `api_key_id` in the `request_logs` table for proxy requests authenticated with an API key. The field MUST be NULL when API key auth is disabled or the request is unauthenticated. +The system SHALL record the `api_key_id` in the `request_logs` table for proxy +requests authenticated with an API key. The field MUST be NULL when API key +auth is disabled or the request is unauthenticated. This applies to error rows +as well as successes: when a shared upstream session (e.g. an HTTP-bridge +session multiplexing requests from multiple API keys) fails its pending +requests, each request's log row MUST be attributed to that request's own +authenticated key. #### Scenario: Authenticated request logged @@ -284,6 +290,16 @@ The system SHALL record the `api_key_id` in the `request_logs` table for proxy r - **WHEN** API key auth is disabled and a proxy request completes - **THEN** the `request_logs` entry has `api_key_id = NULL` +#### Scenario: Bridge failure fan-out preserves per-request key attribution + +- **GIVEN** an HTTP-bridge session holds a pending request authenticated with + API key `key-123` +- **WHEN** the session fails its pending requests (upstream close, send + failure, request timeout, or local terminal error) +- **THEN** the request's `request_logs` error entry has + `api_key_id = "key-123"` even though the session-level failure path has no + single key of its own + ### Requirement: Frontend API Key management The SPA settings page SHALL include an API Key management section with: a toggle for `apiKeyAuthEnabled`, a key list table showing prefix/name/models/limit/usage/expiry/status, a create dialog (name, model selection, assigned-account selection, usage sections multi-select, weekly limit, expiry date), and key actions (edit, delete, regenerate). On key creation, the SPA MUST display the plain key in a copy-able dialog with a warning that it will not be shown again, and the copy action MUST remain functional in secure and non-secure contexts. @@ -520,6 +536,11 @@ Usage reservation의 최종 정산(finalize 또는 release)은 요청 단위에 Reservation 생성 후 upstream API 호출에 진입하지 않고 종료되는 모든 경로에서 reservation이 release되어야 한다. `reserved` 상태로 남는 reservation이 존재하면 안 된다. 시스템은 이 동작을 SHALL 보장해야 한다. +After admission commits an owned reservation, rate-limit response-header +calculation before upstream work remains part of the early-exit cleanup window. +If that calculation fails, the system MUST attempt to release the owned +reservation exactly once before propagating the original header failure. + #### Scenario: no_accounts 즉시 종료 시 release - **WHEN** reservation 생성 후 `_stream_with_retry()`가 사용 가능한 계정 없음(`no_accounts`)으로 즉시 종료되면 @@ -536,6 +557,17 @@ Reservation 생성 후 upstream API 호출에 진입하지 않고 종료되는 - **WHEN** API key auth가 비활성이거나 reservation이 생성되지 않은 상태에서 요청이 종료되면 - **THEN** 정산 로직이 안전하게 스킵되어야 하며 에러가 발생하지 않아야 한다 (SHALL) +#### Scenario: Rate-limit header preparation fails after admission + +- **GIVEN** a limited API key has committed an owned reservation for a + streaming Responses, collected Responses, compact Responses, or audio + transcription request +- **WHEN** rate-limit response-header calculation fails before upstream work + begins +- **THEN** the reservation is released exactly once +- **AND** its reserved quota is restored +- **AND** the header failure propagates without starting upstream work + ### Requirement: Compact 경로 예외 무관 reservation cleanup `_compact_responses()` 경로에서 reservation이 존재할 때, 어떤 예외 타입이 발생하더라도 reservation이 정리되어야 한다. 특정 예외 타입에만 의존하는 cleanup은 허용되지 않는다. 시스템은 이 동작을 SHALL 보장해야 한다. @@ -1108,7 +1140,27 @@ API-key limit and usage-reporting paths used by subscription-backed requests. ### Requirement: Stream reservation settlement is detached from the response path -Settling a stream API-key reservation MUST NOT block the response/stream close, with one deliberate exception: when a keyed websocket stream terminates with an account-health error, the finalizer MUST wait for the settlement to commit before the load-balancer health write (the settlement-ordering invariant), so that error path intentionally blocks on settlement. In all other cases the settlement MUST run as a tracked background task; when it fails or is cancelled, the reservation MUST still be released by the tracking fallback, and the request's finalization path MUST NOT double-release a transferred settlement. Reservations MUST continue to count toward key limits until finalized or released, so deferred settlement can never admit usage a synchronous settlement would have rejected. +Settling a stream API-key reservation MUST NOT block the response/stream close, +with one deliberate exception: when a keyed websocket stream terminates with an +account-health error, the finalizer MUST wait for the settlement to commit +before the load-balancer health write (the settlement-ordering invariant), so +that error path intentionally blocks on settlement. If the primary settlement +fails, the finalizer MUST wait for fallback release to commit before recording +account health. If neither operation confirms settlement, the account-health +write MUST remain unapplied. Tracked persistence ownership MUST remain +registered through an ordering-sensitive fallback release, including +cancellation before the primary coroutine starts or during that release, so +graceful shutdown drains both phases. When the existing stream-retry path +deliberately defers an +account-health penalty until the same ordering-sensitive settlement, it MUST +likewise apply neither that penalty nor an immediately following terminal health +write unless settlement is confirmed, and it MUST NOT start a second settlement +for the transferred reservation. In all other cases the settlement MUST run as +a tracked background task; when it fails or is cancelled, the reservation MUST +still be released by the tracking fallback, and the request's finalization path +MUST NOT double-release a transferred settlement. Reservations MUST continue to +count toward key limits until finalized or released, so deferred settlement can +never admit usage a synchronous settlement would have rejected. #### Scenario: Response close precedes settlement completion @@ -1129,10 +1181,33 @@ Settling a stream API-key reservation MUST NOT block the response/stream close, - **WHEN** the finalizer settles the reservation - **THEN** it waits for the settlement to commit before recording the account-health error +#### Scenario: Websocket health waits for fallback settlement + +- **GIVEN** a keyed websocket stream that terminates with an account-health error +- **AND** its primary settlement fails +- **WHEN** fallback release remains in progress +- **THEN** the finalizer does not record the account-health error +- **AND** it records the error only after fallback release commits + +#### Scenario: Unconfirmed websocket settlement leaves health unapplied + +- **GIVEN** a keyed websocket stream that terminates with an account-health error +- **WHEN** both primary settlement and fallback release fail +- **THEN** the finalizer does not record the account-health error +- **AND** the upstream connection is still scheduled for reconnect and retirement + +#### Scenario: Unconfirmed retry settlement drops deferred health + +- **GIVEN** a keyed stream retry has deferred an account-health penalty until replacement selection +- **WHEN** neither primary settlement nor fallback release confirms settlement +- **THEN** the deferred penalty and any immediately following terminal health write remain unapplied +- **AND** the retry path does not start a second settlement for the transferred reservation + #### Scenario: Shutdown drains pending settlements - **WHEN** the service shuts down gracefully with settlements in flight - **THEN** shutdown waits for them up to the configured drain timeout +- **AND** a pending ordering-sensitive fallback release remains part of that drain despite cancellation before primary startup or during fallback ### Requirement: Untrusted forwarded headers do not grant unauthenticated proxy locality @@ -1255,3 +1330,92 @@ The system SHALL track `api_keys.last_used_at` through a process-local write-beh - **GIVEN** a settlement task that outlived the shutdown drain of persistence tasks - **WHEN** it records a touch after the flusher has stopped and performed its final flush - **THEN** the touch is flushed to the database immediately by the recording path rather than being lost at process exit + +### Requirement: GPT-5.6 personality pricing is recognized + +The system MUST recognize `gpt-5.6`, `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` when computing request costs. The bare `gpt-5.6` alias MUST resolve to Sol, and suffixed aliases for each personality model MUST resolve to the matching canonical pricing entry. Standard, Flex, Priority, and requests with more than 272K input tokens MUST use the published rates applicable to the model and tier. + +#### Scenario: Canonical GPT-5.6 models use personality-specific pricing + +- **WHEN** a standard-tier request completes for `gpt-5.6-sol`, `gpt-5.6-terra`, or `gpt-5.6-luna` +- **THEN** the system computes cost using that model's standard input, cached-input, and output rates + +#### Scenario: Bare GPT-5.6 alias resolves to Sol pricing + +- **WHEN** a request completes for `gpt-5.6` +- **THEN** the system resolves it to the canonical Sol pricing entry +- **AND** the system does not use the generic `gpt-5` pricing entry + +#### Scenario: Suffixed GPT-5.6 model resolves to its personality price + +- **WHEN** a request completes for a suffixed GPT-5.6 personality model ID +- **THEN** the system resolves it to the matching canonical Sol, Terra, or Luna pricing entry +- **AND** the system does not use the generic `gpt-5` pricing entry + +#### Scenario: GPT-5.6 service tiers use published tier rates + +- **WHEN** a GPT-5.6 request completes with `service_tier: "flex"` or `service_tier: "priority"` +- **THEN** the system computes cost using the published rates for that model and service tier + +#### Scenario: GPT-5.6 long-context request uses published uplift + +- **WHEN** a standard-tier or Flex GPT-5.6 request completes with more than 272K input tokens +- **THEN** the system computes cost using the published long-context input, cached-input, and output rates for that model and tier + +### Requirement: API-key limit rule identities are unique + +The system SHALL reject an API-key create or update payload when it contains +more than one limit rule with the same `(limit_type, limit_window, +model_filter)` identity. Rejection MUST use the typed API-key validation error +and MUST occur before a create request persists an API key or limit row. +The validation message MUST identify the duplicate rule identity. + +#### Scenario: Duplicate rules are rejected during creation + +- **WHEN** an administrator submits `POST /api/api-keys` with two limit rules + sharing the same type, window, and model filter +- **THEN** the API returns `400` with `invalid_api_key_payload` +- **AND** no API key or limit row is persisted + +### Requirement: Stale usage-reservation reclamation enforces a hard age ceiling + +Stale usage-reservation reclamation MUST reclaim `reserved` reservations whose +age exceeds a hard ceiling on creation time regardless of how recently their +`updated_at` was refreshed. This is the backstop for orphaned reservation +heartbeats: a leaked heartbeat task keeps touching `updated_at`, which would +otherwise exempt its reservation from the heartbeat-based staleness cutoff +forever. The ceiling MUST be far larger than any legitimate request lifetime +so it can never reclaim an in-flight reservation, and reclamation past the +ceiling MUST restore the reserved quota the same way heartbeat-based +reclamation does. + +#### Scenario: Orphaned heartbeat cannot exempt a reservation forever + +- **GIVEN** a `reserved` usage reservation created before the hard age ceiling +- **AND** a leaked heartbeat keeps refreshing its `updated_at` +- **WHEN** stale usage-reservation reclamation runs +- **THEN** the reservation is released and its reserved quota is restored + +#### Scenario: Fresh reservations are untouched by the ceiling + +- **GIVEN** a `reserved` usage reservation created within the hard age ceiling +- **AND** its `updated_at` is current +- **WHEN** stale usage-reservation reclamation runs +- **THEN** the reservation stays `reserved` + +### Requirement: API keys can enforce the Ultrafast service tier + +The dashboard API key CRUD surface MUST accept and persist `ultrafast` as a canonical enforced service tier. The service MUST return the same canonical value and MUST NOT normalize it to `priority`. + +#### Scenario: Create an API key with Ultrafast enforcement + +- **WHEN** a dashboard client creates an API key with `enforcedServiceTier: "ultrafast"` +- **THEN** the request is accepted +- **AND** the persisted and returned enforced service tier is `ultrafast` + +#### Scenario: Enforce Ultrafast on an advertising model + +- **GIVEN** an account model advertises the `ultrafast` service tier +- **WHEN** a request uses an API key whose enforced service tier is `ultrafast` +- **THEN** the upstream request carries `service_tier: "ultrafast"` + diff --git a/openspec/specs/audio-transcriptions-compat/spec.md b/openspec/specs/audio-transcriptions-compat/spec.md index 0790a5599d..133d3800e4 100644 --- a/openspec/specs/audio-transcriptions-compat/spec.md +++ b/openspec/specs/audio-transcriptions-compat/spec.md @@ -86,3 +86,67 @@ The system MUST enforce a configurable total request budget for transcription pr - **THEN** the retry uses the refreshed account metadata - **AND** the retry only proceeds if enough request budget remains for another attempt +### Requirement: Transcription multipart uploads are authorized and bounded + +`POST /backend-api/transcribe` and `POST /v1/audio/transcriptions` MUST complete their existing proxy authorization dependencies before reading multipart body bytes. Each request MUST contain exactly one file part no greater than 25,000,000 bytes, no more than 32 text fields of at most 256 KiB each, and a complete multipart body no greater than 32 MiB (33,554,432 bytes). + +The service MUST enforce the body limit against both a usable declared `Content-Length` and actual streamed bytes. It MUST enforce file and text limits before retaining crossing bytes, close multipart spools before usage reservation, account selection, or upstream forwarding, preserve ordered text-field forwarding for configured model sources, and add no new runtime setting. + +This route-owned policy MUST take precedence over the generic raw HTTP body budget for both transcription operations. Their exact-path content-encoding gate MUST run outside the generic raw and decompression guards regardless of the declared media type. Requests handled by that gate, and unencoded requests declared as multipart, MUST NOT be rejected by the generic guards before proxy authorization or the dedicated parser applies this capability's body limit. An unencoded request that does not declare multipart remains under generic admission and MAY be rejected there before authorization. This exception MUST NOT change generic ingress behavior for any other operation. + +Byte-limit failures MUST return HTTP 413 with OpenAI error `code = payload_too_large` and `type = invalid_request_error`; a file-part failure MUST set `param = file`. Multipart syntax, count, and required-field failures MUST retain OpenAI-compatible invalid-request behavior and MUST NOT reserve usage or call upstream. + +#### Scenario: Unauthorized transcription does not consume the body + +- **WHEN** a transcription request fails the existing proxy API-key authorization +- **THEN** the authentication response is returned before the ASGI request body is consumed +- **AND** no multipart temporary file is created + +#### Scenario: Bounded native transcription remains compatible + +- **WHEN** an authorized `/backend-api/transcribe` request supplies one audio file within 25,000,000 bytes, an optional bounded prompt, and a multipart body within 32 MiB +- **THEN** the service forwards the same audio bytes, filename, content type, and prompt through the existing transcription pipeline + +#### Scenario: Bounded source-model transcription preserves fields + +- **WHEN** an authorized `/v1/audio/transcriptions` request selects a configured model source and all multipart limits are satisfied +- **THEN** the service forwards the audio file and the ordered non-file form fields through the existing source pipeline + +#### Scenario: Declared or streamed transcription body exceeds its limit + +- **WHEN** a usable `Content-Length` exceeds 32 MiB or actual streamed multipart bytes cross 32 MiB +- **THEN** the service returns HTTP 413 with OpenAI error `code = payload_too_large` and `type = invalid_request_error` +- **AND** no usage reservation, account selection, or upstream request occurs + +#### Scenario: Transcription file exceeds its limit + +- **WHEN** the audio file part exceeds 25,000,000 bytes +- **THEN** the service returns HTTP 413 with OpenAI error `code = payload_too_large`, `type = invalid_request_error`, and `param = file` +- **AND** bytes beyond the file limit are not retained in a spool or handler buffer + +#### Scenario: Transcription field resources are bounded + +- **WHEN** a request exceeds 32 text fields, one file part, or 256 KiB in any text part +- **THEN** the service rejects the request with the documented OpenAI-compatible count or byte-limit response +- **AND** it does not invoke transcription route logic + +#### Scenario: Compressed transcription is rejected without prebuffering + +- **GIVEN** the transcription request has passed proxy authorization +- **WHEN** either transcription route declares a non-identity `Content-Encoding` +- **THEN** the service returns HTTP 400 with OpenAI error `code = invalid_request_error` and `type = invalid_request_error` before reading the request body +- **AND** a no-op `identity` encoding is handled as an ordinary multipart request governed by the 32 MiB dedicated body limit + +#### Scenario: Generic ingress does not preempt encoded transcription authorization + +- **GIVEN** a request to either transcription route fails proxy authorization +- **WHEN** it declares a non-identity `Content-Encoding` and a `Content-Length` greater than the generic raw HTTP budget +- **THEN** the existing authentication response is returned instead of a generic HTTP 413 or encoded-body HTTP 400 +- **AND** the request body is not consumed + +#### Scenario: Transcription cleanup preserves transport failures + +- **WHEN** parsing succeeds, fails a limit, encounters malformed multipart, receives a client disconnect, or is cancelled +- **THEN** every created multipart spool is closed +- **AND** disconnect and cancellation are not converted to HTTP 413 + diff --git a/openspec/specs/audit-logging/spec.md b/openspec/specs/audit-logging/spec.md new file mode 100644 index 0000000000..78b5fbcd78 --- /dev/null +++ b/openspec/specs/audit-logging/spec.md @@ -0,0 +1,49 @@ +# audit-logging Specification + +## Purpose +Ownership and shutdown-drain guarantees for asynchronous dashboard audit-log writes, so records are neither lost nor left to leaked tasks. +## Requirements +### Requirement: Asynchronous audit writes remain owned until completion + +The system MUST execute `AuditService.log_async()` writes in tracked background tasks without making the calling request wait for database persistence. Each task MUST remain strongly owned until it finishes, and success, cancellation, or failure MUST remove it from the tracked set. An unexpected task failure MUST be consumed and reported rather than becoming an unobserved task exception. + +#### Scenario: Audit logging remains fire-and-forget + +- **GIVEN** an audit-log database write is blocked +- **WHEN** application code calls `AuditService.log_async()` +- **THEN** the call returns before the database write completes +- **AND** the pending write remains tracked until completion + +#### Scenario: Failed audit task is cleaned up + +- **WHEN** an asynchronous audit task fails unexpectedly +- **THEN** the failure is reported +- **AND** the completed task is removed from the tracked set + +### Requirement: Graceful shutdown drains pending audit writes + +Immediately after the in-flight drain attempt returns, graceful shutdown MUST synchronously close asynchronous audit-task admission before any further shutdown await. An `AuditService.log_async()` call after this cutoff MUST remain non-blocking, MUST report the rejected action, and MUST NOT construct a write coroutine or task. Graceful shutdown MUST wait for audit-log tasks accepted before the cutoff for up to `shutdown_drain_timeout_seconds` before closing shared database resources. The drain MUST include tasks that complete or become visible while task-completion callbacks are running. If the deadline expires, the system MUST report each audit task that did not drain before continuing shutdown. + +#### Scenario: Late audit producer is rejected after in-flight timeout + +- **GIVEN** an HTTP handler remains alive after the in-flight drain timeout +- **AND** graceful shutdown has closed control-plane task admission +- **WHEN** the handler calls `AuditService.log_async()` +- **THEN** the call returns without waiting +- **AND** the rejected action is reported +- **AND** no audit write coroutine or task is created + +#### Scenario: Shutdown preserves a pending audit row + +- **GIVEN** an asynchronous audit write is still pending when graceful shutdown begins +- **WHEN** the write completes within the configured drain timeout +- **THEN** shutdown waits for the write +- **AND** shared database resources remain open until the write finishes + +#### Scenario: Overdue audit write is reported + +- **GIVEN** an asynchronous audit write remains pending for the full configured drain timeout +- **WHEN** graceful shutdown drains audit tasks +- **THEN** the drain reports that task as overdue +- **AND** shutdown is allowed to continue + diff --git a/openspec/specs/chat-completions-compat/context.md b/openspec/specs/chat-completions-compat/context.md index b54250040b..7106aac4b5 100644 --- a/openspec/specs/chat-completions-compat/context.md +++ b/openspec/specs/chat-completions-compat/context.md @@ -19,12 +19,16 @@ See `openspec/specs/chat-completions-compat/spec.md` for normative requirements. - Oversized image data URLs (>8MB) are dropped from user inputs. - Audio input (`input_audio`) is not supported and is rejected. - Built-in Responses tools are preserved only on the Responses-shaped passthrough path; ordinary chat-message payloads keep the narrower chat tool policy. +- Omitted top-level `tools` stay omitted on the mapped Responses payload. `default_factory=list` plus an unconditional `to_responses_request()` write used to synthesize `"tools": []` and mark the field as set, which bypassed the Responses omit path (issue #1184). An explicit client-sent `[]` is still forwarded. - `response_format` is translated to `text.format` with JSON schema validation. ## Failure Modes - **Upstream stream failure:** Emit an error chunk, then terminate with `data: [DONE]`. -- **Non-stream failures:** Return an OpenAI error envelope with 5xx status. +- **Non-stream failures:** Return an OpenAI error envelope. HTTP status follows + the same map as non-stream `/v1/responses` (`429` for `rate_limit_exceeded`, + not a blanket 502). The upstream Responses generator is closed so reservation + finalizers run even when the first collected event is `response.failed`. - **Invalid content types:** Reject with `invalid_request_error`. ## Examples diff --git a/openspec/specs/chat-completions-compat/spec.md b/openspec/specs/chat-completions-compat/spec.md index f30904f0c4..1321844fb8 100644 --- a/openspec/specs/chat-completions-compat/spec.md +++ b/openspec/specs/chat-completions-compat/spec.md @@ -52,6 +52,23 @@ The service MUST map chat messages into the Responses request format by merging - **WHEN** the client sets `tool_choice` to `none`, `auto`, or `required` - **THEN** the service forwards the value consistently in the mapped Responses request +### Requirement: Chat Completions omit unset tools on the mapped Responses payload + +When a `/v1/chat/completions` request omits the top-level `tools` field, the mapped Responses request MUST leave `tools` unset and the forwarded upstream payload MUST NOT include `tools`. An explicit client-sent empty `tools` array MUST still be forwarded as `[]`. + +#### Scenario: Omitted chat tools stay omitted upstream + +- **GIVEN** a Chat Completions request with `messages` and no `tools` field +- **WHEN** the service maps the request to Responses and forwards it +- **THEN** `tools` is absent from the mapped request field set +- **AND** the upstream payload does not include `tools` + +#### Scenario: Explicit empty chat tools stay explicit + +- **GIVEN** a Chat Completions request that sends `"tools": []` +- **WHEN** the service maps the request to Responses +- **THEN** the mapped payload includes `"tools": []` + ### Requirement: Preserve service_tier in Chat Completions mapping When a Chat Completions request includes `service_tier`, the service MUST preserve that field when mapping the request to the internal Responses payload. @@ -106,6 +123,29 @@ When `stream` is `false` or omitted, the service MUST return a single `chat.comp - **WHEN** the upstream indicates a tool call sequence - **THEN** the returned `chat.completion` includes `tool_calls` and a `finish_reason` of `tool_calls` +### Requirement: Non-streaming chat collect closes the upstream generator + +When `stream` is `false` or omitted, `POST /v1/chat/completions` MUST close the upstream Responses generator after collect returns or raises, including when the first consumed event is `response.failed` or `error`. Closing MUST run the generator finalizer so an open API-key reservation is released or settled before the HTTP response is returned. + +#### Scenario: First-event rate limit releases the reservation + +- **WHEN** the startup probe did not consume the stream +- **AND** the first upstream event is `response.failed` with + `code=rate_limit_exceeded` +- **AND** the request reserved API-key usage +- **THEN** the reservation is released before the error response is returned + +### Requirement: Non-streaming chat errors use the Responses status map + +When non-streaming `POST /v1/chat/completions` returns an OpenAI error envelope collected from the upstream Responses stream, the HTTP status MUST match the non-streaming `/v1/responses` mapping for that envelope (`429` for `rate_limit_exceeded`, `503` for unavailable-selection codes, `401`/`400` where that path already maps them). The envelope body MUST remain an OpenAI error object. + +#### Scenario: Collected rate limit is 429 + +- **WHEN** non-streaming chat collect returns + `{ "error": { "code": "rate_limit_exceeded", ... } }` +- **THEN** the HTTP status is `429` +- **AND** the body is that OpenAI error envelope + ### Requirement: response_format mapping If the client sends `response_format`, the service MUST translate it to the Responses `text.format` controls. For `json_schema`, the schema payload MUST be validated and missing `json_schema` MUST result in a 4xx error with an OpenAI error envelope. @@ -142,6 +182,10 @@ For upstream failures or invalid requests, the service MUST return an OpenAI err - **WHEN** the upstream returns a failure during streaming - **THEN** the service emits an error chunk and terminates the stream with `data: [DONE]` +#### Scenario: Non-streaming collected rate limit +- **WHEN** non-streaming collect returns `rate_limit_exceeded` +- **THEN** the service returns that OpenAI error envelope with HTTP 429 + ### Requirement: Drop unknown message-object fields during coercion The service MUST drop unknown keys on a chat message object when coercing the message into a Responses API input message item. Specifically, when a chat message is converted into an `input` message item (role `user` / `assistant` without tool_calls, or the message-content half of an assistant message that also has tool_calls), the emitted item MUST contain exactly the keys `role` and `content`. Other fields on the inbound chat message — the documented but unsupported `name` field, any other standard chat-message field that has no Responses input-item equivalent, and any arbitrary client-supplied key (including keys starting with `_`) — MUST NOT appear on the emitted item. diff --git a/openspec/specs/data-retention/spec.md b/openspec/specs/data-retention/spec.md index 4c72b6e1ee..b47c491cfa 100644 --- a/openspec/specs/data-retention/spec.md +++ b/openspec/specs/data-retention/spec.md @@ -172,3 +172,32 @@ retention MUST NOT run a pass. - **WHEN** the scheduler ticks - **THEN** no retention pass runs +### Requirement: Disabled request-log pruning is explained and presets are non-destructive + +When the effective request-log retention value is `0`, the Settings data retention card SHALL show neutral informational text that request-log pruning is disabled, logs are retained indefinitely, and storage will grow over time, and SHALL offer 30-day and 90-day request-log retention presets. The informational text MUST NOT characterize disabled pruning as unsafe or direct the operator to change it. Activating a preset MUST update only the local request-log retention form value and MUST NOT persist any setting until the operator activates the existing explicit save action. Rendering the information and presets MUST NOT change the stored override or any other retention policy. + +#### Scenario: Effective disabled state shows information and presets + +- **GIVEN** effective request-log retention is `0` +- **WHEN** an operator views the data retention card +- **THEN** the card explains neutrally that request-log pruning is disabled and + logs are retained indefinitely +- **AND** the text notes that storage will grow over time without directing the + operator to change the policy +- **AND** the card offers 30-day and 90-day request-log retention presets +- **AND** no settings update is submitted + +#### Scenario: Preset selection requires explicit save + +- **GIVEN** effective request-log retention is `0` +- **WHEN** an operator activates the 30-day or 90-day preset +- **THEN** the request-log retention form value changes to the selected number +- **AND** no settings update is submitted until the operator activates save +- **AND** usage-history retention remains unchanged + +#### Scenario: Enabled effective policy does not show disabled-state information + +- **GIVEN** effective request-log retention is greater than `0` +- **WHEN** an operator views the data retention card +- **THEN** the disabled-state information and presets are not shown + diff --git a/openspec/specs/database-backends/spec.md b/openspec/specs/database-backends/spec.md index 0d5e16e1a3..24786da2e2 100644 --- a/openspec/specs/database-backends/spec.md +++ b/openspec/specs/database-backends/spec.md @@ -198,7 +198,6 @@ advisory-lock behavior. - **WHEN** an account status transition is persisted - **THEN** the write executes inside the shared SQLite writer section - ### Requirement: Telemetry write transactions relax commit durability on PostgreSQL A write transaction is classified as a **telemetry write** when it only appends observability rows whose loss on a database-server crash changes nothing about accounting semantics: request-log inserts (`request_logs`) and usage-history appends (`usage_history`, `additional_usage_history`). API-key usage-reservation accounting is explicitly NOT telemetry (see the reservation-durability requirement below). @@ -266,3 +265,113 @@ Rationale: the "crash loses the in-flight request anyway" argument that justifie - **GIVEN** a PostgreSQL backend holding a stale usage reservation (heartbeat stopped or past the maximum age) - **WHEN** the stale-reservation release settles a batch (status flip to `released` plus its limit-counter adjustments) - **THEN** no batch transaction executes `SET LOCAL synchronous_commit = off` + +### Requirement: Asyncpg PostgreSQL sessions pin time zone to UTC + +When `database_url` resolves to a PostgreSQL backend through the asyncpg driver, the application MUST configure each SQLAlchemy async engine connection with a database session time zone of `UTC`. + +This requirement applies to the request-path `engine`, the optional background +`_background_engine`, and any app-created PostgreSQL async engine that uses the +shared PostgreSQL engine kwargs helper. + +#### Scenario: Asyncpg sessions ignore non-UTC database defaults + +- **GIVEN** `database_url` uses `postgresql+asyncpg://` +- **AND** the PostgreSQL role, database, container, or server default time zone + is not UTC +- **WHEN** the application opens a new asyncpg connection through its engine + configuration +- **THEN** `SHOW TIME ZONE` on that connection reports `UTC` +- **AND** naive UTC datetimes written by the application are interpreted as UTC + before PostgreSQL stores them in `timestamptz` columns + +#### Scenario: SQLite backends are not affected + +- **GIVEN** `database_url` resolves to a SQLite backend +- **WHEN** the application creates its async engine +- **THEN** PostgreSQL asyncpg `server_settings` are not configured +- **AND** existing SQLite PRAGMAs, busy timeout, and pooling behavior remain + unchanged + +### Requirement: PostgreSQL connection budgets include every pooled engine + +The application SHALL define its per-worker PostgreSQL connection capacity as the configured per-engine pool capacity multiplied by the declared set of independently pooled engine roles. The request-path and background-task engine creation paths MUST each use the shared role-aware PostgreSQL engine factory, and the engine-count budget MUST be derived from those declared roles. The owned server launcher MUST run the supported one worker per replica explicitly, rather than allowing `WEB_CONCURRENCY` to multiply worker processes and their pools. + +#### Scenario: One replica reaches configured pool capacity + +- **WHEN** both declared PostgreSQL engine roles in one application worker reach `database_pool_size + database_max_overflow` +- **THEN** the worker's aggregate application connection capacity is `2 * (database_pool_size + database_max_overflow)` +- **AND** both engines were created through the role-aware factory counted by that formula + +#### Scenario: WEB_CONCURRENCY cannot multiply owned-launcher pools + +- **GIVEN** `WEB_CONCURRENCY` is greater than 1 +- **WHEN** the application starts through the owned `app.cli` launcher used by Helm +- **THEN** the launcher explicitly starts one Uvicorn worker +- **AND** the replica creates only the request-path and background-task pools +- **AND** operators MUST scale supported deployments through replicas rather than custom multi-worker launchers + +#### Scenario: Test database disables pooling + +- **WHEN** `CODEX_LB_TEST_DATABASE_URL` selects `NullPool` +- **THEN** pool sizing controls and the production pooled-engine budget do not apply to that test engine + +### Requirement: SQLAlchemy-rendered Windows SQLite paths are percent-decoded before opening + +When a SQLite database URL is converted to a filesystem path for direct filesystem use (e.g. startup directory creation, startup integrity checks, migration locks, or the usage repository's read-only helper), a path that matches a recognizable SQLAlchemy-rendered Windows form — an encoded drive marker (`%3A` followed by an encoded or raw path separator) or an encoded UNC prefix (`%5C%5C`) — MUST be percent-decoded before being handed to the filesystem. SQLAlchemy's `URL.render_as_string()` percent-encodes a Windows-style default path (`C:\Users\...` -> `C%3A%5CUsers%5C...`); without decoding, the literal escaped string either fails to open with "unable to open database file" or creates a stray 0-byte database next to the current working directory, which breaks account/usage reads with `no such table`. + +Paths that do NOT match those rendered Windows forms MUST be preserved literally. Settings builds the default SQLite URL directly from the configured data directory without URL-encoding it, so a percent sequence in a POSIX or raw Windows path (e.g. `/var/lib/codex%20lb/store.db`) names a real directory and MUST NOT be rewritten by decoding. + +#### Scenario: Windows default path resolves to the real file + +- **GIVEN** the default SQLite URL on Windows (`sqlite+aiosqlite:///C:\Users\...\store.db`) +- **WHEN** `URL.render_as_string()` percent-encodes it into `sqlite:///C%3A%5CUsers%5C...%5Cstore.db` +- **AND** the path is extracted and decoded +- **THEN** `sqlite3.connect()` receives `C:\Users\...\store.db` (the real file), not the percent-escaped literal + +#### Scenario: Encoded drive with URL slash separators resolves to the real file + +- **GIVEN** a Windows SQLite URL with an encoded drive colon and normal URL path separators (`sqlite:///C%3A/Users/me/.codex-lb/store.db`) +- **WHEN** the path is extracted and decoded +- **THEN** the filesystem path is `C:/Users/me/.codex-lb/store.db`, not the literal `C%3A/Users/me/.codex-lb/store.db` + +#### Scenario: Startup uses the decoded SQLite path + +- **GIVEN** a percent-encoded SQLite file URL whose decoded parent directory differs from the percent-literal parent +- **WHEN** `init_db()` prepares the SQLite directory and runs the startup integrity check +- **THEN** the decoded parent directory is created +- **AND** the integrity check receives the decoded database path + +#### Scenario: URL normalization preserves decoded Windows path characters + +- **GIVEN** an encoded Windows SQLite URL whose decoded database path contains spaces, literal `%`, or `#` +- **WHEN** the URL is normalized for SQLAlchemy consumers +- **THEN** the returned URL contains the real decoded Windows filesystem path +- **AND** filesystem extraction from that normalized URL returns the same decoded path +- **AND** a raw Windows URL containing a literal percent sequence such as `%23` is not decoded unless it first matched a SQLAlchemy-rendered encoded Windows form + +#### Scenario: Literal percent sequences in POSIX paths are preserved + +- **GIVEN** a POSIX SQLite URL whose path contains a literal percent sequence (`sqlite+aiosqlite:////var/lib/codex%20lb/store.db`) built directly from the configured data directory +- **WHEN** the path is extracted for filesystem use or the URL is normalized +- **THEN** the filesystem path remains `/var/lib/codex%20lb/store.db` and the URL is unchanged (the sequence is not decoded to a space) + +#### Scenario: Normalized UNC paths keep fragment characters + +- **GIVEN** an encoded UNC SQLite URL whose decoded share path contains a legal `#` character (`sqlite:///%5C%5Cserver%5Cshare%23x%5Cstore.db`) +- **WHEN** the URL is normalized and the filesystem path is then extracted from the normalized URL +- **THEN** the extracted path is `\\server\share#x\store.db` +- **AND** the path is not truncated at the `#` as if it were a URL fragment separator + +#### Scenario: POSIX paths are unchanged + +- **GIVEN** a POSIX-style SQLite URL (`sqlite+aiosqlite:///var/lib/codex-lb/store.db`) +- **WHEN** the path is extracted and decoded +- **THEN** the result is identical to the input path (no `%` to decode; behavior is a no-op) + +#### Scenario: In-memory databases are not treated as file paths + +- **GIVEN** a `:memory:` SQLite URL +- **WHEN** the path is extracted +- **THEN** no filesystem path is returned and no file is created + diff --git a/openspec/specs/database-migrations/spec.md b/openspec/specs/database-migrations/spec.md index 0f72c33a1d..dd8eebf530 100644 --- a/openspec/specs/database-migrations/spec.md +++ b/openspec/specs/database-migrations/spec.md @@ -240,3 +240,95 @@ Migration state inspection SHALL classify `alembic_version` revisions that are n - **WHEN** the upgrade runs - **THEN** it fails with the ahead-specific guidance rather than a generic unsupported-revision remap error +### Requirement: Migration CLI distinguishes omitted and empty targets + +The `app.db.migrate` / `codex-lb-db` CLI SHALL use the settings-derived database URL only when `--db-url` is omitted. If `--db-url` is explicitly supplied as an exact empty string, the CLI MUST terminate with an argument error before resolving or opening a settings-derived database target. This validation MUST apply to every supported migration subcommand: `upgrade`, `current`, `check`, `wait-for-head`, `wait-for-connection`, and `stamp`. + +#### Scenario: Explicit empty target is rejected before side effects + +- **GIVEN** settings would resolve a valid database target +- **WHEN** any supported migration subcommand is invoked with `--db-url ""` +- **THEN** the CLI exits nonzero with an argument-validation error +- **AND** it does not select, connect to, create, inspect, migrate, or stamp the settings-derived target + +#### Scenario: Omitted target retains the settings fallback + +- **GIVEN** settings resolve a valid database target +- **WHEN** a supported migration subcommand is invoked without `--db-url` +- **THEN** the CLI uses the settings-derived database target + +### Requirement: Capability lineage uses one additive opaque-marker table + +The migration MUST descend from the current single Alembic head and create one +`capability_lineage_markers` table containing only an opaque SHA-256 marker +primary key plus creation and last-seen timestamps. It MUST NOT modify existing +sticky-session, account, usage, quota, request-log, or durable-bridge columns or +foreign keys, and MUST NOT backfill historical rows. + +#### Scenario: Upgrade creates an empty marker table +- **WHEN** a database at the previous head upgrades to the new head +- **THEN** the marker table exists with its primary-key uniqueness contract +- **AND** no existing application table is scanned or rewritten for backfill + +#### Scenario: Downgrade removes only the marker table +- **WHEN** the migration is downgraded to its parent revision +- **THEN** only `capability_lineage_markers` is removed +- **AND** existing application data remains unchanged + +#### Scenario: Migration graph remains single-head +- **WHEN** the repository migration graph is inspected after this change +- **THEN** it has exactly one head containing the marker-table revision + +### Requirement: SQLite maintenance releases file handles before filesystem mutation + +Synchronous SQLite maintenance operations MUST explicitly close every native +connection they open after completing or rolling back its transaction. A +pre-migration backup MUST release its source and destination connections before +retention deletes an older snapshot. Recovery with `--replace` MUST release +connections used for integrity checking, dump export, and dump import before it +renames either the source database or recovered output. Correctness MUST NOT +depend on garbage collection or interpreter object-finalization timing. + +#### Scenario: Backup retention deletes an old snapshot on Windows + +- **GIVEN** SQLite pre-migration backups have reached their retention limit +- **WHEN** a new online snapshot is complete and retention deletes the oldest + snapshot +- **THEN** every connection opened for the completed snapshot is explicitly + closed before deletion +- **AND** backup rotation succeeds on platforms that prohibit deleting an open + database file + +#### Scenario: Recovery replaces a database on Windows + +- **GIVEN** a file-backed SQLite database is recovered through the CLI with + `--replace` +- **WHEN** dump export and import complete +- **THEN** the integrity-check, source, and output connections are explicitly + closed before either database file is renamed +- **AND** the original is preserved under the corrupt-backup name while the + recovered database is moved into the original path + +### Requirement: Alembic Config escapes percent characters in the SQLAlchemy URL + +When the application builds an Alembic `Config` for migration inspection or upgrade (`_build_alembic_config`), any `%` in the SQLAlchemy URL MUST be escaped to `%%` before being stored via `set_main_option`. Alembic stores option values in a `configparser` using `BasicInterpolation`, which treats a bare `%` as interpolation syntax; a percent-encoded Windows path (`C%3A%5CUsers%5C...`) otherwise raises `ValueError: invalid interpolation syntax` during startup. `get_main_option` decodes `%%` back to `%`, so the URL handed to SQLAlchemy is unchanged. + +#### Scenario: Windows path does not crash migration inspection + +- **GIVEN** the default SQLite URL on Windows, percent-encoded by `URL.render_as_string()` into `sqlite:///C%3A%5CUsers%5C...%5Cstore.db` +- **WHEN** the Alembic `Config` is built for migration inspection +- **THEN** no `ValueError: invalid interpolation syntax` is raised +- **AND** `get_main_option("sqlalchemy.url")` returns the normalized sync URL whose path is the decoded Windows filesystem path (`sqlite:///C:\Users\...\store.db`), because `to_sync_database_url` normalizes recognizable SQLAlchemy-rendered Windows SQLite URLs before the value is stored in the Alembic `Config` + +#### Scenario: Round-trip preserves an already-encoded percent + +- **GIVEN** a path that already contains a percent-encoded `%` (rendered as `%25`) +- **WHEN** the escape turns it into `%%25` and `get_main_option` decodes it +- **THEN** the value SQLAlchemy receives decodes back to `%25`, i.e. the original URL is preserved exactly + +#### Scenario: Non-Windows URLs are unaffected + +- **GIVEN** a SQLite or PostgreSQL URL whose path contains no `%` +- **WHEN** the escape and decode round-trip is applied +- **THEN** the URL is unchanged and migration behavior is identical to before + diff --git a/openspec/specs/date-display-format/spec.md b/openspec/specs/date-display-format/spec.md new file mode 100644 index 0000000000..9e76c8b5b8 --- /dev/null +++ b/openspec/specs/date-display-format/spec.md @@ -0,0 +1,96 @@ +# date-display-format Specification + +## Purpose +Operator-selectable dashboard date/time rendering (localStorage preference, ISO 8601 contract) without disturbing chart axis formats. +## Requirements +### Requirement: Date format preference is stored in localStorage + +The system SHALL persist a date display format preference in localStorage under the key `codex-lb-date-display-format`. The valid values SHALL be `"default"` and `"iso8601"`. The default value SHALL be `"default"`. The preference SHALL apply only to read-only date/time presentation text. + +#### Scenario: No stored preference + +- **WHEN** the preference has never been saved +- **THEN** the system SHALL use `"default"` format + +#### Scenario: User selects ISO 8601 + +- **WHEN** the user selects "ISO 8601" as the date format +- **THEN** the system SHALL persist `"iso8601"` to localStorage under `codex-lb-date-display-format` +- **AND** all applicable read-only date/time presentation text SHALL use ISO 8601 formatting + +#### Scenario: User switches back to Default + +- **WHEN** the user selects "Default" as the date format +- **THEN** the system SHALL persist `"default"` to localStorage +- **AND** all applicable read-only date/time presentation text SHALL revert to locale-dependent formatting + +#### Scenario: Interactive date and time controls retain their own format + +- **GIVEN** a date or time is shown within an interactive control used to enter, edit, select, or filter a value +- **WHEN** the user switches between "Default" and "ISO 8601" +- **THEN** inputs, calendars, date pickers, selectors, and equivalent interactive controls SHALL retain the format provided by their component or browser +- **AND** the preference SHALL NOT change the control's value representation or interaction behavior + +#### Scenario: Verbatim API and data representations remain unchanged + +- **GIVEN** a date or timestamp appears inside a verbatim API or data representation +- **WHEN** the user switches between "Default" and "ISO 8601" +- **THEN** raw JSON, request and response payloads, metadata, copied values, filenames, downloads, and exports SHALL preserve their source representation +- **AND** the preference SHALL NOT rewrite those values + +#### Scenario: Read-only presentation text follows the selected format + +- **GIVEN** a date or timestamp is presented as non-interactive text in a table cell, detail field, status, or informational label +- **WHEN** the user switches between "Default" and "ISO 8601" +- **THEN** the rendered text SHALL update immediately to the selected format + +#### Scenario: Daily report table follows the selected format + +- **GIVEN** the daily report breakdown table is mounted +- **WHEN** the user switches between "Default" and "ISO 8601" +- **THEN** the table's Day column SHALL update immediately +- **AND** Default SHALL use locale-dependent formatting +- **AND** ISO 8601 SHALL use `YYYY-MM-DD` formatting + +#### Scenario: Quota planner decision peak follows the selected format + +- **GIVEN** a quota planner decision presents `target_peak_at` as a read-only Peak label +- **WHEN** the user switches between "Default" and "ISO 8601" +- **THEN** the Peak label SHALL update immediately to the selected format +- **AND** the underlying decision details value SHALL remain unchanged + +### Requirement: ISO 8601 format spec for date/time rendering + +When the date display format is `"iso8601"`, the `formatTimeLong` function SHALL return `{ time: "HH:MM:SS", date: "YYYY-MM-DD" }` for read-only presentation text where: +- `time` is always the clock-time portion in 24-hour format (2-digit hour, 2-digit minute, 2-digit second, colon-separated) +- `date` is always the calendar-date portion in ISO 8601 format (4-digit year, 2-digit month, 2-digit day, hyphen-separated) +The semantic meaning of these fields SHALL remain stable across date display formats. Rendered date/time surfaces that display ISO values SHALL order the `date` value before the `time` value. + +#### Scenario: ISO 8601 rendering of a UTC timestamp + +- **GIVEN** the date display format is `"iso8601"` +- **WHEN** formatting a timestamp corresponding to August 9, 2026 at 14:30:45 local time +- **THEN** `formatTimeLong` SHALL return `{ time: "14:30:45", date: "2026-08-09" }` + +#### Scenario: Default rendering unchanged + +- **GIVEN** the date display format is `"default"` +- **WHEN** formatting any timestamp +- **THEN** `formatTimeLong` SHALL return locale-dependent values as before (unchanged behavior) + +### Requirement: Chart axes are not affected by date format + +The date display format setting SHALL NOT affect Recharts x-axis tick formatting or data preparation. Charts (account trend, API trend, reports) SHALL continue to use their own x-axis formats regardless of the selected date display format. + +#### Scenario: ISO 8601 setting does not change chart tooltips + +- **GIVEN** the date display format is `"iso8601"` +- **WHEN** hovering over a point on any chart +- **THEN** the tooltip heading SHALL use `formatChartDateTime` as before (locale-dependent short month + day + time) + +#### Scenario: Reports chart x-axis unchanged + +- **GIVEN** the date display format is `"iso8601"` +- **WHEN** rendering a reports chart (tokens per day, cost per day, etc.) +- **THEN** the x-axis ticks SHALL remain `MM-DD` strings (e.g., `"08-09"`) + diff --git a/openspec/specs/deployment-installation/context.md b/openspec/specs/deployment-installation/context.md index a91bbcdb43..d26484a1e1 100644 --- a/openspec/specs/deployment-installation/context.md +++ b/openspec/specs/deployment-installation/context.md @@ -10,6 +10,48 @@ fixed, and how removed settings are retired. See `openspec/specs/deployment-installation/spec.md` for normative requirements. +## Timeout Invariant Linter Scope + +The timeout invariant linter is a startup `Settings` guardrail. Strict mode is +an opt-in startup or CI failure path for violating startup configuration, not a +general runtime timeout validator. + +Validated inputs: + +- The `Settings` object materialized at startup. +- Explicitly imported code constants used by the two constant-backed rules: + model-registry refresh cadence and durable HTTP bridge retry-circuit TTL. + +Known non-goals and follow-ups: + +- Per-request `ContextVar` overrides are not revalidated. Current anchors: + `app/core/clients/proxy.py:3450-3467`, + `app/modules/proxy/_service/streaming/helpers.py:861-868`, + `app/modules/proxy/_service/compact.py:727-738`, + `app/modules/proxy/_service/transcribe.py:230-232`, + `app/core/clients/files.py:77-90`, and + `app/modules/proxy/service.py:1464-1478`. +- Runtime clamps and derived effective values are not fully modeled. Current + anchors: `app/core/clients/proxy.py:1049-1088`, + `app/core/auth/refresh.py:391-395`, and + `app/modules/proxy/load_balancer.py:1846-1856`. +- Runtime DB, API-key, and model-source settings can affect timeout-bearing + paths without startup revalidation. Current anchors: + `app/core/config/settings_cache.py:22-36`, + `app/modules/settings/api.py:547-710`, + `app/modules/proxy/_service/streaming/retry.py:153-165`, and + `app/modules/model_sources/forwarding.py:112-221`. + +Example: `python -m app.core.timeout_invariants --strict` validates the +startup `Settings` view and exits nonzero when any enforced rule fails. +Running the same command without `--strict` reports violations but exits zero, +matching the default startup behavior. + +`CODEX_LB_TIMEOUT_INVARIANT_VALIDATION_STRICT` is intentionally a setting +rather than a hard default because existing deployments may carry legacy timeout +values that deserve CRITICAL diagnostics first, not surprise startup refusal. +The default remains non-strict; operators and CI opt into fail-fast behavior. + ## Helm termination-grace upgrade contract The graceful-shutdown chart adds a render-time guard: diff --git a/openspec/specs/deployment-installation/spec.md b/openspec/specs/deployment-installation/spec.md index b112b3f1ba..c8b57f77dd 100644 --- a/openspec/specs/deployment-installation/spec.md +++ b/openspec/specs/deployment-installation/spec.md @@ -405,3 +405,173 @@ enable switches. - **WHEN** Codex bridge requests are served - **THEN** no session prewarm is attempted and visible requests record `prewarm_status=not_applicable` + +### Requirement: Response-create dump directory is bounded without configuration + +The oversized response-create dump directory under `/debug/response-create-dumps` MUST be bounded on the base install path with no operator configuration. When the service captures an oversized `response.create` payload, it MUST NOT write a new dump if a dump for the same payload fingerprint is already stored, and after storing a dump it MUST remove the oldest stored dumps so that at most a fixed number of dump pairs remain. Each dump is a pair of a gzipped payload file and a meta file that MUST be added and removed together. Suppressing a duplicate MUST remain operator-visible in the logs, because the recurrence signal is the reason the dump path exists. + +#### Scenario: Repeated identical payloads are stored once + +- **GIVEN** an oversized `response.create` payload has already been dumped +- **WHEN** a retry of the byte-identical payload is dumped again +- **THEN** no additional dump pair is written +- **AND** the originally stored dump pair is retained +- **AND** the suppressed duplicate is logged with its payload fingerprint and the path of the existing dump + +#### Scenario: Distinct payloads are stored separately + +- **GIVEN** an oversized `response.create` payload has already been dumped +- **WHEN** a different oversized payload is dumped +- **THEN** a separate dump pair is written for it + +#### Scenario: Oldest dumps are pruned once the directory is full + +- **GIVEN** the dump directory already holds the maximum number of dump pairs +- **WHEN** a dump for a new payload is written +- **THEN** the oldest dump pairs are removed so the maximum is not exceeded +- **AND** each removed payload file has its meta file removed with it +- **AND** the newly written dump pair is retained + +#### Scenario: Dump retention needs no setting + +- **GIVEN** a default installation with no dump-related configuration +- **WHEN** oversized response-create dumps are captured over time +- **THEN** duplicate suppression and pruning apply +- **AND** no `CODEX_LB_*` setting is required to bound the directory + +### Requirement: External secret references support provider-native layouts + +When `externalSecrets.enabled=true`, the Helm chart MUST render an +`external-secrets.io/v1` ExternalSecret. The database URL and encryption key +MUST each accept an independent remote key and an optional JSON property. An +empty remote key MUST default to the release fullname, and the default +properties MUST preserve the existing `database-url` and `encryption-key` JSON +layout. Explicitly nulled remote reference overrides MUST render the default +layout instead of failing the template. + +#### Scenario: Existing JSON secret layout remains the default + +- **WHEN** external secrets mode is enabled without remote reference overrides +- **THEN** both target keys read from the remote secret named after the release +- **AND** they extract the `database-url` and `encryption-key` JSON properties +- **AND** the rendered ExternalSecret uses `external-secrets.io/v1` + +#### Scenario: Individual remote secrets need no JSON property + +- **WHEN** an operator configures separate absolute remote keys for the database URL and encryption key +- **AND** leaves both property values empty +- **THEN** each target key reads the complete value of its configured remote secret +- **AND** the rendered remote references omit `property` + +#### Scenario: Nulled overrides fall back to the default layout + +- **WHEN** an operator explicitly nulls `externalSecrets.remoteRefs` or one of its subtrees +- **THEN** rendering succeeds +- **AND** the affected target keys use the release fullname and their default JSON properties + +### Requirement: Helm PostgreSQL capacity guidance accounts for both application pools + +Helm sizing documentation and production-oriented values SHALL calculate maximum application PostgreSQL connections as `(databasePoolSize + databaseMaxOverflow) * 2 pooled engines * 1 supported worker * maxReplicas`. Values described as fitting PostgreSQL's default `max_connections=100` MUST reserve at least 20 raw server slots for PostgreSQL-reserved connections, the migration path's two-connection peak, administration, and transient non-application clients. + +#### Scenario: Default chart reaches its HPA ceiling + +- **WHEN** the default chart scales to `autoscaling.maxReplicas` +- **THEN** both application pools across all replicas require no more than 80 PostgreSQL connections +- **AND** at least 20 raw server slots remain outside the application-pool budget + +#### Scenario: Production overlay reaches its HPA ceiling + +- **WHEN** `values-prod.yaml` scales to `autoscaling.maxReplicas` +- **THEN** both application pools across all replicas require no more than 80 PostgreSQL connections +- **AND** at least 20 raw server slots remain available for PostgreSQL reservations, migrations, administration, and transient non-application clients + +### Requirement: Helm Grafana dashboard titles are configurable + +The Helm chart MUST allow operators to override the titles of packaged Grafana +dashboards by JSON filename. The default values MUST preserve the packaged +dashboard titles. + +#### Scenario: Operator uses concise titles in a folder hierarchy + +- **GIVEN** Grafana dashboard provisioning is enabled +- **AND** title overrides map `codex-lb.json` to `Overview` and + `ttft-breakdown.json` to `TTFT Breakdown` +- **WHEN** the chart renders the Grafana dashboard ConfigMap +- **THEN** each dashboard JSON document contains its configured title +- **AND** dashboard UIDs and all panel definitions remain unchanged + +#### Scenario: Default titles remain compatible + +- **GIVEN** Grafana dashboard provisioning is enabled +- **AND** the operator does not customize dashboard titles +- **WHEN** the chart renders the Grafana dashboard ConfigMap +- **THEN** the overview title remains `codex-lb` +- **AND** the TTFT title remains `codex-lb TTFT Breakdown` +- **AND** each ConfigMap value remains byte-identical to the chart's raw-file rendering + +### Requirement: Helm preStop shares the application drain deadline + +The Helm lifecycle hook MUST start local drain and poll its strict status. The configured routing dwell and application deadline MUST be measured from Python preStop-helper start. The hook MUST convey its helper-anchored absolute monotonic drain deadline to the loopback drain-start endpoint; that deadline-bearing request MUST commit the one-way process barrier. The application MUST reject non-finite values, clamp the supplied deadline so it cannot exceed the configured application timeout measured from receipt, and return the effective committed absolute deadline. The hook MUST validate that response and use the earlier of its local and returned deadlines. Local drain-start request latency or an earlier process deadline MUST therefore consume that single absolute budget rather than create another period. The hook MUST exit once the dwell has elapsed with `draining=true` and `in_flight=0`, or when the effective application drain deadline is exhausted. It MUST NOT add a second fixed drain period. A start, status, or status-schema failure MUST end the hook promptly so kubelet can deliver SIGTERM as the fallback, without rolling back a barrier already accepted by the application. Kubernetes termination grace MUST be documented as beginning before helper launch, with exec/Python launch latency consuming the hard grace but not restarting or shortening the helper-anchored application budget. + +#### Scenario: Routing dwell completes with no in-flight work + +- **WHEN** the Python preStop helper starts the routing dwell and status reports zero in-flight work +- **THEN** the hook waits through the routing dwell measured from helper start +- **AND** the loopback drain-start request establishes the helper-start-anchored application deadline +- **AND** local drain-start request latency does not restart that dwell +- **AND** exits without waiting through the rest of the drain timeout + +#### Scenario: Drain-start request cannot extend the deadline + +- **WHEN** the loopback drain-start request reaches the application after helper start +- **THEN** the application uses no deadline later than the hook's supplied absolute deadline +- **AND** clamps that value to no later than its configured timeout from receipt +- **AND** commits the process barrier and returns the effective deadline +- **AND** the hook bounds all later polling by that returned deadline +- **AND** rejects a non-finite supplied deadline + +#### Scenario: Work remains after routing dwell + +- **WHEN** routing dwell has elapsed and status still reports positive `in_flight` +- **THEN** the hook continues polling until `in_flight=0` or the shared deadline + +#### Scenario: Drain start or status fails + +- **WHEN** the local drain start request, status request, or status schema fails +- **THEN** preStop exits promptly with failure +- **AND** it does not blindly sleep through another timeout + +#### Scenario: Helm timing values are unsafe + +- **WHEN** `config.shutdownDrainTimeoutSeconds` is shorter than `preStopSleepSeconds` +- **OR** `terminationGracePeriodSeconds` is shorter than `config.shutdownDrainTimeoutSeconds + 32` +- **THEN** chart rendering fails with a helpful timing-contract error + +#### Scenario: Operator reads shutdown documentation + +- **WHEN** an operator inspects Helm shutdown tuning +- **THEN** documentation states that preStop and SIGTERM share one application deadline +- **AND** distinguishes the earlier Kubernetes hard-grace start from the Python helper's application-deadline start +- **AND** uses the nested `config.shutdownDrainTimeoutSeconds` values key +- **AND** warns that an old or custom `terminationGracePeriodSeconds` from a values file, `--set`, or `--reuse-values` below `config.shutdownDrainTimeoutSeconds + 32` makes Helm rendering fail before resources are applied +- **AND** states that the minimum is the configured drain timeout plus 32 seconds, is 62 seconds at the default 30-second drain timeout, and that the chart default is 65 seconds +- **AND** directs the operator to remove the override or raise it to at least the computed minimum before installing or upgrading +- **AND** states that omitting the key under `--reuse-values` retains the stored low value, so that path must set at least the computed minimum explicitly, while adopting the chart default requires an intentional non-reuse or `--reset-values` upgrade with the key absent + +### Requirement: Shipped launch paths use the pre-connection drain server + +Every shipped or documented launch path for the main application MUST delegate to the project CLI so direct SIGTERM commits the application drain barrier before Uvicorn closes connections. Development Compose MUST preserve source-watch behavior without replacing the project server with Uvicorn's reload supervisor. + +#### Scenario: Development Compose watches application source + +- **WHEN** the development Compose service is started with watch enabled +- **THEN** it launches the main application through `python -m app.cli` +- **AND** an application source sync restarts that service +- **AND** it does not launch direct Uvicorn reload + +#### Scenario: Operator follows a shipped local command + +- **WHEN** an operator follows a repository-documented command for the main application +- **THEN** that command delegates to `app.cli` +- **AND** direct SIGTERM reaches the pre-connection drain server + diff --git a/openspec/specs/deployment-networking/spec.md b/openspec/specs/deployment-networking/spec.md index 3d40f98c32..65eb698207 100644 --- a/openspec/specs/deployment-networking/spec.md +++ b/openspec/specs/deployment-networking/spec.md @@ -98,3 +98,79 @@ The default responses-ingress sticky mechanism MUST NOT rely on `nginx.ingress.k - **WHEN** the operator sets a non-empty `ingress.responses.nginx.configurationSnippet` - **THEN** the `configuration-snippet` annotation renders with the configured content + +### Requirement: Helm Gateway API routes support rule-level matches and filters + +The Helm chart MUST allow operators to configure an ordered list of HTTPRoute +rules containing Gateway API `matches` and `filters`. The chart MUST attach the +codex-lb Service backend to every configured rule. The feature MUST be optional +and preserve the existing backend-only catch-all rule when no rules are set. + +#### Scenario: Paths use different Gateway filters + +- **GIVEN** `gatewayApi.enabled=true` +- **AND** `gatewayApi.rules` contains an unfiltered API rule matching `/v1`, + `/backend-api/codex`, `/backend-api/wham`, `/backend-api/transcribe`, + `/backend-api/files`, and `/api/codex`, followed by a filtered `/` catch-all + rule +- **WHEN** the chart renders its HTTPRoute +- **THEN** both rules retain their configured matches in order +- **AND** only the catch-all rule contains the configured filter +- **AND** both rules target the chart-managed codex-lb Service and port +- **AND** WHAM identity discovery, file-upload, and Codex usage/reset-credit + paths retain their own caller-authentication contracts instead of traversing + the dashboard filter + +#### Scenario: Empty rule configuration preserves the default route + +- **GIVEN** `gatewayApi.enabled=true` +- **AND** `gatewayApi.rules` is empty +- **WHEN** the chart renders its HTTPRoute +- **THEN** it contains one backend-only rule targeting the chart-managed + codex-lb Service and port + +### Requirement: Helm chart can create an application-specific Gateway + +The Helm chart MUST allow operators to render a Gateway API `Gateway` +dedicated to the release in the release namespace instead of attaching to a +pre-existing shared Gateway. The mode MUST be optional and default off, +preserving the existing `gatewayApi.parentRefs` attachment. When enabled, the +chart MUST require an operator-supplied GatewayClass name, MUST default the +Gateway to a single HTTP listener on port 80 while honoring operator-defined +listeners verbatim, and MUST attach the chart-managed HTTPRoute to the +chart-managed Gateway while ignoring `gatewayApi.parentRefs`. + +#### Scenario: Chart-managed Gateway with default listener + +- **GIVEN** `gatewayApi.enabled=true` +- **AND** `gatewayApi.gateway.create=true` with a GatewayClass name +- **WHEN** the chart renders its Gateway API resources +- **THEN** a Gateway named after the release renders in the release namespace + with the configured GatewayClass and one HTTP listener on port 80 +- **AND** the HTTPRoute's only parent reference is the chart-managed Gateway + +#### Scenario: Operator-defined listeners + +- **GIVEN** `gatewayApi.gateway.create=true` with a GatewayClass name +- **AND** `gatewayApi.gateway.listeners` contains an HTTPS listener with TLS + configuration +- **WHEN** the chart renders the Gateway +- **THEN** the configured listeners replace the default HTTP listener verbatim + +#### Scenario: Missing GatewayClass name fails rendering + +- **GIVEN** `gatewayApi.gateway.create=true` +- **AND** `gatewayApi.gateway.gatewayClassName` is empty +- **WHEN** the chart renders +- **THEN** rendering fails with an error naming + `gatewayApi.gateway.gatewayClassName` + +#### Scenario: Default configuration keeps existing Gateway attachment + +- **GIVEN** `gatewayApi.enabled=true` +- **AND** `gatewayApi.gateway.create` is unset +- **WHEN** the chart renders its Gateway API resources +- **THEN** no Gateway resource renders +- **AND** the HTTPRoute attaches to the operator-supplied + `gatewayApi.parentRefs` + diff --git a/openspec/specs/files-upload-protocol/spec.md b/openspec/specs/files-upload-protocol/spec.md index aee0895211..206d3f6a28 100644 --- a/openspec/specs/files-upload-protocol/spec.md +++ b/openspec/specs/files-upload-protocol/spec.md @@ -54,6 +54,30 @@ The system SHALL expose `POST /backend-api/files/{file_id}/uploaded` as an authe - **WHEN** upstream returns 404 for an unknown `file_id` - **THEN** the service returns HTTP 404 with an OpenAI-format error envelope +### Requirement: File finalize uses durable replica-shared ownership + +When `POST /backend-api/files/{file_id}/uploaded` references a live durable file pin, the service MUST resolve that pin from the shared database and route finalization only through the owning account. The owner decision MUST NOT use a process-local cache. Expiry, reclaim, and cleanup MUST use database-authoritative time. If durable owner resolution fails, the service MUST fail closed before selecting or invoking an unpinned fallback account. + +#### Scenario: another replica finalizes through the durable owner + +- **GIVEN** one replica registered `file_xyz` through `account_a` +- **WHEN** another replica handles `POST /backend-api/files/file_xyz/uploaded` +- **THEN** it MUST resolve the shared durable pin +- **AND** it MUST finalize only through `account_a` + +#### Scenario: finalize owner lookup failure does not fall back + +- **GIVEN** `file_xyz` requires a durable owner decision +- **WHEN** the shared database lookup fails +- **THEN** finalization MUST fail before any unpinned account selection or upstream invocation + +#### Scenario: an expired identifier can be reclaimed using database time + +- **GIVEN** the durable pin for `file_xyz` has expired according to the database clock +- **WHEN** a later upload claims `file_xyz` through `account_b` +- **THEN** the durable owner MUST become `account_b` +- **AND** every replica's next finalize decision MUST resolve `account_b` + ### Requirement: File proxy routes share account-selection and request-log plumbing File proxy routes MUST select an upstream account using the same load-balancer / freshness / 401-retry pattern as `/backend-api/transcribe`, and MUST persist a request-log entry on every attempt. Log entries MUST use synthetic model identifiers `files-create` (for `POST /backend-api/files`) and `files-finalize` (for `POST /backend-api/files/{file_id}/uploaded`) so dashboard request-log queries can filter file activity. Transport MUST be recorded as HTTP. File requests MUST NOT count against per-model API key limits unless the API key explicitly allows the synthetic model identifiers. diff --git a/openspec/specs/fleet-summary/spec.md b/openspec/specs/fleet-summary/spec.md index d4f7ef2955..5689576ce4 100644 --- a/openspec/specs/fleet-summary/spec.md +++ b/openspec/specs/fleet-summary/spec.md @@ -72,7 +72,13 @@ sticky-session account distribution. ### Requirement: Fleet summary requires API key authentication -The system SHALL expose `GET /api/fleet/summary` for trusted local fleet consumers. The route MUST require a valid Bearer API key even when global proxy API-key authentication is disabled. For callers allowed to view upstream usage, each account SHALL expose `lastRefreshAt` as OAuth token freshness and `usageRefreshedAt` as quota-snapshot freshness. `usageRefreshedAt` MUST equal the newest `recorded_at` value among the persisted usage samples used to build that account summary, or `null` when no such sample exists. +The system SHALL expose `GET /api/fleet/summary` for trusted local fleet +consumers. The route MUST require a valid Bearer API key even when global proxy +API-key authentication is disabled. For callers allowed to view upstream +usage, each account SHALL expose `lastRefreshAt` as OAuth token freshness and +`usageRefreshedAt` as quota-snapshot freshness. `usageRefreshedAt` MUST equal +the newest `recorded_at` value among the persisted usage samples used to build +that account summary, or `null` when no such sample exists. #### Scenario: Missing fleet summary key is rejected @@ -84,13 +90,15 @@ The system SHALL expose `GET /api/fleet/summary` for trusted local fleet consume - **WHEN** a client calls `GET /api/fleet/summary` with a valid Bearer API key - **THEN** the response includes `accounts[]` -- **AND** each account includes `accountId`, `displayName`, `email`, `status`, `planType`, `primary`, `secondary`, `lastRefreshAt`, and `usageRefreshedAt` +- **AND** each account includes `accountId`, `displayName`, `email`, `status`, + `planType`, `primary`, `secondary`, `lastRefreshAt`, and `usageRefreshedAt` - **AND** each window includes `remainingPercent`, `resetAt`, and `windowMinutes` #### Scenario: Usage refresh advances independently of OAuth refresh - **GIVEN** an account has an existing quota snapshot and OAuth refresh time -- **WHEN** force probe or fleet refresh persists a newer usage sample without refreshing OAuth credentials +- **WHEN** force probe or fleet refresh persists a newer usage sample without + refreshing OAuth credentials - **THEN** `usageRefreshedAt` advances to the newer usage sample time - **AND** `lastRefreshAt` remains unchanged @@ -132,3 +140,48 @@ The route MUST preserve existing usage-refresh rules for disabled refresh, fresh - **WHEN** a valid client calls `POST /api/fleet/refresh` - **THEN** active accounts are eligible for the refresh attempt - **AND** paused, reauth-required, and deactivated accounts are not attempted + +### Requirement: Fleet refreshes participate in graceful shutdown + +The system MUST strongly own every accepted `POST /api/fleet/refresh` task from creation until its dedicated session has finished and closed, regardless of whether its caller remains attached. Task creation and registry insertion MUST occur synchronously before the route first awaits the task. Graceful shutdown MUST wait for all such tracked refreshes for up to `shutdown_drain_timeout_seconds` before stopping usage-refresh singleflight work or closing shared HTTP and database resources. If the deadline expires, the system MUST report each fleet refresh that did not drain before continuing shutdown. + +#### Scenario: Caller cancellation does not orphan fleet refresh work + +- **GIVEN** a fleet refresh is running in its dedicated session +- **WHEN** the requesting client disconnects or its request task is cancelled +- **THEN** the refresh continues independently of the cancelled caller +- **AND** it remains tracked until its session exits + +#### Scenario: Shutdown begins before caller cancellation + +- **GIVEN** a fleet refresh was accepted and its caller remains attached +- **WHEN** the in-flight drain times out and graceful shutdown starts draining fleet tasks +- **THEN** the refresh is already present in the fleet task registry +- **AND** cancelling the caller afterward does not remove the refresh from shutdown ownership + +#### Scenario: Shutdown waits for a detached fleet refresh + +- **GIVEN** a cancelled-request fleet refresh is still pending when graceful shutdown begins +- **WHEN** the refresh completes within the configured drain timeout +- **THEN** shutdown waits for the refresh +- **AND** usage singleflight, shared HTTP clients, and database engines remain available until it finishes + +#### Scenario: Overdue fleet refresh is reported + +- **GIVEN** a detached fleet refresh remains pending for the full configured drain timeout +- **WHEN** graceful shutdown drains fleet tasks +- **THEN** the drain reports that task as overdue +- **AND** shutdown is allowed to continue + +### Requirement: Post-cutoff fleet refreshes are rejected before resource work + +Immediately after the in-flight drain attempt returns, graceful shutdown MUST synchronously close fleet task admission before any further shutdown await. A `POST /api/fleet/refresh` request that reaches its producer after this cutoff MUST return the dashboard `503 service_unavailable` error envelope and MUST NOT create a refresh coroutine, task, background session, or other refresh resource work. + +#### Scenario: Late fleet producer receives service unavailable + +- **GIVEN** graceful shutdown has closed control-plane task admission +- **WHEN** an authenticated caller requests `POST /api/fleet/refresh` +- **THEN** the response status is 503 +- **AND** the dashboard error code is `service_unavailable` +- **AND** no fleet refresh task or background session starts + diff --git a/openspec/specs/frontend-architecture/context.md b/openspec/specs/frontend-architecture/context.md index ba054dc673..f18efef6ba 100644 --- a/openspec/specs/frontend-architecture/context.md +++ b/openspec/specs/frontend-architecture/context.md @@ -35,9 +35,10 @@ interaction away. `ADVANCED_NAV_ITEMS` are flat `as const` arrays in the header component (no separate nav-items module); the CI simplicity budget manifest (`.github/simplicity-budgets.toml`, `[core_nav]`) points at this file. -- **No route changes.** `/automations` deep links and the legacy `/firewall` → - `/settings` redirect are compatibility surfaces and keep working; regression - tests cover both. +- **No new routes.** `/automations` deep links stay as-is. The legacy + `/firewall` compatibility route redirects to `/settings?advanced=1#firewall` + so Advanced expands and the firewall section is in view; plain `/settings` + stays collapsed by default. Regression tests cover both. ### Example diff --git a/openspec/specs/frontend-architecture/spec.md b/openspec/specs/frontend-architecture/spec.md index 0da0717919..87176c2129 100644 --- a/openspec/specs/frontend-architecture/spec.md +++ b/openspec/specs/frontend-architecture/spec.md @@ -929,30 +929,52 @@ selected value using the settings API field `preferEarlierResetWindow`. ### Requirement: Dashboard account cards show live credit state -Account summary responses SHALL expose the latest upstream credit metadata for -each account as nullable `creditsHas`, `creditsUnlimited`, and `creditsBalance` -fields. The dashboard account schema SHALL accept those fields. +Account summary responses SHALL expose nullable upstream purchased-credit metadata as `creditsHas`, `creditsUnlimited`, and `creditsBalance`, alongside calculated remaining subscription credits for each available quota window. Dashboard card and list views MUST present calculated subscription quota and purchased credits as separate labeled metrics and MUST NOT use one as a fallback replacement for the other. -The dashboard account card SHALL render a compact Credits row. If -`creditsUnlimited` is true, the value SHALL be `Unlimited`. Otherwise, when a -numeric credit balance is available it SHALL render that balance. If no credit -balance is available, the card MAY fall back to the account's remaining weekly -or primary credit value, and SHALL render `-` when no credit value is known. +When `creditsUnlimited` is true, the purchased-credit metric SHALL render `Unlimited`. Otherwise, it SHALL render the numeric `creditsBalance` when available and `-` when unavailable. The subscription metric SHALL select remaining credits with the following precedence: monthly credits for monthly-only accounts; secondary credits for weekly-only accounts; otherwise secondary credits when available, falling back to primary credits. It SHALL render `-` when the selected value is unavailable. + +The compact list SHALL sort subscription and purchased credits independently. A persisted legacy `credits` sort preference SHALL migrate to the purchased-credit sort so existing operator preferences remain valid after upgrade. + +#### Scenario: Zero purchased balance does not hide subscription quota + +- **WHEN** an account summary has `creditsBalance = 0.0` +- **AND** `remainingCreditsSecondary = 35910.0` +- **THEN** the dashboard shows subscription quota `35910.00` +- **AND** separately shows purchased credits `0.00` + +#### Scenario: Unlimited applies only to purchased credits + +- **WHEN** an account summary has `creditsUnlimited = true` +- **THEN** the purchased-credit metric shows `Unlimited` +- **AND** the subscription metric still shows its own remaining quota value or `-` + +#### Scenario: Missing metrics render independently + +- **WHEN** an account summary has no purchased credit balance and no calculated remaining subscription credits +- **THEN** both separately labeled metrics show `-` #### Scenario: Unlimited credits render explicitly - **WHEN** an account summary has `creditsUnlimited = true` -- **THEN** the dashboard account card shows `Credits: Unlimited` +- **THEN** the dashboard account card shows purchased credits as `Unlimited` +- **AND** the subscription quota remains independently visible #### Scenario: Positive credit balance renders on the card - **WHEN** an account summary includes `creditsBalance = 1.5` -- **THEN** the dashboard account card shows that numeric credit balance +- **THEN** the dashboard account card shows purchased credits as `1.50` +- **AND** does not replace the subscription quota value #### Scenario: Missing credit data renders a placeholder -- **WHEN** an account summary has no credit balance and no remaining credit fallback -- **THEN** the dashboard account card shows `Credits: -` +- **WHEN** an account summary has no purchased credit balance and no remaining subscription credit value +- **THEN** the dashboard account card shows `-` for both separately labeled metrics + +#### Scenario: Legacy credit sort remains valid + +- **WHEN** local dashboard preferences contain the legacy `credits` sort key +- **THEN** the dashboard migrates it to the purchased-credit sort key +- **AND** persists the migrated preference ### Requirement: Dashboard settings must expose upstream proxy routing controls The settings dashboard MUST allow operators to inspect upstream proxy routing state, enable or disable routing, choose the default proxy pool, create proxy endpoints, create proxy pools, and add endpoints to pools. @@ -2383,3 +2405,946 @@ unchanged. - **GIVEN** the dashboard principal has role `admin` - **WHEN** the dashboard view selector opens - **THEN** it exposes both Request Logs and Conversations + +### Requirement: Dashboard routes are code-split + +Each dashboard route's page component MUST load lazily so the entry chunk excludes the code of pages the operator has not visited; the built entry chunk MUST NOT statically import or modulepreload page chunks. + +#### Scenario: Entry chunk excludes unvisited pages + +- **WHEN** the dashboard entry page loads +- **THEN** only the visited route's page chunk is fetched +- **AND** the built entry chunk neither statically imports nor modulepreloads the other pages' chunks + +### Requirement: Dashboard assets are fully self-hosted + +The dashboard MUST NOT load fonts or other render-blocking resources from external origins; all font assets ship with the build and declare `font-display: swap`. + +#### Scenario: No external origins in the built shell + +- **WHEN** the dashboard shell is built +- **THEN** `index.html` and the emitted assets reference no external font or stylesheet origins + +#### Scenario: First paint proceeds without network egress + +- **GIVEN** a deployment without outbound internet access +- **WHEN** an operator loads the dashboard +- **THEN** first paint is not blocked on any external request and monospace text renders via the bundled font or the system fallback + +### Requirement: Dashboard supports Korean runtime locale + +The dashboard SHALL support Korean (`ko`) as a runtime locale in addition to +English (`en`) and Simplified Chinese (`zh-CN`). Korean language detection SHALL +select `ko` for browser language tags whose base language is `ko`, and the +language switcher SHALL let users choose Korean without reloading the page. + +#### Scenario: First visit with a Korean browser + +- **WHEN** a user opens the dashboard for the first time with `navigator.language = "ko-KR"` and no persisted preference +- **THEN** the dashboard renders the translated in-scope surface in Korean +- **AND** `localStorage` contains `codex-lb-language=ko` +- **AND** `document.documentElement.lang` is set to `ko` + +#### Scenario: User toggles Korean + +- **WHEN** the user activates the language switcher and selects Korean +- **THEN** the dashboard re-renders translated strings in Korean without a full page reload +- **AND** the selected language persists across reloads + +### Requirement: Dashboard feature surfaces render in the active locale + +Dashboard feature surfaces SHALL render user-visible copy through the active +i18n locale, including page headings, section headings, empty states, table +headings, filter labels, button labels, accessible labels, dialog titles, +dialog descriptions, validation messages, and client-side toast fallback copy. +This requirement applies to Accounts, Dashboard, API Keys, APIs, Reports, +Automations, Firewall, Model Sources, Quota Planner, Sticky Sessions, Settings +subsections, and shared dashboard components. + +The dashboard MAY keep protocol names, product names, model/API terminology, +quota window abbreviations, and compact operational abbreviations in English +when the English form is the clearest operator-facing label. + +#### Scenario: Korean feature page rendering + +- **WHEN** a user selects `ko` +- **AND** opens Accounts, Dashboard, API Keys, APIs, Reports, Automations, Firewall, Model Sources, Quota Planner, Sticky Sessions, or Settings subsections +- **THEN** user-visible labels, headings, empty states, dialog copy, accessible labels, and client-side toast fallback copy render in Korean +- **AND** technical terms such as `API Key`, `Model`, `TOTP`, `OAuth`, `TTFT`, `TPS`, and `Fast Mode` MAY remain English where appropriate + +#### Scenario: Simplified Chinese feature page rendering + +- **WHEN** a user selects `zh-CN` +- **AND** opens a dashboard feature page beyond the original auth/header/settings coverage +- **THEN** newly migrated user-visible strings render in Simplified Chinese +- **AND** the page does not fall back to English because a locale key is missing + +#### Scenario: Locale bundles stay in sync + +- **WHEN** the frontend locale bundles are compared +- **THEN** `en`, `zh-CN`, and `ko` expose the same translation keys + +### Requirement: Reports endpoint rejects inverted date ranges before repository work + +After applying defaults for any omitted date bound, the Reports service MUST reject a `start_date` later than `end_date` before converting report boundaries or awaiting any repository operation. `GET /api/reports` MUST map that domain failure to HTTP 400 with the exact dashboard envelope `{"error":{"code":"invalid_report_date_range","message":"start_date must be on or before end_date"}}`. Valid one-day ranges and valid inclusive ranges of 730 calendar days MUST remain accepted. + +#### Scenario: Explicit inverted Reports range is rejected + +- **WHEN** an authenticated operator requests `GET /api/reports` with `start_date` later than `end_date` +- **THEN** the endpoint returns HTTP 400 +- **AND** the response body is exactly `{"error":{"code":"invalid_report_date_range","message":"start_date must be on or before end_date"}}` +- **AND** the Reports repository receives no call + +#### Scenario: Defaulted end date makes the range inverted + +- **WHEN** an authenticated operator requests `GET /api/reports` with an explicit `start_date` later than the defaulted current `end_date` +- **THEN** the endpoint returns the same `invalid_report_date_range` HTTP 400 before repository work + +#### Scenario: Boundary-valid Reports ranges remain accepted + +- **WHEN** an authenticated operator requests a one-day range whose `start_date` equals `end_date` +- **THEN** the endpoint accepts the request and reports data for that day +- **WHEN** an authenticated operator requests an inclusive range of exactly 730 calendar days +- **THEN** the endpoint accepts the request under the existing range limit + +### Requirement: Reports date controls prevent, explain, and recover from inverted ranges + +The `/reports` start-date input MUST use the earlier of the browser-local current day and a present end date as its native `max`, and the end-date input MUST use a present start date as its native `min` while retaining the browser-local current day as its `max`. If both values are present and the start date is later than the end date, both controls MUST expose `aria-invalid`, both MUST reference the same localized inline corrective message through an accessible description, and neither the filtered Reports query nor the relaxed Reports filter-catalog query MAY send a request. Correcting either bound so the range is ordered MUST clear the invalid state and resume each distinct Reports query with the corrected bounds. + +#### Scenario: Reciprocal native bounds prevent routine inverted selection + +- **GIVEN** `/reports` has a selected start date and end date +- **THEN** the start-date control's `max` is the earlier of the end date and the browser-local current day +- **AND** the end-date control's `min` is the start date +- **AND** the end-date control's `max` remains the browser-local current day + +#### Scenario: Bypassed inverted input is accessible and sends no Reports request + +- **WHEN** typed, restored, or programmatically supplied Reports dates have a start date later than the end date +- **THEN** both date controls expose `aria-invalid` +- **AND** both controls reference one visible localized message that tells the operator to place the start date on or before the end date +- **AND** no `GET /api/reports` request is sent for either Reports query + +#### Scenario: Retry while inverted only retries Accounts + +- **GIVEN** `/reports` has an inverted date range and loading account options failed +- **WHEN** the operator activates the page-level Retry action +- **THEN** the Accounts query sends a retry request +- **AND** neither Reports query sends a request + +#### Scenario: Correcting either invalid bound resumes Reports queries + +- **GIVEN** `/reports` has an inverted date range and both Reports queries are disabled +- **WHEN** the operator corrects either date bound so the start date is on or before the end date +- **THEN** both controls clear the invalid state and accessible description +- **AND** the corrective message is removed +- **AND** each distinct Reports query sends one request using the corrected ordered bounds + +### Requirement: Dashboard overview and request-log listing fail independently + +The Dashboard SHALL gate overview-backed statistics, quota, projections, and account controls only on dashboard overview availability. The Request Logs section SHALL own the initial loading, terminal error, and ready states of its listing query without hiding healthy overview-backed content. + +When the initial request-log listing reaches a terminal error, the Request Logs section MUST remain visible, MUST render the listing error inside that section, MUST announce that error through an alert semantic local to the section, and MUST expose a keyboard-operable, accessibly named Retry action. Activating Retry MUST refetch only the request-log listing query and MUST NOT refetch or hide healthy overview-backed content. + +#### Scenario: Initial request-log failure preserves healthy overview + +- **GIVEN** dashboard overview, projections, and request-log filter options load successfully +- **WHEN** the initial request-log listing reaches a terminal error +- **THEN** overview statistics, quota, and account content remain rendered +- **AND** the page-wide Dashboard loading skeleton is not rendered +- **AND** the Request Logs section contains and announces the listing error and exposes a Retry action + +#### Scenario: Request-log retry recovers independently + +- **GIVEN** healthy overview-backed content is rendered and the initial request-log listing has failed +- **WHEN** the listing endpoint recovers and the operator activates Retry +- **THEN** only the request-log listing query is refetched +- **AND** healthy overview-backed content remains visible throughout recovery +- **AND** the recovered request-log rows render in the Request Logs section + +#### Scenario: Request logs load inside their section + +- **GIVEN** dashboard overview data is available +- **WHEN** the initial request-log listing is still pending +- **THEN** overview-backed content is rendered +- **AND** the Request Logs section renders its own loading state +- **AND** the page-wide Dashboard loading skeleton is not rendered + +#### Scenario: Initial overview loading keeps the existing page skeleton + +- **WHEN** the dashboard overview is not yet available +- **THEN** the Dashboard renders its existing page-wide loading skeleton +- **AND** it does not render overview-backed content prematurely + +### Requirement: App header brand links to dashboard + +The app header brand area SHALL render a `` wrapping the +logo and "Codex LB" text so that clicking the brand navigates back to the +dashboard home page. The link SHALL preserve the existing visual layout (logo +size, gradient background, text styling) and SHALL include keyboard +focus-visible ring styling matching the project's existing interactive-element +conventions. + +#### Scenario: Brand click navigates to dashboard + +- **WHEN** an operator clicks the header brand area (logo or "Codex LB" text) +- **THEN** the SPA navigates to `/dashboard` + +#### Scenario: Brand link is keyboard-accessible + +- **WHEN** an operator tabs to the header brand +- **THEN** the brand area receives a visible focus ring +- **AND** pressing Enter navigates to `/dashboard` + +#### Scenario: Brand link preserves visual appearance + +- **WHEN** the header renders +- **THEN** the logo and "Codex LB" text appear visually identical to the prior + non-interactive `
` layout + +### Requirement: Dashboard metrics expose conversation-bearing requests + +The dashboard overview response MUST expose +`summary.metrics.conversationRequests` as the count of non-warmup request-log +rows in the selected timeframe whose trimmed `conversation_id` is nonblank. +The existing `requests` field MUST continue counting all non-warmup rows, and +the existing `conversations` field MUST continue counting distinct nonblank +conversation IDs in that timeframe. + +#### Scenario: Requests without conversation IDs are excluded from the new count + +- **GIVEN** a timeframe contains four requests with nonblank conversation IDs + and two requests with null or whitespace-only IDs +- **WHEN** the dashboard overview is requested +- **THEN** `conversationRequests` is `4` +- **AND** `requests` includes all six non-warmup requests + +### Requirement: Dashboard conversation card shows the filtered average + +The dashboard conversation card MUST be labeled `Active Conversations` with +the selected timeframe, and its secondary metadata MUST show `Avg req/conv` +followed by `conversationRequests / conversations`, formatted to one decimal +place. When `conversations` is zero, the metadata MUST show an em dash instead +of dividing by zero. + +#### Scenario: Average uses only conversation-bearing requests + +- **GIVEN** `conversationRequests` is `5` and `conversations` is `2` +- **WHEN** the dashboard card is rendered +- **THEN** its metadata shows `Avg req/conv 2.5` + +#### Scenario: Average is safe when no conversations exist + +- **GIVEN** `conversationRequests` is `4` and `conversations` is `0` +- **WHEN** the dashboard card is rendered +- **THEN** its metadata shows `Avg req/conv —` + +### Requirement: Dashboard and report labels identify active conversations + +The dashboard and report conversation summary cards MUST use the localized +equivalent of `Active Conversations`; their numeric values and ordering MUST +remain unchanged. The report card MUST NOT gain the dashboard average. + +#### Scenario: Report uses the active-conversation label + +- **WHEN** the report summary cards render +- **THEN** the conversation card label is `Active Conversations` in English +- **AND** its numeric value remains the existing distinct conversation total +- **AND** no `Avg req/conv` metadata is rendered on the report card + +### Requirement: Simplified Chinese locale bundle covers all dashboard keys + +The `zh-CN` locale bundle SHALL provide an entry for every user-visible i18n +key present in the `en` bundle, so no dashboard surface falls back to English +because of a missing key. Values MAY keep protocol names, product names, +model/API terminology, and compact operational abbreviations in English when +the English form is the clearest operator-facing label. + +#### Scenario: zh-CN rendering without English fallback + +- **WHEN** a user selects `zh-CN` +- **AND** opens Accounts, Dashboard, API Keys, APIs, Reports, Automations, Firewall, Model Sources, Quota Planner, Sticky Sessions, Upstream Proxy, or Settings subsections +- **THEN** user-visible labels, headings, empty states, dialog copy, accessible labels, and client-side toast fallback copy render through the `zh-CN` bundle +- **AND** no string falls back to English because of a missing locale key +- **AND** technical terms such as `API Key`, `Model`, `OAuth`, `TOTP`, `Credits`, and `Quota` MAY remain English where appropriate + +### Requirement: zh-CN terminology stays consistent across feature surfaces + +Translated `zh-CN` strings SHALL reuse established dashboard terminology for +repeated concepts, and labels that sit inside a label group whose siblings are +already translated SHALL render in Simplified Chinese as well. + +#### Scenario: Consistent wording for repeated concepts + +- **WHEN** a concept already has an established `zh-CN` translation on one surface (e.g. 账户消耗预测 in the settings appearance section) +- **THEN** other surfaces referencing the same concept reuse that wording instead of introducing a synonym + +#### Scenario: Mixed-label groups render fully in Chinese + +- **WHEN** a filter group or table header contains several labels and some already render in Simplified Chinese (e.g. 状态, 类型) +- **THEN** the remaining labels in that group render in Simplified Chinese (e.g. 触发方式) instead of falling back to English + +### Requirement: Dashboard numeric units stay locale-independent + +Dashboard quantities that use compact formatting SHALL use `K`, `M`, and `B` +suffixes regardless of the selected interface locale so requests, tokens, +balances, pool totals, projections, and configured thresholds remain directly +comparable. Dashboard USD values SHALL use the `$` prefix across locales. + +#### Scenario: Simplified Chinese compact quantity display + +- **WHEN** a user selects `zh-CN` +- **AND** views compact request, token, or credit quantities +- **THEN** 10,200 renders as `10.2K` +- **AND** 1,500,000 renders as `1.5M` +- **AND** 1,500,000,000 renders as `1.5B` +- **AND** 12 USD renders as `$12.00` + +### Requirement: Reports per-day averages use the inclusive local calendar window + +`GET /api/reports` MUST calculate `summary.avgCostPerDay` and +`summary.avgRequestsPerDay` by dividing the current report totals by exactly +`(end_date - start_date).days + 1`. The divisor MUST represent the selected +inclusive local calendar-date window and MUST NOT be derived from the +UTC-converted filter boundaries. + +#### Scenario: Offset-to-zero transition keeps a two-day divisor + +- **WHEN** an operator requests `2026-02-15` through `2026-02-16` in + `Africa/Casablanca` and the report totals are 60 cost units and 30 requests +- **THEN** `avgCostPerDay` is `30` +- **AND** `avgRequestsPerDay` is `15` + +#### Scenario: Offset-from-zero transition keeps a two-day divisor + +- **WHEN** an operator requests `2026-03-22` through `2026-03-23` in + `Africa/Casablanca` and the report totals are 60 cost units and 30 requests +- **THEN** `avgCostPerDay` is `30` +- **AND** `avgRequestsPerDay` is `15` + +### Requirement: Dashboard status separates service readiness from usage synchronization + +The fixed dashboard status bar MUST render independent `Service ready` and +`Usage synced` signals. `Service ready` MUST use the existing `/health/ready` +response and MUST treat a failed request or a non-`ok` status as not ready. +`Usage synced` MUST remain derived only from the dashboard overview +`lastSyncAt` value and MUST be fresh only while that timestamp is less than 60 +seconds old. The service-readiness signal MUST NOT use upstream account or +provider health. The dashboard layout MUST reserve at least the status bar's +rendered height so wrapped status rows do not cover page content. + +#### Scenario: Ready service with stale usage + +- **WHEN** `/health/ready` returns `status: "ok"` +- **AND** `lastSyncAt` is absent or at least 60 seconds old +- **THEN** the status bar shows the service as ready +- **AND** independently shows usage as stale + +#### Scenario: Unready service with fresh usage + +- **WHEN** `/health/ready` fails or returns a non-`ok` status +- **AND** `lastSyncAt` is less than 60 seconds old +- **THEN** the status bar shows the service as not ready +- **AND** independently shows usage as synced + +#### Scenario: Readiness is still being checked + +- **WHEN** the initial `/health/ready` request has not completed +- **THEN** the service-readiness signal shows a checking state +- **AND** the usage synchronization signal remains independently derived from + `lastSyncAt` + +#### Scenario: Status signals wrap onto additional rows + +- **WHEN** the fixed status bar grows because its signals wrap +- **THEN** the dashboard updates its reserved bottom space to the rendered + status-bar height +- **AND** the fixed status bar does not cover page content + +### Requirement: Dashboard conversation listing + +The authenticated dashboard MUST expose `GET /api/conversations`. The list +endpoint MUST accept `limit`, `offset`, `search`, `since`, and `timeframe` query +parameters. The server-authoritative `timeframe` parameter MUST accept `1d`, +`7d`, or `30d`; when it is supplied, the server MUST derive the activity window +from the shared dashboard timeframe configuration and the client MUST NOT +substitute a browser-clock-generated `since` value. `timeframe` and `since` MUST +not be supplied together. When `since` is omitted, the server MUST apply a +rolling 30-day lower bound; +explicitly older `since` values MUST be capped at that same bound, and incoming +timezone-aware datetimes MUST be normalized to naive UTC before querying. It +MUST aggregate eligible `request_logs` rows by the raw, non-empty +`conversation_id` column, excluding rows whose request kind is `warmup` or +`limit_warmup`, and rows with `deleted_at IS NOT NULL`. Production request-log +writes MUST normalize ASCII padding and blank conversation IDs before storage; +conversation list, facet, and detail queries MUST use raw-column +`conversation_id` predicates and grouping rather than function-wrapped +expressions. + +Search MUST be case-insensitive and match the normalized conversation ID or any +eligible row's user-agent family. Search MUST select whole conversations first: +after a conversation matches, aggregation MUST include all eligible rows in that +conversation, including rows whose user-agent family or ID did not match the +search text. The endpoint MUST derive aggregates from `request_logs` only. + +When `since` is provided, a conversation MUST be selected when at least one +eligible row has `requested_at >= since`. A conversation MAY have eligible rows +before `since` and MUST still be included when it has activity in the window. +The grouped summary MUST aggregate all eligible rows for every selected +conversation, so `firstRequest`, `lastRequest`, `requestCount`, token totals, +cached-token totals, and cost MUST NOT be clipped to the window. Membership MUST +be implemented as an in-window aggregate condition and MUST NOT use a global +pre-window ID set or a pre-window anti-join. + +After page membership is selected, the account, API-key, and model facet +queries for the returned page MUST use the same full eligible-row scope as the +summary, restricted only by the selected page's raw `conversation_id` values. +The facet queries MUST NOT add a `requested_at >= since` restriction after +membership selection; facet representatives and remaining counts MUST include +eligible history before `since` and MUST remain consistent with the full-history +summary aggregates. + +The response MUST contain `conversations`, `total`, and `hasMore` pagination +fields. Each row in `conversations` MUST contain exactly these fields and no +response summary object: + +- `conversationId`: the normalized, non-empty conversation identity. +- `firstRequest`: the earliest `requested_at` among all eligible rows in the + conversation. +- `lastRequest`: the latest `requested_at` among all eligible rows in the + conversation. +- `requestCount`: the number of eligible rows in the conversation. +- `representativeAccount` and `remainingAccountCount`. +- `apiKeyId` and `apiKeyName`. +- `representativeModel` and `remainingModelCount`. +- `totalTokens`. +- `cachedInputTokens`. +- `totalCostUsd`. + +The camelCase names above are the external Dashboard API JSON contract. Python +schema, service, and repository identifiers MAY remain snake_case internally; +internal names MUST NOT be emitted as alternate response fields. + +`totalTokens` MUST equal total input tokens plus total output tokens, with +`reasoning_tokens` used for a row when `output_tokens` is null. +`cachedInputTokens` MUST use the existing per-row clamp: null remains null; +otherwise the cached value is clamped to `[0, input_tokens]` when input tokens +are present. At aggregate level, null per-row values MUST NOT be converted to +zero; when every eligible row has a null cached value, `cachedInputTokens` MUST +be null, and otherwise it MUST equal the sum of the known clamped values. + +Representative account values MUST use `request_count DESC, +latest_requested_at DESC, lexical account ASC`. List model values MUST be +grouped by distinct model, combining all reasoning efforts for that model, and +the representative model MUST use `request_count DESC, latest_requested_at DESC, +model lexical ASC`. Null account values MUST be excluded from account +candidates; if no non-null account exists, `representativeAccount` MUST be null +and `remainingAccountCount` MUST be 0. The list MUST NOT split model values by +`reasoning_effort`; `(model, reasoning_effort)` grouping MUST be used only for +conversation details. + +Nullable and multiple-key conversations MUST be handled deterministically. Null +API-key values MUST not be candidates; if no non-null key exists, both API-key +fields MUST be null. When multiple distinct non-null keys exist, `apiKeyId` MUST be selected by +`request_count DESC, latest_requested_at DESC, lexical API-key ID ASC`, and +`apiKeyName` MUST be the corresponding existing dashboard-safe display name. +`apiKeyName` MUST never expose a secret, hash, or plaintext key material. + +The list order MUST be stable: `lastRequest DESC`, then normalized +`conversationId ASC`. Pagination MUST be applied after this ordering. + +#### Scenario: Pagination uses the stable list order + +- **GIVEN** matching conversations have different latest request times and a + tie exists on `lastRequest` +- **WHEN** the client calls `GET /api/conversations?limit=10&offset=20` +- **THEN** rows are ordered by `lastRequest DESC` and ties by normalized + `conversationId ASC` +- **AND** the response starts at the 21st row in that order and reports the + matching total and whether another page exists + +#### Scenario: Blank IDs, warmups, and soft-deleted rows are excluded + +- **GIVEN** request logs include null IDs, whitespace-only IDs, `warmup` rows, + `limit_warmup` rows, soft-deleted rows, and eligible rows with non-empty IDs +- **WHEN** the client calls `GET /api/conversations` +- **THEN** only rows whose request kind is neither `warmup` nor `limit_warmup`, + which are non-soft-deleted and have non-empty normalized IDs, contribute to + returned conversations + +#### Scenario: Search selects whole conversations + +- **GIVEN** one eligible conversation contains a matching user-agent family on + one row and non-matching user-agent/ID values on other rows +- **WHEN** the client calls `GET /api/conversations?search=opencode` +- **THEN** that conversation is selected +- **AND** all eligible rows in that conversation contribute to its counts, + tokens, cached tokens, and cost +- **AND** rows from conversations with no matching ID or user-agent family are + not returned + +#### Scenario: List search is case-insensitive over normalized IDs and user-agent families + +- **GIVEN** an eligible conversation has a normalized ID and user-agent family + whose letters differ in case from the search text +- **WHEN** the client calls `GET /api/conversations?search=OPENCODE` +- **THEN** the conversation is selected when either the normalized ID or any + eligible row's user-agent family matches case-insensitively + +#### Scenario: Since filter selects conversations active in the window + +- **GIVEN** conversation `conv-old` has its earliest eligible row at `t-10d` + and a later row at `t-1d`, and conversation `conv-new` has its earliest + eligible row at `t-1d` +- **WHEN** the client calls `GET /api/conversations?since=` +- **THEN** both `conv-new` and `conv-old` are returned +- **AND** `conv-old` is included because it has a row inside the window even + though its first message predates the window +- **AND** both conversations' summaries aggregate every eligible row, not only + rows at or after `since` + +#### Scenario: Since membership and facets share the full conversation scope + +- **GIVEN** a selected conversation has eligible account, API-key, and model + values both before and after the `since` boundary +- **WHEN** the client calls `GET /api/conversations?since=` +- **THEN** `firstRequest`, `lastRequest`, `requestCount`, and summary totals + include all eligible rows for the conversation +- **AND** account, API-key, and model facet counts and representatives include + all eligible rows in the selected conversation, including rows before `since` + +#### Scenario: Since filter composes with search and pagination + +- **GIVEN** two conversations have activity inside the `since` window and only + one matches the search text +- **WHEN** the client calls `GET /api/conversations?since=&search=opencode` +- **THEN** only the matching conversation is returned +- **AND** the response total and hasMore reflect the since-and-search filtered + set + +#### Scenario: List model representatives ignore reasoning effort + +- **GIVEN** a conversation has requests for the same model with multiple + reasoning-effort values and requests for another model +- **WHEN** the client calls `GET /api/conversations` +- **THEN** the list groups the same model's requests into one model value +- **AND** the representative model is ordered by request count descending, + latest request descending, and model lexical ascending +- **AND** the remaining model count counts distinct models, not model/effort + combinations + +#### Scenario: API-key representation is safe and deterministic + +- **GIVEN** a conversation has null API-key rows and multiple non-null API-key + values with tied counts +- **WHEN** the client calls `GET /api/conversations` +- **THEN** null values do not become the representative +- **AND** the non-null representative is selected by count, latest request, and + lexical API-key ID +- **AND** the response contains only the corresponding dashboard-safe name and + never secret, hash, or plaintext key material + +### Requirement: Dashboard conversation activity uses the list eligibility scope + +The dashboard overview conversation metrics and per-bucket conversation trend +MUST use the same eligible `request_logs` row scope as the conversation list: +non-empty conversation IDs, request kinds other than `warmup` and +`limit_warmup`, and `deleted_at IS NULL`. This scope MUST apply to both the +distinct conversation count and conversation request count in the overview +summary and to each conversation trend bucket. + +#### Scenario: Soft-deleted-only conversations are absent from dashboard activity + +- **GIVEN** the selected timeframe contains an eligible conversation and a + second conversation whose only rows are soft-deleted +- **WHEN** the client requests the conversation list and dashboard overview for + that timeframe +- **THEN** the list total and summary conversation count include only the + eligible conversation +- **AND** the summary conversation request count and conversation trend contain + no contribution from the soft-deleted-only conversation + +### Requirement: Conversation listing total is served from a short-TTL cache + +The grouped `total` returned by `GET /api/conversations` is display-only +pagination metadata that tolerates short staleness, and the dashboard polls the +endpoint every 30 seconds. Recomputing the grouped count over the full eligible +`request_logs` history on every poll risks the same dashboard-induced +database contention this repository has previously optimized away. + +The conversation listing total MUST be served from the same short-TTL +per-filter-signature cache as the request-log listing total (fixed 30 s TTL +application constant; bounded LRU-ish eviction; per-instance). The cache +signature MUST include every dimension that changes the grouped count: `search` +and the semantic window identity MUST be included, using +`("timeframe", timeframe)` for server-authoritative timeframe requests and +`("since", effective_since)` for legacy `since` requests. `limit` and `offset` +MUST be excluded from the signature because the total is page-independent. Two +requests with different search text or window identities MUST NOT reuse one +another's cached total. + +#### Scenario: Repeated polls reuse the cached conversation total + +- **GIVEN** the conversation listing has computed a total for a given + `search` and semantic window signature +- **WHEN** the dashboard polls the same endpoint within the TTL with the same + signature +- **THEN** the grouped count MUST NOT be recomputed +- **AND** the response total MUST equal the previously computed value + +#### Scenario: Different window signatures isolate cached conversation totals + +- **GIVEN** two listing requests differ only by their timeframe or legacy + `since` window +- **WHEN** their totals are served through the cache +- **THEN** each request MUST use its own cache entry and grouped total + +#### Scenario: Search participates in the conversation total cache signature + +- **GIVEN** two listing requests differ only by the `search` text +- **WHEN** their totals are served through the cache +- **THEN** each request MUST use its own cache entry and grouped total + +### Requirement: Conversation details + +The authenticated dashboard MUST expose +`GET /api/conversations/{conversation_id}`. Detail aggregation MUST use the same +eligible-row scope as listing: normalized non-empty IDs, rows whose request kind +is neither `warmup` nor `limit_warmup`, and `deleted_at IS NULL`. + +For a matching conversation, the detail response MUST expose the conversation ID, +`start` (earliest `requested_at`), `latest` (latest `requested_at`), +`accountCount` (distinct non-null accounts), `totalElapsedTime`, and +`dominantUseragentGroup`. `totalElapsedTime` MUST be +`SUM(COALESCE(latency_ms, 0))` over all eligible rows, never the wall-clock span. +`dominantUseragentGroup` MUST use +`request_count DESC, latest_requested_at DESC, lexical ASC`. + +The response MUST include one model/effort row per distinct +`(model, reasoning_effort)` combination. Each row MUST contain exactly: +`modelEffort`, `reqs`, `totalElapsedTime`, `totalInputTokens`, +`cachedInputTokens`, `totalOutputTokens`, and `totalCostUsd`. The row +elapsed time MUST use `SUM(COALESCE(latency_ms, 0))` for that combination; +output tokens MUST use the reasoning-token fallback; cached input MUST use the +existing per-row clamp. No error-count or other column may be returned. + +The API MUST order model/effort rows by `reqs DESC`, latest request DESC, and +lexical key ASC. It MUST NOT accept a sort query parameter. Client-side sorting +MUST operate only on returned rows. + +An encoded blank path such as `GET /api/conversations/%20` MUST return the +project-standard 404 response. An unknown non-empty conversation ID MUST also +return the project-standard 404 response. The detail route MUST accept any +normalized non-empty stored conversation ID, including IDs containing `/`, when +the client percent-encodes the opaque ID as one path value. + +#### Scenario: Details preserve cumulative elapsed time + +- **GIVEN** a conversation has known latencies across multiple accounts and + model/effort combinations +- **WHEN** the client calls `GET /api/conversations/conv-a` +- **THEN** conversation `totalElapsedTime` is the sum of + `COALESCE(latency_ms, 0)` across eligible rows +- **AND** each model/effort row uses the same cumulative sum over its matching + rows rather than the start/latest wall-clock span + +#### Scenario: Details exclude warmups and soft-deleted rows + +- **GIVEN** a conversation contains normal, `warmup`, `limit_warmup`, and + soft-deleted request logs +- **WHEN** the client calls `GET /api/conversations/conv-a` +- **THEN** the summary and every model/effort row include only rows whose request + kind is neither `warmup` nor `limit_warmup` and which are non-soft-deleted + +#### Scenario: Blank and unknown detail IDs use standard not-found behavior + +- **WHEN** the client calls `GET /api/conversations/%20` or requests an unknown + non-empty ID +- **THEN** the API returns the standard 404 error envelope + +#### Scenario: Slash-containing detail IDs remain addressable + +- **GIVEN** an eligible conversation has the normalized ID `workspace/thread-1` +- **WHEN** the client calls `GET /api/conversations/workspace%2Fthread-1` +- **THEN** the API returns that conversation's details with + `conversationId` equal to `workspace/thread-1` + +### Requirement: Dashboard conversation view + +The dashboard MUST render Request Logs by default. The original uppercase +section-title typography MUST be retained, and the title itself MUST be the +single accessible Radix-style selector trigger with `ChevronDown` for Request +Logs and Conversations. A separate selector MUST NOT render to the title's +right. Selecting Conversations MUST persist `view=conversations` in the URL; +selecting Request Logs MUST return to the existing request-log view. + +The dashboard MUST retain separate URL-backed query state for Request Logs and +Conversations, including each view's applicable filters and pagination. +Switching views MUST NOT reinterpret, overwrite, or clear the inactive view's +query state, and returning to a view MUST restore its retained state. + +The Conversations view MUST NOT render a free-text filter input above the list. +The view MUST render a day-range selector with exactly three options — `1d`, +`7d`, and `30d` — placed at the top-right of the dashboard page alongside the +refresh action and shown only while the Conversations view is active. The +selector MUST default to `7d`. The selected value MUST be persisted in the URL +as `conversationTimeframe`, MUST drive the list endpoint's `timeframe` query +parameter using the same symbolic key (the server derives the effective window), +and MUST NOT generate a browser-clock-derived `since` parameter. It MUST reset +pagination to offset 0 on change. The selector's values and default +MUST mirror the dashboard overview timeframe selector, with no unbounded +"all" option. The view MUST use the list endpoint's +established loading, error, empty, and pagination behavior. +While Conversations is active, the dashboard overview query that supplies the +statistics cards MUST use the active `conversationTimeframe`, including on the +initial render when that value is restored from the URL. The independently +retained `overviewTimeframe` MUST continue to drive the overview query when +Request Logs is active. + +The conversation list MUST render exactly these columns in order: Last request, +Conversation, Accounts, API key, Models, Tokens, Cost, and Details. Last request +MUST use the request-log Time column's two-line time/date presentation. Accounts +MUST resolve the representative account ID through the dashboard account +summaries and display `displayName`, then email, then the ID as a final fallback. +Accounts and models MUST render remaining values as a smaller muted `+ N more` +secondary line. Tokens MUST show total tokens with cached input tokens on a +subordinate line. +When dashboard privacy blur is enabled, an account label resolved from an email +fallback MUST render with the established `privacy-blur` class; display-name +and account-ID fallback labels MUST remain unblurred. +The API-key column MUST use `apiKeyName` only. Details MUST use the existing +Details button treatment. + +The details dialog MUST render row one as conversation ID, start, and latest; +row two as account count, total elapsed time, and dominant user-agent family; +and a model/effort table with exactly these displayed columns, in order: Model +(effort), Reqs, Total elapsed, Total input (with total cache as a +subordinate/parenthetical value), Total output, and Total cost. Total cache MUST +not be a separate displayed column. The table MUST default to Reqs descending +and MUST support client-side sorting for every displayed column without adding a +sort query parameter. +The displayed conversation ID MUST NOT provide a copy action. + +The detail dialog MUST use the established dashboard loading state while the +detail API is pending. Unknown or malformed conversation IDs, including a +standard detail API 404, MUST use the standard dashboard error display and retry +behavior. Nullable optional aggregate values MUST render the established +em-dash or other dashboard fallback value without breaking the row or dialog. +An empty conversation list MUST render the established dashboard empty state. +When an empty conversation list is returned for a nonzero pagination offset, the +Conversations view MUST retain its pagination controls so the operator can +navigate back to the first or previous page. The initial empty state at offset +zero MUST NOT render pagination controls. + +#### Scenario: Request Logs is the default and selector switches views + +- **WHEN** an operator opens the dashboard +- **THEN** Request Logs is visible and active by default +- **WHEN** the operator selects Conversations +- **THEN** the Conversations list renders and the URL contains + `view=conversations` + +#### Scenario: Request Logs and Conversations retain independent URL query state + +- **GIVEN** Request Logs has active filters and pagination and Conversations has + different active filters and pagination retained in the URL +- **WHEN** the operator switches between the two views +- **THEN** each view restores its own filters and pagination +- **AND** switching views does not reinterpret, overwrite, or clear the other + view's query state + +#### Scenario: Conversations has no free-text filter and renders the day selector + +- **WHEN** the operator opens the Conversations view +- **THEN** no free-text filter input is rendered above the list +- **AND** a day-range selector with exactly `1d`, `7d`, and `30d` options is + rendered at the top-right of the dashboard page alongside the refresh action +- **AND** the selector defaults to `7d` and no unbounded "all" option is offered +- **AND** the list renders the specified reordered columns and two-line request + time presentation +- **AND** representative account IDs resolve to display name, then email, then ID +- **AND** smaller muted `+ N more` account/model secondary lines and cached + tokens as a subordinate line are rendered + +#### Scenario: Conversation day selector persists in the URL and drives timeframe + +- **WHEN** the operator changes the Conversations day selector from `7d` to `30d` +- **THEN** the URL gains `conversationTimeframe=30d` (or drops the param when the + default `7d` is selected) +- **AND** the list endpoint is called with `timeframe=30d` +- **AND** the list endpoint does not receive a browser-clock-derived `since` +- **AND** pagination resets to offset 0 + +#### Scenario: Conversation timeframe drives active dashboard statistics + +- **GIVEN** the URL restores `conversationTimeframe=30d` while + `overviewTimeframe` is absent or has a different value +- **WHEN** the operator opens the Conversations view +- **THEN** the statistics-card overview query uses the `30d` timeframe +- **AND** the conversation list uses `timeframe=30d` +- **AND** the independently retained overview timeframe remains unchanged + for the Request Logs view + +#### Scenario: Conversation day selector state is independent per view + +- **GIVEN** the Conversations day selector is set to `30d` +- **WHEN** the operator switches to Request Logs and back to Conversations +- **THEN** the Conversations view restores its retained `30d` selector state +- **AND** the Request Logs view state is unaffected + +#### Scenario: Conversation account privacy blur applies only to email fallback + +- **GIVEN** dashboard privacy blur is enabled and account labels resolve using + display name, email fallback, and account-ID fallback values +- **WHEN** the Conversations list renders +- **THEN** only the email-fallback label has the established `privacy-blur` class +- **AND** the display-name and account-ID fallback labels remain unblurred + +#### Scenario: The original-styled title is the only view selector + +- **WHEN** the list section renders +- **THEN** its uppercase title typography is retained +- **AND** activating the title opens the Request Logs/Conversations selector +- **AND** no separate selector is rendered to the title's right + +#### Scenario: Conversation details use established loading and retry states + +- **WHEN** the detail API is loading for a selected conversation +- **THEN** the dialog uses the established dashboard loading state +- **WHEN** the detail API returns an unknown or malformed-ID error +- **THEN** the dialog uses the standard dashboard error display with retry + +#### Scenario: Nullable detail aggregates use dashboard fallbacks + +- **GIVEN** a successful detail response contains nullable optional aggregate + values +- **WHEN** the operator opens the details dialog +- **THEN** each nullable value renders the established em-dash or dashboard + fallback without breaking the row or dialog + +#### Scenario: Empty conversation results use the existing empty state + +- **GIVEN** the conversation list response contains no rows +- **WHEN** the operator opens the Conversations view +- **THEN** the existing dashboard empty state is rendered + +#### Scenario: Empty later conversation pages retain pagination controls + +- **GIVEN** the operator is on a nonzero Conversations page and the list + response contains no rows +- **WHEN** the Conversations view renders the response +- **THEN** the existing dashboard empty state is rendered +- **AND** pagination controls remain visible +- **AND** the first-page and previous-page controls provide a path back to + earlier results + +#### Scenario: Details dialog has the approved layout and sorting + +- **WHEN** the operator opens a conversation's Details dialog +- **THEN** row one contains conversation ID/start/latest +- **AND** conversation ID has no copy action +- **AND** row two contains account count/total elapsed/dominant user-agent +- **AND** the table displays exactly Model (effort), Reqs, Total elapsed, Total + input (with total cache as a subordinate/parenthetical value), Total output, + and Total cost +- **AND** the table initially sorts by Reqs descending +- **AND** activating any displayed table column header reorders only the returned + rows client-side + +### Requirement: Conversation list exposes grouped request metrics + +`GET /api/conversations` SHALL include `requestCount` and `firstRequest` for +every conversation. `requestCount` SHALL count all eligible request-log rows +in that conversation, and `firstRequest` SHALL be the earliest eligible +`requested_at`. Existing `lastRequest` SHALL remain the latest eligible +`requested_at`. + +#### Scenario: A conversation aggregates request metrics + +- **GIVEN** one conversation has eligible requests at 10:00, 10:07, and + 12:15 +- **WHEN** the conversation list is requested +- **THEN** its `requestCount` is `3` +- **AND** its `firstRequest` is the 10:00 timestamp +- **AND** its `lastRequest` is the 12:15 timestamp +- **AND** warmup, limit-warmup, deleted, blank-ID, and otherwise ineligible + rows do not affect those values + +### Requirement: Conversation list renders metrics and readable duration + +The dashboard SHALL render columns in this order: Last request, Lasted, +Conversation, Accounts, API key, Models, Requests, Tokens, Cost, Details. +The Lasted value SHALL use `lastRequest - firstRequest`, displaying `0s` for +zero duration, seconds for durations under one minute, `xm ys` for durations +under one hour, `xh ym` for durations under one day, and `xd yh` for durations +of at least one day. The conversation-ID cell SHALL be top-aligned. + +#### Scenario: Duration uses two units and preserves zero + +- **WHEN** a row spans 2 hours and 15 minutes +- **THEN** Lasted displays `2h 15m` +- **WHEN** a row spans 2 days and 3 hours +- **THEN** Lasted displays `2d 3h` +- **WHEN** firstRequest equals lastRequest +- **THEN** Lasted displays `0s` + +### Requirement: Fair-share congestion threshold is configurable from routing settings + +The dashboard routing settings MUST expose the API-key fair-share congestion threshold as a numeric field adjacent to the per-account capacity limits, accepting integers from 0 to 100 where 0 disables the gate, with null-inherits-environment semantics matching the per-account capacity overrides. Values outside 0-100 MUST be rejected by both the client-side validation and the settings API. The field's label, description, and validation copy MUST be localized in the en, ko, and zh-CN locale bundles. + +#### Scenario: Threshold round-trips through the settings API + +- **GIVEN** an operator sets the threshold to 80 in routing settings +- **WHEN** the settings are saved and reloaded +- **THEN** the field shows 80 and the settings API reports 80 as the effective value + +#### Scenario: Migrated null row inherits the environment default + +- **GIVEN** a deployment whose dashboard settings row predates the field (a migrated NULL column) +- **AND** an environment-configured threshold +- **WHEN** the effective settings are read +- **THEN** the effective value inherits the environment setting + +#### Scenario: Out-of-range values are rejected + +- **GIVEN** an operator enters 101 or a negative number +- **WHEN** they attempt to save +- **THEN** the client blocks the save and the settings API rejects the value if submitted directly + +#### Scenario: Copy is localized in all three locales + +- **GIVEN** the dashboard language is set to en, ko, or zh-CN +- **WHEN** routing settings render +- **THEN** the threshold label and description display in the selected locale + +### Requirement: Appearance settings include date format toggle + +The Appearance settings section SHALL include a "Date format" toggle row with two options: "Default" and "ISO 8601". The toggle SHALL be placed between the Time format and Account rows settings. Selecting an option SHALL immediately apply the new format to applicable read-only date/time presentation text across the dashboard. + +#### Scenario: Default date format is selected initially + +- **WHEN** a user opens the Appearance settings section with no prior date format preference +- **THEN** the "Default" option SHALL be selected (aria-pressed true) +- **AND** applicable read-only date/time presentation text SHALL render using locale-dependent formatting + +#### Scenario: Switching to ISO 8601 + +- **WHEN** the user clicks the "ISO 8601" option in the Date format row +- **THEN** the "ISO 8601" option SHALL be selected +- **AND** request log and conversation table cells SHALL display date on the top line in `YYYY-MM-DD` format and time on the bottom line in `HH:MM:SS` format +- **AND** the preference persists across page reloads + +### Requirement: Accounts and API trend chart x-axis uses MM-DD format + +The x-axis tick format of the Account Trend and API Trend charts SHALL be `MM-DD` (month and day extracted from the ISO timestamp data key), matching the reports chart convention. This format SHALL be locale-independent. + +#### Scenario: Account trend chart x-axis ticks + +- **WHEN** the Account Trend chart renders with timestamp data +- **THEN** the x-axis tick labels SHALL be in `MM-DD` format (e.g., `"08-09"`) + +#### Scenario: API trend chart x-axis ticks + +- **WHEN** the API Trend chart renders with timestamp data +- **THEN** the x-axis tick labels SHALL be in `MM-DD` format (e.g., `"08-09"`) + diff --git a/openspec/specs/github-automation/spec.md b/openspec/specs/github-automation/spec.md index ddec209a68..2883d6309a 100644 --- a/openspec/specs/github-automation/spec.md +++ b/openspec/specs/github-automation/spec.md @@ -333,3 +333,138 @@ labels, so the override MUST NOT apply there: a change that would leave - **WHEN** a budget is exceeded - **THEN** no pull-request label set is resolved and the check fails regardless of any label on the originating pull request + +### Requirement: Codex review trigger usage-limit backoff + +The Codex label synchronization script MUST NOT post a new `@codex review` comment while the comment sender's latest Codex response within the configured backoff window is a usage-limit reply. A usage-limit reply is a Codex response whose body starts (after optional leading whitespace) with the quota envelope "You have reached your Codex usage limits"; Codex reviews that merely discuss usage limits MUST NOT latch the backoff. Usage-limit evidence MUST be attributed to the sender whose request comment preceded the reply, and a newer normal Codex response for that same sender MUST lift the backoff; a clean THUMBS_UP reaction by a Codex reviewer on the sender's request comment counts as a normal response. Backoff state MUST be shared across all repositories processed in one run, so a usage limit observed in one repository suppresses the remaining review requests in the run; classified timelines from repositories without their own triggers MUST still contribute evidence. Before posting review requests in a repository, the script MUST also gather the repository's recent issue comments (within the backoff window, grouped per issue) as evidence, so quota replies on pull requests outside the current selection — including single `--pr` runs and closed pull requests — still latch the backoff; a failure to gather this evidence degrades to the classified-timeline evidence with a warning. When no quota evidence exists for the sender, the script MUST post the first `@codex review`, wait briefly, reread that pull request's timeline, and suppress the remaining review requests in the run if that probe observed a usage-limit reply; probing MUST stop once a normal Codex response has been observed, and MUST NOT run when the review request was not actually posted (for example after a tolerated write denial). Apply-loop status lines and error reports MUST reference the pull request of the decision being applied. + +The script MUST resolve the sender identity in a way that works with GitHub App installation tokens (which cannot call `GET /user`): it prefers the app slug exported by the workflow (`GH_APP_SLUG`, yielding `[bot]`) and falls back to `GET /user` for PAT-backed runs. Once the run has switched to the fallback token, the app slug no longer describes the active identity: sender resolution MUST ignore it, and review triggers MUST be suppressed with a warning because posted comments would no longer be authored by the resolved sender. The review-request POST itself MUST NOT be silently retried under the fallback token after a rate-limit response: the fallback activates for subsequent calls, but the identity-sensitive comment fails instead of posting under the wrong author. If the sender cannot be resolved, only the review-trigger path is disabled (with a warning per affected decision); label synchronization and workflow-run approvals MUST proceed. + +#### Scenario: Recent usage-limit reply latches the backoff + +- **GIVEN** the sender's `@codex review` comment was answered by a Codex usage-limit reply within the backoff window +- **AND** the sender has no newer normal Codex response +- **WHEN** the script would trigger a missing Codex review +- **THEN** it skips the `@codex review` post and surfaces a write warning naming the usage-limit evidence + +#### Scenario: Reviews that merely discuss usage limits do not latch + +- **GIVEN** a Codex review whose body discusses usage limits but does not start with the quota envelope +- **WHEN** the script classifies Codex responses for the backoff +- **THEN** the response is treated as a normal Codex response, not a usage-limit reply + +#### Scenario: Newer normal response lifts the backoff + +- **GIVEN** the sender received a Codex usage-limit reply within the backoff window +- **AND** the same sender has a newer normal Codex response +- **WHEN** the script would trigger a missing Codex review +- **THEN** it posts the `@codex review` comment + +#### Scenario: Newer clean reaction lifts the backoff + +- **GIVEN** the sender received a Codex usage-limit reply within the backoff window +- **AND** a Codex reviewer later reacted with THUMBS_UP to the sender's `@codex review` comment +- **WHEN** the script would trigger a missing Codex review +- **THEN** it posts the `@codex review` comment + +#### Scenario: Senders are attributed independently + +- **GIVEN** the sender received a Codex usage-limit reply within the backoff window +- **AND** only a different account has a newer normal Codex response +- **WHEN** the script would trigger a missing Codex review +- **THEN** it still skips the `@codex review` post for the sender + +#### Scenario: Backoff persists across repositories in one run + +- **GIVEN** a run selecting multiple repositories +- **AND** the sender's usage limit was observed while processing an earlier repository +- **WHEN** the script would trigger a missing Codex review in a later repository +- **THEN** it skips the `@codex review` post there as well + +#### Scenario: Evidence from a non-triggering repository still counts + +- **GIVEN** an earlier repository whose classified timelines contain the sender's usage-limit reply but whose decisions need no review trigger +- **WHEN** a later repository in the same run would trigger a missing Codex review +- **THEN** the earlier repository's evidence latches the backoff and the post is skipped + +#### Scenario: Quota evidence outside the selected pull requests still counts + +- **GIVEN** a single `--pr` run where the sender's usage-limit reply lives on a different (possibly closed) pull request of the repository +- **WHEN** the script would trigger a missing Codex review +- **THEN** the repository's recent issue comments provide the evidence and the post is skipped + +#### Scenario: Probe requires an actual post + +- **GIVEN** the review-request comment was not posted because the write was denied and tolerated +- **WHEN** the script would otherwise probe for a quota reply +- **THEN** it neither waits nor rereads the pull request timeline for that decision + +#### Scenario: No-data probe latches off remaining triggers + +- **GIVEN** no Codex quota evidence exists for the sender in the classified timelines +- **WHEN** the script posts the first `@codex review` of the run +- **THEN** it waits the configured probe interval, rereads that pull request's timeline, and skips the remaining review requests if the probe observed a usage-limit reply + +#### Scenario: Probing stops after a normal response + +- **GIVEN** a normal Codex response for the sender has already been observed +- **WHEN** the script posts further `@codex review` comments in the run +- **THEN** it does not wait or reread pull request timelines for those posts + +#### Scenario: Installation tokens resolve the sender from the app slug + +- **GIVEN** the run authenticates with a GitHub App installation token and the workflow exports the app slug +- **WHEN** the script resolves the `@codex review` sender +- **THEN** it derives `[bot]` without calling `GET /user` + +#### Scenario: Sender resolution failure only disables review triggers + +- **GIVEN** the sender cannot be resolved from either the app slug or `GET /user` +- **WHEN** the script applies decisions +- **THEN** label synchronization proceeds and each suppressed review trigger surfaces a warning naming the unresolved sender + +#### Scenario: Fallback token activation suppresses review triggers + +- **GIVEN** the run has switched to `GH_FALLBACK_TOKEN` after rate-limit exhaustion +- **WHEN** the script would trigger a missing Codex review +- **THEN** it skips the `@codex review` post with a warning, because the comment author would no longer match the resolved sender + +#### Scenario: The review-request POST is not retried under the fallback identity + +- **GIVEN** the review-request comment POST itself hits the primary token's rate limit +- **WHEN** the fallback token activates +- **THEN** the POST fails instead of being silently retried under the fallback identity, while later calls use the fallback token + +#### Scenario: Apply status is attributed to the applied pull request + +- **WHEN** the script applies decisions for multiple pull requests in one run +- **THEN** each status line and error report references the pull request of the decision being applied + +### Requirement: Apply-time reclassification + +Before performing writes for a classified decision (label changes, legacy label removal, workflow-run approvals, or review triggers), the Codex label synchronization script MUST reclassify the pull request and act on the fresh evidence only. If the head SHA no longer matches the SHA the decision was classified against, the decision MUST be skipped with a warning. If the head is unchanged but the evidence changed (checks, reviews, mergeability), the writes MUST follow the fresh decision, and a review trigger MUST only fire when both the original and the fresh classification want it. The freshly read timeline MUST feed the shared usage-limit backoff so quota replies that arrived after bulk classification suppress the remaining review requests. Reclassification read failures MUST honor `--tolerate-read-errors` (log and skip the decision without failing the run). Decisions without pending writes need not be reclassified. + +#### Scenario: Stale decision is skipped after a head move + +- **GIVEN** a pull request whose head changed between classification and apply +- **WHEN** the script reaches that decision in the apply loop +- **THEN** it skips all writes for the decision and warns that the head moved + +#### Scenario: Same-head evidence changes are applied fresh + +- **GIVEN** a pull request whose head is unchanged but where Codex raised a new finding after classification +- **WHEN** the script reaches that decision in the apply loop +- **THEN** the writes reflect the fresh classification instead of the superseded one + +#### Scenario: Fresh quota evidence suppresses later triggers + +- **GIVEN** a quota reply that arrived between bulk classification and apply-time reclassification of one pull request +- **WHEN** later decisions in the run would trigger missing Codex reviews +- **THEN** the reclassified timeline has latched the backoff and those posts are skipped + +#### Scenario: Reclassification honors tolerant reads + +- **GIVEN** a run with `--tolerate-read-errors` +- **WHEN** apply-time reclassification of one pull request fails with a GitHub read error +- **THEN** the decision is logged and skipped without failing the run + diff --git a/openspec/specs/graceful-shutdown/spec.md b/openspec/specs/graceful-shutdown/spec.md new file mode 100644 index 0000000000..852e4d333d --- /dev/null +++ b/openspec/specs/graceful-shutdown/spec.md @@ -0,0 +1,233 @@ +# graceful-shutdown Specification + +## Purpose +Ordered process drain: a pre-connection barrier, WebSocket admission closure, and finalization of active turns so shutdown never strands in-flight work or settlement. +## Requirements +### Requirement: Process shutdown establishes a pre-connection drain barrier + +The project-owned Uvicorn server MUST commit graceful drain before Uvicorn closes HTTP or WebSocket connections. A deadline-bearing preStop request or the first shutdown transition MUST establish one monotonic application drain deadline; later SIGTERM, shutdown, or lifespan transitions MUST reuse that deadline without extending it. A delayed preStop request MAY tighten a signal-committed deadline only to preserve the earlier absolute deadline carried by that request. A headerless operator drain MUST remain reversible. Once process shutdown is committed, operator actions MUST NOT reopen admission or erase its deadline. + +#### Scenario: Direct SIGTERM precedes connection shutdown + +- **WHEN** the process receives SIGTERM with an admitted Responses turn active and no prior preStop request +- **THEN** drain admission closes before Uvicorn invokes connection shutdown +- **AND** Uvicorn waits for the turn within the remaining application deadline + +#### Scenario: preStop is followed by SIGTERM + +- **WHEN** preStop starts drain and SIGTERM arrives later +- **THEN** SIGTERM reuses the original absolute deadline +- **AND** local preStop request latency has already consumed that deadline +- **AND** it does not start a second drain period + +#### Scenario: SIGTERM overtakes a deadline-bearing preStop request + +- **WHEN** preStop anchors an absolute deadline and SIGTERM commits a later deadline before the local start request is handled +- **THEN** the accepted preStop deadline tightens the committed process deadline +- **AND** every later shutdown stage uses that earlier absolute value +- **AND** an in-flight drain wait that already started re-reads and adopts that earlier deadline + +#### Scenario: SIGTERM interleaves with deadline initialization + +- **WHEN** a drain-start invocation observes no prior drain and a synchronous shutdown signal commits between its later state operations +- **THEN** both invocations publish monotonic deadline candidates +- **AND** every drain stage uses the earlier candidate +- **AND** neither stale continuation can extend the effective deadline + +#### Scenario: Drain stop races committed shutdown + +- **WHEN** an operator drain stop interleaves with process shutdown commitment +- **THEN** committed WebSocket and HTTP admission remains closed +- **AND** the committed deadline remains available to every later drain stage + +#### Scenario: Signal commit precedes matching lifespan startup + +- **WHEN** a server start prepares shutdown state and SIGTERM commits the barrier before its application lifespan begins +- **THEN** matching lifespan startup preserves the committed barrier and deadline +- **AND** it does not reset process shutdown state + +#### Scenario: Completed embedded lifespan is followed by a new start + +- **WHEN** an embedded application lifespan has completed and another embedded lifespan starts in the same process +- **THEN** the new lifecycle starts with admission open and no inherited committed deadline + +### Requirement: Graceful drain closes WebSocket admission + +Once graceful drain begins, the application MUST reject every new external WebSocket connection before invoking the route handler. A Responses WebSocket scope admitted before the barrier MUST remain tracked until its handler exits. Other WebSocket protocols MUST receive the same late-admission rejection but MUST NOT hold the Responses in-flight counter for their full connection lifetime. + +#### Scenario: New WebSocket arrives during drain + +- **WHEN** a new WebSocket connection scope arrives after drain has begun +- **THEN** the application rejects the connection without invoking its route handler +- **AND** the rejected connection does not increase the in-flight count + +#### Scenario: WebSocket crosses the drain barrier + +- **WHEN** a Responses WebSocket scope is admitted immediately before drain begins +- **THEN** it remains in the in-flight count until its route handler exits +- **AND** shutdown waits for that scope within the configured drain timeout + +#### Scenario: Realtime or Live connection predates drain + +- **WHEN** a non-Responses WebSocket scope is admitted before drain +- **THEN** it does not hold the Responses in-flight counter +- **AND** normal Uvicorn connection shutdown remains its lifecycle bound + +### Requirement: Graceful drain preserves active WebSocket turn finalization + +An admitted Responses WebSocket connection MUST stop accepting new `response.create` turns after drain begins. An idle connection MUST close promptly, while a connection with an already registered active turn MUST remain open until that turn reaches terminal downstream delivery, request-log persistence ownership, and API-key settlement ownership or the shared drain deadline expires. + +#### Scenario: Idle admitted WebSocket observes drain + +- **WHEN** drain begins while an admitted Responses WebSocket has no active turn +- **THEN** the server closes that connection promptly +- **AND** the connection no longer holds the shutdown drain + +#### Scenario: Active turn completes during drain + +- **WHEN** drain begins while an admitted Responses WebSocket turn is active +- **THEN** the turn continues through terminal downstream delivery, request logging, and API-key settlement +- **AND** the WebSocket closes after the active turn is finalized + +#### Scenario: Existing connection submits a new turn during drain + +- **WHEN** an admitted Responses WebSocket submits a new `response.create` after drain begins +- **THEN** the server rejects that turn locally +- **AND** the turn is not registered or sent upstream + +#### Scenario: Upstream clean close races a new turn + +- **WHEN** an upstream transport-end frame is received while a new turn is blocked in admission or account ownership +- **THEN** the connection is synchronously marked for reconnect before that attribution wait +- **AND** the reader does not fail or replay-mutate the still-unsent turn +- **AND** the sender checks the latch after all admission and account awaits immediately before send +- **AND** the turn is sent exactly once on a fresh upstream socket +- **AND** the retired account-local create lease is released and the fresh account re-acquires its own lease +- **AND** no post-send replay budget is consumed + +#### Scenario: Generic send failure races a reader-owned clean-close replay + +- **WHEN** a generic upstream send failure occurs after the reader has classified the same sent turn as clean-close replayable +- **THEN** the sender marks the connection for reconnect before cancelling and awaiting the reader +- **AND** harvests the reader's published replay owner before retiring its control state +- **AND** releases the retired account-local create lease before a replacement account acquires its own lease +- **AND** sends the turn exactly once on the replacement socket +- **AND** produces exactly one terminal event, one terminal request log, and one API-key settlement + +#### Scenario: Typed transport send failure races a reader claim + +- **WHEN** a typed transport send failure occurs after the reader has claimed the same sent turn +- **THEN** the sender does not replay the ambiguously delivered turn +- **AND** transfers the reader claim to one registered finalization task before awaiting it +- **AND** produces exactly one `response.failed` with the typed transport error +- **AND** releases or settles the API-key reservation and persists the terminal request log exactly once + +### Requirement: Terminal WebSocket work has explicit cancellation-safe ownership + +After an upstream message is received, processing and downstream delivery MUST be owned by a registered task before terminal handling can remove its request state from the pending queue. Reader or scope cancellation MUST NOT orphan that task. The reader MUST wait for owned terminal work only within the remaining shared application deadline; when no application drain is active, normal scope cancellation MUST instead use the existing bounded task-cancellation timeout. Shutdown persistence drain MUST observe both terminal-message and transport-end child tasks plus any request-log or settlement follow-up work they create. + +#### Scenario: Cancellation lands after terminal state leaves pending + +- **WHEN** a terminal event removes its request state from the pending queue and the upstream reader is then cancelled before settlement or downstream delivery completes +- **THEN** the reader waits for the owned terminal task within the remaining shared application deadline before propagating cancellation +- **AND** actual usage is settled exactly once +- **AND** exactly one terminal event and one terminal request log are produced + +#### Scenario: Cancellation lands before a terminal event + +- **WHEN** a Responses scope is cancelled while its request state remains pending or staged for transparent replay +- **THEN** its API-key reservation is released exactly once +- **AND** exactly one cancelled request log is produced against the last upstream account that owned the turn + +#### Scenario: Cancellation lands after a pending batch is claimed + +- **WHEN** terminal cleanup removes pending request states from the shared queue +- **THEN** it atomically transfers them to a registered, shielded finalization task before releasing the queue lock +- **AND** caller cancellation waits only within the remaining shared deadline without cancelling that sole child owner +- **AND** persistence drain continues to observe the child +- **AND** each request releases its turn admission, account-local create lease, API-key reservation, and create gate and persists its terminal log exactly once + +#### Scenario: Pending account-health failure preserves settlement ordering + +- **GIVEN** a keyed WebSocket turn is claimed by terminal cleanup +- **WHEN** the failure would record an account-health penalty +- **THEN** every claimed API-key reservation release commits before the account-health write +- **AND** every claimed terminal request log is handed to tracked persistence before the account-health write +- **AND** a failed or indeterminate reservation release prevents that account-health write + +#### Scenario: Upstream terminal event preserves account-health ordering + +- **GIVEN** a keyed WebSocket turn receives its upstream terminal event +- **WHEN** finalization would write account health +- **THEN** API-key settlement commits and the terminal request log is handed to tracked persistence before that health write +- **AND** a failed or indeterminate settlement or request-log handoff prevents the health write +- **AND** an account-health write failure does not prevent terminal downstream delivery + +#### Scenario: Reader cancellation occurs outside process drain + +- **WHEN** a reader is cancelled while its owned terminal task ignores cancellation and no application drain deadline exists +- **THEN** the reader waits only for the existing bounded task-cancellation timeout +- **AND** the owned task remains registered for eventual result consumption + +#### Scenario: Active turn exceeds the shared deadline + +- **WHEN** terminal processing remains blocked past the application drain deadline +- **THEN** Uvicorn proceeds with bounded connection and task shutdown +- **AND** the process does not start another application drain timeout + +#### Scenario: Cancelled reader requires transport close + +- **WHEN** scope cleanup cancels an upstream reader whose receive operation waits for transport close before propagating cancellation +- **THEN** cleanup first transfers its local replay and request-state owners to one registered finalization task +- **AND** requests reader cancellation exactly once +- **AND** closes the upstream transport before awaiting the already-cancelled reader +- **AND** bounds that await, lease release, and remaining terminal cleanup by the shared deadline +- **AND** expiry of the caller's wait does not cancel that sole cleanup owner +- **AND** produces exactly one terminal request log and one reservation settlement or release + +#### Scenario: Connection lease release fails during scope cancellation + +- **WHEN** scope cancellation owns a pending turn and releasing the upstream connection lease fails +- **THEN** request finalization completes before connection-lease release is attempted +- **AND** turn admission, account-local create lease, API-key reservation, create gate, and terminal request log are finalized exactly once +- **AND** the lease failure is reported without replacing the original scope cancellation + +### Requirement: Owned launchers preserve shutdown semantics + +The project CLI MUST use the pre-connection drain server with exactly one worker per process while preserving Uvicorn's startup-failure exit status and clean KeyboardInterrupt behavior. Every supported server launch path shipped or documented by the project MUST delegate to that owned CLI rather than invoking raw FastAPI or Uvicorn startup. Development Compose source synchronization MUST restart the owned server instead of relying on Uvicorn's incompatible reload launcher. Ambient `WEB_CONCURRENCY` MUST NOT create an unsupported multiprocess launch. The project MUST declare a Uvicorn version whose launcher API includes `Config.load_app()`. An embedded metrics server MUST NOT replace the main server's process signal handlers. During shutdown, after the shared application drain deadline, the owned server MUST stop awaiting Uvicorn connection and lifespan cleanup after 25 seconds. If that bound expires, it MUST terminate with the most recently captured shutdown signal, or SIGTERM when shutdown was programmatic, rather than return a cancellation-resistant cleanup task to asyncio runner teardown. + +#### Scenario: Lifespan startup fails + +- **WHEN** Uvicorn does not reach its started state +- **THEN** the project CLI exits with Uvicorn's startup-failure status + +#### Scenario: Metrics endpoint is enabled + +- **WHEN** the main process starts the embedded metrics server +- **THEN** only the main application server owns SIGTERM and SIGINT handlers + +#### Scenario: Launcher dependency is resolved + +- **WHEN** the project runtime dependencies are resolved +- **THEN** Uvicorn versions older than 0.47.0 are rejected + +#### Scenario: Ambient worker count is greater than one + +- **WHEN** `WEB_CONCURRENCY` requests multiple workers +- **THEN** the owned launcher still starts exactly one worker for the instance + +#### Scenario: Operator follows a shipped or documented launch path + +- **WHEN** the server is started through a project Compose file or documented local command +- **THEN** that path delegates to the owned pre-connection drain launcher +- **AND** direct SIGTERM commits the barrier before Uvicorn closes connections +- **AND** development source synchronization restarts that owned launcher + +#### Scenario: Lifespan cleanup blocks after application drain + +- **WHEN** Uvicorn connection or lifespan cleanup remains blocked after the shared drain phase +- **THEN** the owned launcher cancels and stops waiting after 25 seconds +- **AND** terminates with the most recently captured signal, or SIGTERM when no signal initiated shutdown +- **AND** does not leave cancellation-resistant cleanup registered for unbounded asyncio runner teardown +- **AND** Helm termination grace reserves two seconds for failed preStop start plus 30 seconds after the application deadline, leaving five seconds after the cleanup bound for process exit before SIGKILL + diff --git a/openspec/specs/http-ingress-limits/spec.md b/openspec/specs/http-ingress-limits/spec.md new file mode 100644 index 0000000000..5798af560c --- /dev/null +++ b/openspec/specs/http-ingress-limits/spec.md @@ -0,0 +1,172 @@ +# http-ingress-limits Specification + +## Purpose +Incremental, budget-reusing bounds on raw HTTP request ingress, including encoded bodies before and after decompression and exact route-owned multipart exceptions. +## Requirements +### Requirement: Raw HTTP request ingress is bounded incrementally + +The service MUST enforce the applicable request-body budget against actual raw bytes received for each guarded HTTP request. It MUST reject the request before exposing a chunk that would make the cumulative raw body exceed the budget, and it MUST NOT prebuffer the complete body solely to enforce this limit. + +#### Scenario: Declared oversized body is rejected before downstream parsing + +- **WHEN** a guarded HTTP request declares a valid `Content-Length` greater than its applicable budget +- **THEN** the service returns HTTP 413 without invoking downstream request-body parsing + +#### Scenario: Chunked body crosses the budget + +- **WHEN** a guarded HTTP request has no usable `Content-Length` and its received chunks cumulatively exceed the applicable budget +- **THEN** the service returns HTTP 413 +- **AND** the chunk that crosses the budget is not exposed to downstream body parsing + +#### Scenario: Exact-boundary body is accepted by the ingress guard + +- **WHEN** a guarded HTTP request's actual raw body size equals its applicable budget +- **THEN** the raw ingress guard allows the complete body to continue downstream + +#### Scenario: Client disconnect remains a disconnect + +- **WHEN** the ASGI server reports `http.disconnect` while a guarded body is being received +- **THEN** the ingress guard propagates the disconnect without converting it into an HTTP 413 response + +### Requirement: HTTP ingress reuses existing budgets + +The service MUST use `max_decompressed_body_bytes` as the general raw and decompressed HTTP request-body budget. When an owning route capability defines a larger budget from an existing route-specific setting, the ingress guard MUST use that route budget. The HTTP ingress guard MUST NOT add another setting or change existing defaults. + +Route-specific budget and error-envelope selection MUST use the application-relative route path after removing any matching ASGI `root_path` prefix. + +The generic guard MUST apply to requests solely because they declare `multipart/form-data`; the client-declared media type MUST NOT grant an exemption. An owning route capability MAY define an exact method/path-scoped authorization-before-read contract and dedicated bounded multipart parser. Only unencoded multipart requests to that exact operation, or requests marked by its outer content-encoding gate, MAY bypass generic admission. The gate MUST identify its operation independently of the declared media type, remove the encoding and mark the scope as handled without consuming the body, and the exception MUST NOT apply to any other operation. + +#### Scenario: Another HTTP path uses the general budget + +- **WHEN** a guarded request targets any other HTTP path +- **THEN** its raw and decompressed HTTP ingress budget is `max_decompressed_body_bytes` + +#### Scenario: Route-owned unencoded multipart uses dedicated admission + +- **GIVEN** an exact operation has a capability-defined authorization-before-read contract and dedicated bounded multipart parser +- **WHEN** an unencoded request to that operation declares media type `multipart/form-data` +- **THEN** the generic raw whole-body guard does not preempt operation authorization or its dedicated parser limit + +#### Scenario: Unrelated unencoded multipart remains guarded + +- **WHEN** an unencoded request outside a route-owned multipart operation declares media type `multipart/form-data` +- **THEN** the service applies the generic raw-body budget +- **AND** the declared media type alone does not bypass admission + +#### Scenario: Encoded multipart remains guarded + +- **WHEN** a `multipart/form-data` request outside a route-owned multipart exception carries a `Content-Encoding` header +- **THEN** the service applies both the raw and decompressed budget checks + +#### Scenario: Route-owned multipart admission can preserve authorization precedence + +- **GIVEN** an exact operation has a capability-defined outer content-encoding gate, authorization-before-read contract, and dedicated bounded multipart parser +- **WHEN** an encoded request targets that operation, regardless of its declared media type +- **THEN** the generic raw and decompressed-body guards do not preempt operation authorization or its dedicated parser limit +- **AND** encoded multipart requests to all other operations remain guarded + +#### Scenario: Mounted Responses route keeps its route-specific policy + +- **GIVEN** the service is mounted under a non-empty ASGI `root_path` +- **WHEN** the request scope path includes that prefix and targets `/v1/responses` relative to the application +- **THEN** the service applies the Responses-specific ingress budget +- **AND** any ingress failure uses the OpenAI-compatible error envelope + +### Requirement: Encoded HTTP bodies are bounded before and after decompression + +For request bodies using `gzip`, `deflate`, `zstd`, `identity`, or supported stacked `Content-Encoding` values that remain under generic ingress admission, the service MUST enforce the applicable budget independently against the encoded raw body and every intermediate and final decoded representation. The service MUST remove stacked encodings in reverse header/application order. Unsupported encodings or malformed compressed bodies under generic admission MUST fail with HTTP 400. Exact route-owned exceptions MUST instead follow their owning capability's authorization and encoded-body rejection contract. + +#### Scenario: Encoded raw body exceeds the budget + +- **WHEN** a generic-guarded encoded request's raw bytes exceed the applicable budget before decompression +- **THEN** the service returns HTTP 413 before attempting to hold an unbounded encoded body + +#### Scenario: Expanded body exceeds the budget + +- **WHEN** a generic-guarded encoded request is within the raw budget but expands beyond the applicable decompressed budget +- **THEN** the service returns HTTP 413 + +#### Scenario: Supported stacked encoding remains compatible + +- **WHEN** a generic-guarded request uses a valid supported stack of `gzip`, `deflate`, `zstd`, or `identity` encodings and both representations fit the budget +- **THEN** the service decodes the body in reverse header/application order, caps every intermediate representation, and continues request handling + +#### Scenario: Invalid compression is rejected + +- **WHEN** a generic-guarded request uses an unsupported content encoding or carries malformed compressed bytes +- **THEN** the service returns HTTP 400 without invoking route logic + +### Requirement: HTTP ingress failures use the path-family error envelope + +Ingress failures on `/v1/*`, `/backend-api/*`, `/api/codex/*`, and `/internal/bridge/*` MUST use an OpenAI-compatible error envelope with `type = invalid_request_error`. Equivalent paths MUST be classified after the existing outer path canonicalization. Other ingress paths MUST retain the dashboard-compatible error envelope. Oversized requests MUST use `code = payload_too_large`; malformed or unsupported compression MUST use `code = invalid_request_error` on OpenAI paths and `code = invalid_request` on other paths. + +#### Scenario: OpenAI path rejects an oversized body + +- **WHEN** a raw or decompressed request body on an OpenAI-compatible proxy path exceeds its budget +- **THEN** the service returns HTTP 413 +- **AND** the response has OpenAI error `code = payload_too_large` and `type = invalid_request_error` + +#### Scenario: OpenAI path rejects invalid compression + +- **WHEN** a request on an OpenAI-compatible proxy path uses unsupported or malformed compression +- **THEN** the service returns HTTP 400 +- **AND** the response has OpenAI error `code = invalid_request_error` and `type = invalid_request_error` + +#### Scenario: Dashboard settings path rejects an oversized body + +- **WHEN** a raw or decompressed request body on `/api/settings` exceeds its budget +- **THEN** the service returns HTTP 413 +- **AND** the response has dashboard error `code = payload_too_large` + +#### Scenario: Dashboard settings path rejects invalid compression + +- **WHEN** a request on `/api/settings` uses unsupported or malformed compression +- **THEN** the service returns HTTP 400 +- **AND** the response has dashboard error `code = invalid_request` + +#### Scenario: Duplicated Codex alias is classified after canonicalization + +- **WHEN** an ingress failure targets `/backend-api/codex/v1/responses/` +- **THEN** the service applies the same Responses budget and OpenAI-compatible envelope as `/backend-api/codex/responses/` + +### Requirement: Ingress admission preserves endpoint authorization + +The HTTP ingress guard MUST NOT authenticate callers or replace, bypass, or relocate existing dashboard, proxy API-key, ChatGPT-identity, or internal-bridge authorization. Requests that reach dependency resolution MUST continue through the endpoint's existing authorization path. Existing FastAPI parsing order remains unchanged, so ingress rejection or syntactically invalid typed bodies can fail before router-level authorization. + +#### Scenario: Admitted unauthenticated request still reaches proxy authorization + +- **WHEN** a syntactically valid under-limit request without required credentials targets an API-key-protected proxy route +- **THEN** the ingress guard allows normal routing to continue +- **AND** the existing proxy authorization rejects the request with its established authentication response + +#### Scenario: Declared oversized generic-guarded request fails before authorization + +- **WHEN** a guarded request outside a route-owned admission exception declares a body larger than its ingress budget +- **THEN** the service returns the deterministic ingress 413 without invoking router-level authorization + +### Requirement: Multipart ingress exceptions are exact and route-owned + +The generic raw HTTP body guard MUST exempt an unencoded `multipart/form-data` request only when the request is `POST` to `/api/accounts/import`, `/backend-api/transcribe`, `/v1/audio/transcriptions`, or `/v1/images/edits`, including their application-relative trailing-slash and mounted equivalents. Each exempt operation MUST apply its capability-defined authorization-before-read contract and dedicated bounded multipart parser. + +For the same exact operations, an outer content-encoding gate MUST remove `Content-Encoding` and mark the copied request scope as route-owned without reading the body. The generic raw and decompression guards MUST honor that internal marker regardless of the declared media type so `identity` reaches dedicated multipart admission and non-identity encoding reaches the post-authorization rejection path. + +The client-declared multipart media type MUST NOT exempt any other method or path. Every unrelated unencoded or encoded multipart request MUST remain under generic raw admission, and encoded requests MUST also retain generic decompressed-body admission. + +#### Scenario: Exact unencoded multipart operation uses its dedicated parser + +- **WHEN** an unencoded multipart request targets one of the four exact route-owned `POST` operations +- **THEN** generic admission does not preempt operation authorization +- **AND** the operation's dedicated multipart body limit remains authoritative + +#### Scenario: Exact encoded operation preserves authorization precedence + +- **WHEN** a request with `Content-Encoding` targets one of the four exact route-owned `POST` operations and declares either multipart or another media type +- **THEN** the outer gate marks the request without consuming its body +- **AND** generic raw or decompressed-body admission does not preempt operation authorization or its encoded-body contract + +#### Scenario: Unrelated multipart media type grants no exemption + +- **WHEN** an unencoded or encoded request outside the four exact route-owned `POST` operations declares `multipart/form-data` +- **THEN** the generic raw-body budget remains enforced +- **AND** an oversized declared body is rejected before downstream parsing + diff --git a/openspec/specs/images-api-compat/spec.md b/openspec/specs/images-api-compat/spec.md index a9e242af5c..4db6021a62 100644 --- a/openspec/specs/images-api-compat/spec.md +++ b/openspec/specs/images-api-compat/spec.md @@ -89,7 +89,16 @@ When a client requests `stream=true` on `/v1/images/generations` or `/v1/images/ ### Requirement: Image routes participate in usage accounting and policy -The system SHALL apply API-key allowed-model policy and model-scoped usage limits to `/v1/images/*` using the publicly-requested `gpt-image-*` value as the effective model. The system SHALL record the publicly-requested `gpt-image-*` value (not the internal host model) in the request log's `model` column once the upstream response id becomes known. +The system SHALL apply API-key allowed-model policy and model-scoped usage +limits to `/v1/images/*` using the publicly-requested `gpt-image-*` value as the +effective model. The system SHALL record the publicly-requested `gpt-image-*` +value (not the internal host model) in the request log's `model` column once the +upstream response id becomes known. A successful image generation or edit that +owns a limited API-key reservation SHALL transfer that reservation exactly once +to persistence-drained settlement using captured `tool_usage.image_gen` tokens, +while the internal Responses stream SHALL NOT receive a second settlement +owner. Failed or cancelled finalization SHALL preserve the completed public +image response and transfer ownership to the tracked retrying release fallback. #### Scenario: API key allowed-model policy blocks gpt-image-2 @@ -101,6 +110,17 @@ The system SHALL apply API-key allowed-model policy and model-scoped usage limit - **WHEN** an `/v1/images/*` request completes successfully against an internal host Responses model (for example `gpt-5.5`) - **THEN** the resulting `request_logs` row has `model` equal to the publicly requested value (for example `gpt-image-2`) so dashboards and usage views surface the user-visible model rather than the internal host model +#### Scenario: Failed image-token settlement retains tracked release ownership + +- **GIVEN** a limited API key owns a reservation for a successful image generation or edit request +- **AND** the internal Responses stream receives no API-key reservation +- **AND** the image adapter captures authoritative `tool_usage.image_gen` tokens +- **WHEN** tracked finalization fails or is cancelled while the reservation remains `reserved` +- **THEN** the completed public Images JSON response or SSE completion remains available +- **AND** settlement ownership transfers to a persistence-drained fallback release task +- **AND** transient release failures keep that task tracked and retrying until release succeeds or graceful persistence drain reports timeout +- **AND** a successful fallback restores pre-reserved quota exactly once without recording `response.usage` or starting a second image settlement + ### Requirement: Image routes expose bounded operational observability The system SHALL emit structured route-completion logs and Prometheus metrics for `/v1/images/generations` and `/v1/images/edits`. Observability labels MUST be bounded to route, effective public model, stream flag, HTTP status, and outcome, and MUST NOT include prompts, image bytes, file names, access tokens, or raw upstream payloads. @@ -154,3 +174,71 @@ surface. (log and metrics) with the same `generations`/`edits` route label as the `/v1` counterpart, exactly once +### Requirement: Image edit multipart uploads are authorized and bounded + +`POST /v1/images/edits` MUST complete its existing proxy authorization dependencies before reading multipart body bytes. It MUST accept at most 16 source-image file parts across `image` and `image[]`, at most one `mask`, no unknown file-part names, no more than 32 text fields of at most 256 KiB each, every individual file smaller than 50,000,000 bytes, fewer than 50,000,000 bytes across all source images and the mask, and a complete multipart body no greater than 64 MiB (67,108,864 bytes). + +The service MUST enforce the body limit against both a usable declared `Content-Length` and actual streamed bytes. It MUST enforce file, aggregate-binary, and text limits before retaining crossing bytes, close multipart spools before usage reservation, account selection, base64 conversion, or internal Responses forwarding, and add no new runtime setting. + +This route-owned policy MUST take precedence over the generic raw HTTP body budget for `POST /v1/images/edits`. Its exact-path content-encoding gate MUST run outside the generic raw and decompression guards regardless of the declared media type. Requests handled by that gate, and unencoded requests declared as multipart, MUST NOT be rejected by the generic guards before proxy authorization or the dedicated parser applies this capability's body limit. An unencoded request that does not declare multipart remains under generic admission and MAY be rejected there before authorization. This exception MUST NOT change generic ingress behavior for any other operation. + +Byte-limit failures MUST return HTTP 413 with OpenAI error `code = payload_too_large` and `type = invalid_request_error`; a known file-part failure MUST set `param = image` or `param = mask`. Multipart syntax, count, and required-field failures MUST retain OpenAI-compatible invalid-request behavior. Every parser rejection MUST emit exactly one bounded image-route observation with HTTP status and `outcome = invalid_request`. + +#### Scenario: Unauthorized image edit does not consume the body + +- **WHEN** an image-edit request fails the existing proxy API-key authorization +- **THEN** the authentication response is returned before the ASGI request body is consumed +- **AND** no multipart temporary file is created +- **AND** exactly one auth-error route observation is recorded without parsing the multipart body, using bounded pre-parse labels + +#### Scenario: Bounded image edit remains compatible + +- **WHEN** an authorized image-edit request supplies at least one source image, an optional mask, required text fields, and all parts are within their limits +- **THEN** the service preserves the ordered `image` and `image[]` bytes, content types, mask, and validated form fields through the existing image-edit pipeline + +#### Scenario: Source image count combines canonical and bracketed keys + +- **WHEN** the combined number of `image` and `image[]` file parts exceeds 16, the request contains more than one `mask`, or an unknown file-part name is present +- **THEN** the service returns an OpenAI-compatible HTTP 400 invalid-request response +- **AND** no image bytes are base64-encoded or forwarded internally + +#### Scenario: Declared or streamed image-edit body exceeds its limit + +- **WHEN** a usable `Content-Length` exceeds 64 MiB or actual streamed multipart bytes cross 64 MiB +- **THEN** the service returns HTTP 413 with OpenAI error `code = payload_too_large` and `type = invalid_request_error` +- **AND** no usage reservation, account selection, base64 conversion, or internal Responses request occurs + +#### Scenario: Image binary limit is exceeded + +- **WHEN** one source image or mask reaches 50,000,000 bytes, or their combined binary bytes reach 50,000,000 +- **THEN** the service returns HTTP 413 with OpenAI error `code = payload_too_large`, `type = invalid_request_error`, and the applicable `image` or `mask` parameter +- **AND** bytes beyond the applicable limit are not retained in a spool or handler buffer + +#### Scenario: Image text-field resources are bounded + +- **WHEN** an image-edit request exceeds 32 text fields or 256 KiB in any text part +- **THEN** the service rejects the request with the documented OpenAI-compatible count or byte-limit response +- **AND** it records one invalid-request route observation without invoking image-edit route logic + +#### Scenario: Compressed image edit is rejected without prebuffering + +- **GIVEN** image edit has passed proxy authorization +- **WHEN** it declares a non-identity `Content-Encoding` +- **THEN** the service returns HTTP 400 with OpenAI error `code = invalid_request_error` and `type = invalid_request_error` before reading the request body +- **AND** a no-op `identity` encoding is handled as an ordinary multipart request governed by the 64 MiB dedicated body limit +- **AND** exactly one invalid-request route observation is recorded without parsing the multipart body + +#### Scenario: Generic ingress does not preempt encoded image-edit authorization + +- **GIVEN** an image-edit request fails proxy authorization +- **WHEN** it declares a non-identity `Content-Encoding` and a `Content-Length` greater than the generic raw HTTP budget +- **THEN** the existing authentication response is returned instead of a generic HTTP 413 or encoded-body HTTP 400 +- **AND** the request body is not consumed +- **AND** exactly one auth-error route observation is recorded + +#### Scenario: Image-edit cleanup preserves transport failures + +- **WHEN** parsing succeeds, fails a limit, encounters malformed multipart, receives a client disconnect, or is cancelled +- **THEN** every created multipart spool is closed +- **AND** disconnect and cancellation are not converted to HTTP 413 + diff --git a/openspec/specs/model-catalog-compat/spec.md b/openspec/specs/model-catalog-compat/spec.md index fd742b4043..65417be830 100644 --- a/openspec/specs/model-catalog-compat/spec.md +++ b/openspec/specs/model-catalog-compat/spec.md @@ -169,33 +169,76 @@ When serving `GET /v1/models`, the system SHALL preserve upstream speed-tier met ### Requirement: GPT-5.6 bootstrap metadata matches the upstream bundled catalog -The GPT-5.6 bootstrap catalog entries (`gpt-5.6-sol`, `gpt-5.6-terra`, `gpt-5.6-luna`) MUST mirror the upstream bundled catalog (`codex-rs/models-manager/models.json` at codex release rust-v0.144.1) field-for-field for every metadata field codex-lb serves. In particular each -entry MUST carry: `context_window` and `max_context_window` of `372000`; -`minimal_client_version` `"0.144.0"`; `tool_mode` `"code_mode_only"`; -`use_responses_lite` `true`; `apply_patch_tool_type` `"freeform"`; -`web_search_tool_type` `"text_and_image"`; `supports_image_detail_original` -`true`; `truncation_policy` `{"mode": "tokens", "limit": 10000}`; -`comp_hash` `"3000"`; `reasoning_summary_format` `"experimental"`; -`default_reasoning_summary` `"none"`; `include_skills_usage_instructions` -`false`; `experimental_supported_tools` `[]` (a field the Codex client's -deserializer requires); `supports_search_tool` `true`; `additional_speed_tiers` -`["fast"]`; the `priority`/`Fast` service tier entry; `shell_type` -`"shell_command"`; `prefer_websockets` `true`; and the 21-plan -`available_in_plans` list upstream advertises (including `edu_plus`, +The GPT-5.6 bootstrap catalog entries (`gpt-5.6-sol`, `gpt-5.6-terra`, +`gpt-5.6-luna`) MUST mirror the upstream bundled catalog +(`codex-rs/models-manager/models.json` at Codex release `rust-v0.145.0`) +field-for-field for every metadata field codex-lb serves, with one tracked +exception: `max_context_window`, which upstream raised from `272000` to +`872000` in openai/codex commit +`2eee483e49f88b868f67364134a658b3298e6c14` (openai/codex#39102) and which no +`rust-v*` release tag carries as of `rust-v0.148.0-alpha.21`. In particular +each entry MUST carry: `context_window` of `272000` and `max_context_window` +of `872000`; `minimal_client_version` `"0.144.0"`; `tool_mode` +`"code_mode_only"`; `use_responses_lite` `true`; `apply_patch_tool_type` +`"freeform"`; `web_search_tool_type` `"text_and_image"`; +`supports_image_detail_original` `true`; `truncation_policy` `{ "mode": +"tokens", "limit": 10000 }`; `comp_hash` `"3000"`; `reasoning_summary_format` +`"experimental"`; `default_reasoning_summary` `"none"`; +`include_skills_usage_instructions` `false`; `experimental_supported_tools` +`[]` (a field the Codex client's deserializer requires); `supports_search_tool` +`true`; `additional_speed_tiers` `["fast"]`; the `priority`/`Fast` service tier +entry; `shell_type` `"shell_command"`; `prefer_websockets` `true`; and the +21-plan `available_in_plans` list upstream advertises (including `edu_plus`, `edu_pro`, `enterprise_cbp_automation`, and `sci`). `multi_agent_version` MUST be `"v2"` for Sol and Terra and `"v1"` for Luna. Sol MUST carry the upstream -`availability_nux` message while Terra and Luna carry `null`. Default -reasoning levels MUST be `low` for Sol and `medium` for Terra and Luna, and +`availability_nux` message while Terra and Luna carry `null`. Default reasoning +levels MUST be `low` for Sol and `medium` for Terra and Luna, and reasoning-level descriptions MUST be the verbatim upstream strings. +`context_window` is the default input budget and `max_context_window` is the +ceiling a client may opt into; the two MUST NOT be collapsed into one value +for these entries. + The ~16.5 KB upstream `base_instructions` prompt and the personality-templated `model_messages` object are deliberately NOT bundled in the bootstrap catalog; the first successful live registry refresh supplies them. This is the only -sanctioned divergence from the upstream GPT-5.6 entries. +sanctioned divergence from the upstream GPT-5.6 entries beyond the +`max_context_window` exception above. + +#### Scenario: GPT-5.6 bootstrap entries retain the corrected upstream context budget + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` report + `context_window=272000` + +#### Scenario: GPT-5.6 bootstrap entries advertise the raised upstream ceiling + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` report + `context_window=272000` +- **AND** each reports `max_context_window=872000` + +#### Scenario: OpenAI-compatible metadata keeps the default input budget + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /v1/models` +- **THEN** each GPT-5.6 entry reports `context_window=272000` and + `input_context_window=272000` +- **AND** the raised Codex-native ceiling is not promoted into the + OpenAI-compatible input budget fields #### Scenario: GPT-5.6 entries expose upstream tool and multi-agent metadata - **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded - **WHEN** a client calls `GET /backend-api/codex/models` - **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` carry `tool_mode: "code_mode_only"`, `use_responses_lite: true`, `experimental_supported_tools: []`, and `minimal_client_version: "0.144.0"` - **AND** `multi_agent_version` is `"v2"` for Sol and Terra and `"v1"` for Luna @@ -203,6 +246,7 @@ sanctioned divergence from the upstream GPT-5.6 entries. #### Scenario: GPT-5.6 entries expose upstream reasoning-summary and plan metadata - **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded - **WHEN** a client calls `GET /backend-api/codex/models` - **THEN** each GPT-5.6 entry carries `default_reasoning_summary: "none"`, `reasoning_summary_format: "experimental"`, and `comp_hash: "3000"` - **AND** each GPT-5.6 entry's `available_in_plans` includes `edu_plus`, `edu_pro`, `enterprise_cbp_automation`, and `sci` @@ -375,6 +419,22 @@ capability. Requests that omit a tier or use the omit-equivalent `auto` or `default` tiers MUST use model-only account filtering, including when reusing an HTTP bridge session. +A service tier imposed by an API key's enforced service tier is not an explicit +request for that tier. When the requested tier originates from API key +enforcement and the model's catalog does not advertise that tier at all, the +system MUST remove the tier from the account-routed request, MUST select +accounts and reuse HTTP bridge sessions using model-only filtering, MUST +reserve, settle, and log API-key usage at the effective default tier, and MUST +omit the unsupported tier from the upstream request. This account-catalog +fallback MUST NOT alter a request selected for an external model source, and an +unknown or account-catalog-absent model MUST retain the enforced tier. When the model's catalog +does advertise the tier, account-level tier filtering MUST continue to apply +regardless of the tier's origin. A tier supplied explicitly by the client MUST +continue to filter accounts even when it equals the enforced value or uses an +equivalent alias and the model does not advertise it. When an +unavailable service tier is what excluded every account, the selection error +MUST name that tier. + #### Scenario: Same-plan accounts expose different models - **GIVEN** two active accounts share a plan @@ -390,6 +450,38 @@ an HTTP bridge session. - **WHEN** a request explicitly asks for priority - **THEN** selection considers only the account that advertised priority +#### Scenario: Enforced tier does not exclude a model that never advertises it + +- **GIVEN** an active account advertises a model at its default tier +- **AND** the model's catalog advertises no `priority` service tier +- **AND** an API key sets `enforced_service_tier` to `priority` +- **WHEN** account selection is requested for that model +- **THEN** the enforced tier is removed from the account-routed request +- **AND** API-key accounting, bridge compatibility, and upstream forwarding use the effective default tier +- **AND** the advertising account is selected + +#### Scenario: Account-catalog fallback does not alter a model source + +- **GIVEN** an API key enforces the `priority` service tier +- **AND** the selected model is routed through an external model source +- **WHEN** the subscription-account catalog does not advertise `priority` for that model +- **THEN** the source-routed request retains `priority` + +#### Scenario: Explicitly requested unadvertised tier is still rejected + +- **GIVEN** an active account advertises a model at its default tier +- **AND** the model's catalog advertises no `priority` service tier +- **WHEN** a client explicitly requests that model with `priority` or an equivalent `fast` alias +- **THEN** no account is selected + +#### Scenario: Unavailable advertised tier names the tier in the error + +- **GIVEN** a model's catalog advertises the `priority` service tier +- **AND** no active account carries `priority` for that model +- **WHEN** account selection is requested for that model with `priority` +- **THEN** no account is selected +- **AND** the selection error names the `priority` service tier + ### Requirement: Unknown account catalogs degrade without false exclusion The system MUST distinguish an account catalog that successfully omitted a @@ -833,3 +925,195 @@ At startup every replica SHALL load the persisted model-registry snapshot into i - **WHEN** the replica loses leadership and its next reconcile runs (poller callback or refresh-tick backstop) - **THEN** it drops the unpublished catalog, reverts to the bootstrap floor, and invalidates its account-selection cache +### Requirement: Every Codex-native catalog entry is wire-parseable + +Every model entry returned by `GET /backend-api/codex/models` or the equivalent +`GET /v1/models?client_version=` route MUST include the non-defaulted +Codex wire fields `truncation_policy` and `experimental_supported_tools`, even +when the entry comes from hidden retained bootstrap metadata or a persisted +legacy registry snapshot. When either field is absent from stored raw metadata, +the mapper MUST provide a conservative model-compatible default. Wire-valid +values provided by a live upstream catalog or model source MUST remain +authoritative and MUST NOT be overwritten by the compatibility defaults. When +`experimental_supported_tools` is not a list, the mapper MUST emit an empty +list. When it contains non-string members, the mapper MUST omit those members +rather than failing the complete catalog. A wire-valid `truncation_policy` MUST +use the `bytes` or `tokens` mode and a JSON integer representable by Codex's +signed 64-bit `limit` field. When an explicit policy does not satisfy that wire +shape, the mapper MUST emit the same conservative model-compatible policy used +when the field is absent. + +#### Scenario: Hidden bootstrap metadata cannot invalidate the live catalog + +- **GIVEN** a successful live refresh omits an older bundled model +- **AND** codex-lb retains that model as hidden metadata whose raw payload lacks + required Codex wire fields +- **WHEN** a Codex client requests the native model catalog +- **THEN** the hidden entry includes a valid `truncation_policy` +- **AND** it includes `experimental_supported_tools` as a list +- **AND** the complete catalog can be deserialized instead of falling back to + bundled client metadata + +#### Scenario: Explicit valid upstream compatibility values win + +- **GIVEN** a live catalog or model source provides `truncation_policy` or + `experimental_supported_tools` +- **WHEN** codex-lb renders the Codex-native catalog entry +- **THEN** it preserves those explicit values unchanged + +#### Scenario: Invalid source tool members cannot fail the catalog + +- **GIVEN** a model source provides `experimental_supported_tools` with both + string and non-string members +- **WHEN** codex-lb renders the Codex-native catalog entry +- **THEN** it retains the string tool names +- **AND** it omits non-string members instead of returning a server error + +#### Scenario: Non-list source tool metadata cannot fail the catalog + +- **GIVEN** a model source provides a non-list value for + `experimental_supported_tools` +- **WHEN** codex-lb renders the Codex-native catalog entry +- **THEN** it emits an empty list instead of returning a server error + +#### Scenario: Malformed source truncation policy cannot fail the catalog + +- **GIVEN** a model source provides an invalid `truncation_policy`, such as a + null, non-object, incomplete object, unknown mode, non-integer limit, or + out-of-range limit +- **WHEN** codex-lb renders the Codex-native catalog entry +- **THEN** it emits the conservative model-compatible truncation policy +- **AND** it does not return a server error + +#### Scenario: Client-version alias has the same complete contract + +- **WHEN** Codex requests `GET /v1/models` with a non-empty `client_version` +- **THEN** every returned `models` entry satisfies the same required-field + contract as `GET /backend-api/codex/models` + +### Requirement: Fresh additional-quota evidence can establish account support + +For a model canonically mapped to a separately metered additional quota, account selection MUST allow fresh account-specific additional-quota telemetry to establish model support when an authoritative general per-account model catalog omits that model. The system MUST continue to enforce registry plan and service-tier restrictions and MUST apply the existing additional-quota freshness, exhaustion, account-health, cooldown, capacity, security, and routing gates before selecting an account. When such a selected account is bound to an HTTP bridge session, every existing-session reuse entry point, including direct key lookup, previous-response alias fallback, and in-flight creation waiters, MUST enforce exact normalized model, canonical quota key, and normalized effective service-tier compatibility before returning the session. For a genuinely catalog-omitted account, reuse MUST re-evaluate current registry plan and requested service-tier plan eligibility without synchronously re-reading quota telemetry. This behavior MUST NOT apply to unknown models or to an unrelated additional-limit key supplied independently of the requested model. + +#### Scenario: Fresh Spark quota overrides general account-catalog omission + +- **GIVEN** an authoritative general account catalog omits `gpt-5.3-codex-spark` for a plan-compatible active account +- **AND** that account has fresh, non-exhausted `codex_spark` quota telemetry +- **WHEN** account selection is requested for `gpt-5.3-codex-spark` +- **THEN** the general account-catalog omission does not remove that account from consideration +- **AND** the account proceeds through the remaining additional-quota and routing gates + +#### Scenario: Quota-admitted bridge session remains reusable + +- **GIVEN** an account omitted from the authoritative general account catalog was selected for `gpt-5.3-codex-spark` using fresh, non-exhausted `codex_spark` telemetry +- **AND** an HTTP bridge session records that selection's normalized model, canonical quota key, and effective service tier +- **WHEN** a later turn requests the same normalized model, canonical quota mapping, and effective service tier +- **THEN** the existing bridge session remains reusable +- **AND** the synchronous reuse check does not re-read quota telemetry + +#### Scenario: Bridge admission provenance is narrowly bound + +- **GIVEN** an HTTP bridge session carries quota-backed catalog-omission provenance +- **WHEN** a later request reaches that session through direct key lookup, previous-response alias fallback, or an in-flight creation waiter with a different normalized model, canonical quota key, or effective service tier +- **THEN** that provenance does not bypass the normal catalog and service-tier checks +- **AND** a catalog-supported account rejected by the requested account-level service-tier index remains rejected + +#### Scenario: Reuse rechecks current plan-tier eligibility for a catalog omission + +- **GIVEN** an HTTP bridge session carries exact quota-backed catalog-omission provenance for a requested service tier +- **AND** the registry's current requested service-tier plan restrictions exclude the session account's current plan +- **WHEN** a later request reaches that session through any reuse entry point +- **THEN** the existing session is not returned under the recorded provenance +- **AND** the current request follows a request-scope fork or fail-closed path without synchronously re-reading quota telemetry or mutating the existing live session + +#### Scenario: Incompatible request preserves another request's live bridge state + +- **GIVEN** a live or in-flight HTTP bridge session is compatible with its creator request +- **AND** another direct, previous-response-alias, turn-state-alias, or in-flight-waiter request has mismatched quota-backed admission provenance or current plan-tier eligibility +- **WHEN** bridge request compatibility rejects that second request +- **THEN** an unanchored request uses an independent collision-resistant request-scope session, or an anchored request alone fails closed +- **AND** the creator's session remains registered, open, and unscheduled for close with its request model, service tier, and transport unchanged +- **AND** live previous-response and turn-state aliases remain unchanged so a subsequent compatible request can resolve and reuse the owner +- **AND** an alias mapping is removed only when its target is missing, closed, or inactive + +#### Scenario: Forwarded prompt-cache mismatch forks on the receiving owner + +- **GIVEN** two bridge replicas agree that a prompt-cache key belongs to one canonical owner +- **AND** that owner has an open quota-admitted Spark session whose effective service tier is incompatible with a priority request already forwarded to the owner +- **AND** the priority request's collision-resistant `internal_request_parallel` fork key rendezvous-hashes to the other replica +- **WHEN** compatibility rejects either the registered session or a session returned to an in-flight creation waiter +- **THEN** the receiving canonical owner creates and owns the request-local mismatch fork without forwarding again +- **AND** both requests can complete on independent transports while the creator session remains open and registered +- **AND** normal rendezvous ownership remains unchanged for canonical prompt-cache, session, turn-state, previous-response, and unforwarded fork keys + +#### Scenario: Catalog-supported account-level service-tier exclusion remains authoritative + +- **GIVEN** an authoritative general per-account catalog includes a mapped separately metered model for two plan-compatible accounts +- **AND** the authoritative requested service-tier account index includes only one of those accounts +- **AND** both accounts have fresh, non-exhausted additional-quota telemetry for the model +- **WHEN** account selection requests that model and service tier +- **THEN** the account absent from the requested service-tier account index is not selected +- **AND** quota evidence does not reclassify that catalog-supported account as model-catalog-omitted + +#### Scenario: Plan incompatibility remains authoritative + +- **GIVEN** a requested separately metered model is mapped to an additional quota +- **AND** an account's plan is excluded by the model registry's plan or requested service-tier restrictions +- **WHEN** account selection evaluates that account +- **THEN** the account is not selected even if additional-quota telemetry exists + +#### Scenario: Missing or stale quota evidence fails closed + +- **GIVEN** the general account catalog omits a mapped separately metered model +- **AND** no plan-compatible account has fresh additional-quota telemetry for that model +- **WHEN** account selection is requested +- **THEN** selection fails with the existing additional-quota data-unavailable behavior +- **AND** the system does not route based only on bootstrap metadata +- **AND** no quota-backed HTTP bridge session is admitted from that failed selection + +#### Scenario: Explicit unrelated quota cannot bypass model support + +- **GIVEN** a caller supplies an additional-limit key that is not the requested model's canonical quota mapping +- **WHEN** the general per-account catalog excludes an account for that model +- **THEN** the supplied quota key does not override the account-catalog exclusion + +### Requirement: Model catalog reservations are released on every exit path + +The model catalog builders for `GET /v1/models` and `GET /backend-api/codex/models` SHALL release the API-key usage reservation after acquisition on normal return, exception, or cancellation. The builders MUST preserve the existing reservation amount and successful response shape. + +#### Scenario: OpenAI-compatible catalog lookup fails + +- **WHEN** `_list_enabled_source_catalog_models` raises after reservation + acquisition while serving `GET /v1/models` +- **THEN** the reservation row is released +- **AND** its reserved usage is no longer charged to the key + +#### Scenario: Codex-native catalog lookup fails + +- **WHEN** `_list_enabled_source_catalog_models` raises after reservation + acquisition while serving `GET /backend-api/codex/models` +- **THEN** the reservation row is released +- **AND** its reserved usage is no longer charged to the key + +#### Scenario: Catalog request is cancelled + +- **WHEN** either model catalog builder is cancelled after reservation + acquisition +- **THEN** the reservation is released before cancellation propagates + +### Requirement: Ultrafast routing follows live account entitlement + +The system MUST treat `ultrafast` as an access-controlled service tier and MUST derive account eligibility from live or retained per-account upstream catalog metadata. The bundled bootstrap catalog MUST NOT invent Ultrafast entitlement. + +#### Scenario: Only an advertising account is eligible + +- **GIVEN** two accounts advertise `gpt-5.6-sol` +- **AND** only one account advertises the `ultrafast` service tier +- **WHEN** a request explicitly asks for `service_tier: "ultrafast"` +- **THEN** account selection considers only the advertising account + +#### Scenario: Bootstrap metadata does not grant preview access + +- **WHEN** no live or retained account catalog advertises `ultrafast` +- **THEN** bootstrap model metadata does not expose or grant that tier + diff --git a/openspec/specs/outbound-http-clients/spec.md b/openspec/specs/outbound-http-clients/spec.md index b54a5a8d5f..f682880ac5 100644 --- a/openspec/specs/outbound-http-clients/spec.md +++ b/openspec/specs/outbound-http-clients/spec.md @@ -46,43 +46,6 @@ Explicit configuration MUST still override auto-detection. - **AND** standard outbound proxy environment variables are set - **THEN** upstream websocket handshakes connect directly without using those proxies -### Requirement: Proxied WebSocket setup closes fail as pre-dispatch transport errors - -When an upstream WebSocket uses an HTTP proxy and the transport closes while -TLS setup is transferring the transport to the WebSocket protocol, before the -protocol's `connection_made()` initializes receive state, the service MUST -complete connection-lost bookkeeping without dereferencing uninitialized -receive or transport attributes. The service MUST retry exactly one fresh -tunnel on the same account because no application frame was dispatched. If -that retry also fails, it MUST return a typed pre-dispatch transport error -without penalizing or rotating accounts and MUST NOT leave an HTTP Responses -stream pending without a terminal event. Once `connection_made()` has run, -the dependency's established-connection close semantics MUST remain unchanged. - -#### Scenario: proxy transport closes before connection setup completes - -- **GIVEN** a secure upstream Responses WebSocket is routed through an HTTP proxy -- **AND** the proxy transport closes before `connection_made()` initializes the receive assembler -- **WHEN** the WebSocket dependency reports `connection_lost()` -- **THEN** the service completes the connection-lost waiter without raising an event-loop callback exception -- **AND** retries one fresh proxy tunnel on the same account -- **AND** no request is treated as having reached upstream - -#### Scenario: shared proxy setup retry is exhausted - -- **GIVEN** the fresh same-account proxy tunnel also closes before setup completes -- **WHEN** the service returns the connection failure -- **THEN** the failure identifies exhausted shared-proxy setup -- **AND** the selected account is not backed off or excluded -- **AND** no other account is tried through the same failing shared proxy - -#### Scenario: established proxied connection keeps normal close semantics - -- **GIVEN** the proxied WebSocket completed `connection_made()` and initialized receive state -- **WHEN** the established connection closes -- **THEN** the dependency's normal close path handles pending receives, pings, and drain waiters -- **AND** the adapter does not weaken or suppress the established-connection failure classification - ### Requirement: Runtime version status checks latest GitHub release The service SHALL expose a dashboard-auth protected runtime version status API that reports the running codex-lb version, the latest known GitHub release version when available, whether an update is available, and the time of the latest lookup attempt. The lookup MUST be cached in-process to avoid per-request GitHub traffic, and lookup failures MUST NOT cause the API to fail. diff --git a/openspec/specs/proxy-admission-control/context.md b/openspec/specs/proxy-admission-control/context.md index a9d11002e1..035f9c308c 100644 --- a/openspec/specs/proxy-admission-control/context.md +++ b/openspec/specs/proxy-admission-control/context.md @@ -23,6 +23,22 @@ Persistent rebind was rejected because admission completes at different points i Session `S` is mapped to account A. A has all response-create slots in use, while account B has capacity. A new self-contained request carrying only `S` may run on B, but the stored mapping still points to A. A later request that references a response created on B follows that response's hard owner index; it does not rely on `S`. +## Account Cap Sizing Across Replicas + +Under the default `proxy_account_caps_scope = "partitioned"`, `proxy_account_stream_limit` (default 8) and `proxy_account_response_create_limit` are **cluster-wide targets**, not per-replica values. Each replica derives its own share of a positive cap locally from the sorted bridge-ring membership: `max(1, floor(cap / R) + 1 extra when its rank < cap mod R)` (`app/modules/proxy/cap_partitioning.py`). A cap of `0` stays unlimited everywhere. With the default stream cap of 8 and three replicas the shares are 3/3/2 — a single account can hold at most 2–3 concurrent streams per replica, which surprises operators who read the setting as per-replica. `proxy_account_caps_scope = "replica"` is the supported opt-out: every replica then enforces the full configured cap with no partitioning. + +The effective caps are the **dashboard-persisted** values (`configured_account_concurrency_caps`); the environment settings only seed the initial dashboard row, and there is no dashboard path back to an unset state — so on an initialized deployment the cap is changed from the dashboard, and an env change alone never takes effect. + +Sizing guidance: + +- Choose a positive cap for the **cluster**: the total concurrent upstream streams one account may hold. Scaling replicas out does not raise it; it only re-partitions it. +- Every share is floored at one slot so an account never becomes unroutable on a replica; when `cap < replica_count` the cluster-wide aggregate therefore equals the replica count — and grows with each added replica — rather than honoring the configured cap. +- `proxy_account_stream_recovery_reserve` (default 1) is subtracted at **selection time only** — it keeps slots free for recovery/reconnect traffic and is deliberately not consulted when a warm session reacquires its lease between turns. On small per-replica shares the reserve is proportionally heavy: with a share of 2, selection sees 1 usable slot. +- Membership changes apply with hysteresis: a replica adopts a share **increase** only after the new partition has been stable for a window, while decreases apply immediately — so a missed heartbeat or rolling replacement cannot transiently inflate the aggregate toward upstream. +- Since turn-scoped leases (#1476), idle warm sessions do not occupy slots. A slot lost to an abnormal condition is reclaimed by the stale-lease sweep, whose stream threshold is NOT the raw `proxy_account_lease_ttl_seconds` (default 900s): a legitimately long-running stream must not be reclaimed mid-flight, so the effective bound is `max(lease TTL, longest stream/request budget) + 60s grace` (`_account_lease_stale_ttl_seconds`) — 7260s with the default 7200s Responses budgets. That is the true worst-case recovery time for a leaked stream slot. + +Symptom of undersizing: persistent `account_stream_cap` errors and "Waiting for account capacity" retries while replicas are mostly idle. First response is raising `proxy_account_stream_limit` toward `desired-per-account-concurrency` (a common operating point is `~8 × replica_count`), not adding replicas. + ## Operational Notes Operators can distinguish local account pressure through the stable `account_response_create_cap` and `account_stream_cap` reasons. The spillover behavior is zero-config because it mutates no ownership state; rollback restores conservative fail-closed selection without data conversion. diff --git a/openspec/specs/proxy-admission-control/spec.md b/openspec/specs/proxy-admission-control/spec.md index cc47fa7cac..45192cad8b 100644 --- a/openspec/specs/proxy-admission-control/spec.md +++ b/openspec/specs/proxy-admission-control/spec.md @@ -50,6 +50,8 @@ For `/v1/responses`, `/backend-api/codex/responses`, and compact Responses traff When an account is at either cap, new soft-affinity work MUST prefer another eligible account before returning local overload. A bare process-session mapping MAY supply soft locality only while the request is self-contained, pre-visible, and has no required owner. Account-cap spillover MUST be decided during account selection and MUST NOT switch an account after a request enters shared transport, replay, or durable bridge ownership. Hard-continuity work MUST remain on its required owner and MAY fail closed when that owner is saturated. Hard Codex ownership rows MUST bypass soft sticky fallback/reallocation so pressure cannot delete or rewrite them. +An unanchored parallel fork bridge session whose payload is self-contained (no `previous_response_id`, no `conversation`, and no input file references) and whose current request context has no turn-state owner or anchored forwarding provenance carries no continuity ownership. When its preferred account is rejected by a local account cap (`account_stream_cap` or `account_response_create_cap`) during session creation, the proxy MUST drop the preferred-account hint exactly once for that request and retry account selection among eligible accounts before entering the recoverable account-capacity wait. Requests that carry any continuity owner signal MUST NOT spill and MUST keep the existing preferred-owner behavior, even when durable alias lookup resolves to an `internal_unanchored_parallel` canonical key. + #### Scenario: Soft work avoids saturated account - **GIVEN** account A is at its account response-create cap @@ -79,6 +81,27 @@ When an account is at either cap, new soft-affinity work MUST prefer another eli - **THEN** the request follows the existing hard bridge-capacity behavior - **AND** account-cap spillover does not publish a replacement bridge under the same canonical identity +#### Scenario: Unanchored parallel fork spills off a capped preferred account + +- **GIVEN** an unanchored parallel fork bridge session creation whose payload carries no previous response, conversation, or input file reference +- **AND** its preferred account is rejected with `account_stream_cap` +- **AND** another eligible account is below its stream cap +- **WHEN** session creation retries selection after dropping the preferred-account hint +- **THEN** the fork session is created on the eligible account instead of waiting on the capped account + +#### Scenario: Owner-bearing fork payloads do not spill + +- **GIVEN** a parallel fork bridge session creation whose payload carries a `previous_response_id` +- **WHEN** its preferred account is rejected with a local account cap +- **THEN** the preferred-account hint is kept and the existing preferred-owner behavior applies + +#### Scenario: Turn-state aliases do not spill through an unanchored canonical key + +- **GIVEN** a request carries a turn-state alias whose durable row resolves to an `internal_unanchored_parallel` canonical key +- **AND** that row has a latest turn state but no latest response ID +- **WHEN** its owner account is rejected with a local account cap +- **THEN** the preferred-account hint is kept and the request does not spill to another account + ### Requirement: Local overload reasons are stable and distinguishable Local Responses overload failures MUST expose stable low-cardinality reason fields in logs and metrics so operators can distinguish `bridge_queue_full`, `response_create_gate_timeout`, `hard_affinity_saturated`, `previous_response_owner_unavailable`, `global_admission_timeout`, `capacity_exhausted_active_sessions`, `account_response_create_cap`, and `account_stream_cap`. These local reasons MUST NOT be reported as upstream rate limits. @@ -458,3 +481,195 @@ Per-account concurrency caps are partitioned per bridge-ring replica and are cor - **WHEN** the process loads its settings at startup - **THEN** startup fails with a settings validation error naming `CODEX_LB_WORKERS_PER_INSTANCE` - **AND** the error states multi-worker-per-instance is not supported and directs the operator to run one worker per pod/container and scale via replicas + +### Requirement: Stream leases reflect in-flight turns, not session lifetime + +An HTTP bridge session's per-account stream lease MUST be held only while the session has in-flight work. When a session's last in-flight turn detaches — no queued requests, no admission waiters, and no pending requests — the session MUST release its account stream lease while remaining alive for reuse, so a warm idle upstream WebSocket does not occupy a per-account stream slot for its idle TTL. Cancellation MUST NOT interrupt that idle lease settlement after the lease is detached from the session. A turn admitted to a session holding no lease MUST reacquire one under normal cap admission before it is counted into the session queue, and a denied reacquisition MUST fail with the standard HTTP 429 `account_stream_cap` envelope so the recoverable capacity wait and client retry semantics apply unchanged. Reacquisition MUST carry the turn's usage-budget token estimate into the lease, matching initial bridge selection and reconnect, so capacity-weighted routing pressure continues to see turns running on reused warm sessions. The stream recovery reserve MUST NOT be consulted at reacquisition, consistent with the reserve being a selection-time reserve. Session close MUST keep its existing lease settlement; a session that already released while idle has nothing further to settle. + +The lease remains per-session, matching the pre-existing lease lifecycle: a session MUST hold at most one stream lease at a time, and turns queued on a session that already holds a lease MUST NOT acquire additional leases — queued turns multiplex over the session's single upstream stream, which is what the per-account stream cap bounds. If the session closes while a reacquisition is in flight, the freshly acquired lease MUST be released back rather than installed on the closed session, and the turn MUST fail with the standard closed-bridge error envelope. Cancellation MUST NOT interrupt release of that detached lease. A submit MUST be registered as in-flight work (admission waiter) atomically with its lease reacquisition, so a completed turn's finalizer running concurrently cannot observe the session as idle and release the reacquired lease before the new turn is counted into the session queue. Any failure after waiter registration and before queue admission MUST remove that waiter and settle an otherwise-idle lease. Reconnect and reacquisition MUST serialize changes to the session lease so a reconnect lease cannot be overwritten and leaked by a concurrent reacquisition. Cancellation MUST NOT interrupt settlement of a lease detached during reconnect replacement. If prewarm fails after the upstream reader closes the session and defers retirement for that admission waiter, removing the final waiter MUST retire the closed session and release its stream lease. Prewarm cancellation MUST NOT interrupt removal of the admission waiter or settlement of an otherwise-idle stream lease. + +#### Scenario: Finished turn returns the account's stream slot + +- **GIVEN** a bridge session whose only in-flight turn completes +- **WHEN** the turn's stream finalizes and detaches +- **THEN** the session releases its account stream lease +- **AND** the session remains alive for reuse within its idle TTL + +#### Scenario: Idle sessions do not starve new admissions + +- **GIVEN** an account at its stream cap where some leases belong to idle sessions +- **WHEN** those sessions' turns complete +- **THEN** the freed slots admit new work immediately +- **AND** the freed slots are not held until the idle sessions' TTL expiry + +#### Scenario: Next turn on an idle session passes cap admission + +- **GIVEN** an idle bridge session that released its stream lease +- **WHEN** a new turn is admitted to that session +- **THEN** the session reacquires a stream lease before the turn is counted into the session queue + +#### Scenario: Reacquisition denial uses the standard cap envelope + +- **GIVEN** an idle bridge session whose account is at its stream cap +- **WHEN** a new turn's lease reacquisition is denied +- **THEN** the turn fails with HTTP 429 and `error.code = "account_stream_cap"` +- **AND** the recoverable account-capacity wait applies to the retry + +#### Scenario: Close racing reacquisition does not leak the slot + +- **GIVEN** an idle bridge session whose stream lease reacquisition is awaiting cap admission +- **WHEN** the session is closed or evicted before the acquisition completes +- **THEN** the freshly acquired lease is released back to the account +- **AND** the turn fails with the standard closed-bridge error envelope + +#### Scenario: Cancellation during close-race settlement does not leak the slot + +- **GIVEN** a session closes while reacquisition is awaiting cap admission +- **AND** the submit is cancelled while the freshly acquired lease is being returned +- **WHEN** lease settlement completes +- **THEN** cancellation propagates only after the lease is released + +#### Scenario: Stale finalizer cannot release a lease reacquired for a new turn + +- **GIVEN** a warm session whose new turn has reacquired a stream lease but is not yet counted into the session queue +- **WHEN** a previous turn's finalizer runs its idle-release check concurrently +- **THEN** the session is not considered idle +- **AND** the reacquired lease is retained for the new turn + +#### Scenario: Failed queue admission removes its waiter + +- **GIVEN** a submit has registered an admission waiter before queue admission +- **WHEN** its final lease check fails +- **THEN** the admission waiter is removed +- **AND** an otherwise-idle session releases its stream lease + +#### Scenario: Reconnect racing reacquisition retains one lease + +- **GIVEN** a reconnect and idle-session lease reacquisition overlap +- **WHEN** both acquire a stream lease before either operation completes +- **THEN** the session retains exactly one of those leases +- **AND** the losing lease is released immediately + +#### Scenario: Queued turns share the session's single stream slot + +- **GIVEN** a bridge session that holds a stream lease for an active turn +- **WHEN** additional turns are admitted to the session queue +- **THEN** no additional stream leases are acquired +- **AND** the session continues to hold exactly one stream lease + +#### Scenario: Prewarm failure retires a closed session after its waiter leaves + +- **GIVEN** a new turn has reacquired a stream lease and registered an admission waiter +- **AND** the upstream reader closes the session during prewarm and defers retirement for that waiter +- **WHEN** prewarm fails and the final admission waiter is removed +- **THEN** the closed session is retired +- **AND** its stream lease is released + +#### Scenario: Prewarm cancellation completes lease cleanup + +- **GIVEN** a new turn has reacquired a stream lease and registered an admission waiter +- **WHEN** the downstream task is cancelled during prewarm +- **THEN** cleanup removes the admission waiter before propagating cancellation +- **AND** an otherwise-idle session releases its stream lease + +#### Scenario: Grouped terminal errors release an abandoned session's lease + +- **GIVEN** a bridge session whose only pending turns are detached follow-ups (no downstream consumers remain) +- **WHEN** a grouped terminal error (for example `previous_response_not_found`) settles all of them together +- **THEN** the session releases its account stream lease +- **AND** the freed slot admits new work without waiting for session close or idle TTL expiry + +#### Scenario: Busy sessions keep their lease + +- **GIVEN** a bridge session with another turn still queued or pending +- **WHEN** one of its turns detaches +- **THEN** the session's stream lease is retained + +### Requirement: Stream admission applies congestion-aware per-API-key fair share + +When `proxy_api_key_fair_share_congestion_threshold_pct` is greater than zero, stream-lease selection MUST evaluate a per-API-key fair-share gate over the selection's candidate account set before admitting a stream. Pool capacity MUST be computed as the candidate-account count multiplied by each account's effective stream slots (`max(1, stream_limit - stream_reserve_slots)`), pool in-flight as the sum of the candidate accounts' in-flight stream leases, and both compared with integer arithmetic: the pool is congested if and only if `pool_inflight * 100 >= pool_capacity * threshold_pct`. When the pool is not congested the gate MUST admit unconditionally. When the pool is congested the gate MUST admit a key only if the key's in-flight stream count on the candidate accounts plus one does not exceed `max(2, pool_capacity // active_keys)`, where `active_keys` is the number of API keys holding at least one in-flight stream lease on the candidate accounts with the requester counted exactly once. The gate MUST NOT apply when the configured threshold is zero, when the request carries no API key, when the selection is for a reattach stage, when the lease kind is not stream, or when the effective stream limit is nonpositive; keyless streams MUST still count toward pool in-flight. The gate MUST NOT read the database and MUST evaluate under the same runtime lock that guards lease counters. + +#### Scenario: Disabled threshold changes no admission outcome + +- **GIVEN** `proxy_api_key_fair_share_congestion_threshold_pct` is 0 (the default) +- **WHEN** any mix of API keys saturates the pool's stream slots +- **THEN** every selection outcome is identical to the behavior before this change + +#### Scenario: Uncongested pool admits an already-heavy key + +- **GIVEN** a threshold of 80 and pool utilization below 80% +- **AND** one key already holds more streams than `pool_capacity // active_keys` +- **WHEN** that key requests another stream +- **THEN** the request is admitted + +#### Scenario: Congested pool denies a key at or above its fair share + +- **GIVEN** a threshold of 80 and pool utilization at or above 80% +- **AND** a key holding at least `max(2, pool_capacity // active_keys)` in-flight streams +- **WHEN** that key requests another stream +- **THEN** selection returns the stable reason `api_key_stream_fair_share` and no lease is acquired + +#### Scenario: Minimum guarantee admits light keys under congestion + +- **GIVEN** a congested pool dominated by another key's streams +- **WHEN** a key holding fewer than two in-flight streams requests a stream +- **THEN** the fair-share gate admits it + +#### Scenario: Requester is counted exactly once in the divisor + +- **GIVEN** a congested pool where the requester already holds in-flight streams +- **WHEN** the fair share is computed +- **THEN** `active_keys` counts the requester once and does not change whether the requester is currently active or newly arriving + +#### Scenario: Keyless requests bypass the gate but consume capacity + +- **GIVEN** a congested pool +- **WHEN** a request without an API key selects an account +- **THEN** the fair-share gate does not deny it +- **AND** its in-flight stream counts toward pool in-flight for keyed requesters + +#### Scenario: Reattach-stage selection bypasses the gate + +- **GIVEN** a congested pool and a heavy key at its fair share +- **WHEN** that key's reattach-stage selection resumes an existing in-flight response +- **THEN** the fair-share gate does not deny it + +### Requirement: Fair-share denials reuse local capacity-wait semantics + +A fair-share denial MUST surface the stable local-overload reason `api_key_stream_fair_share` and MUST inherit the existing account-capacity handling: the transport layer parks the request with `waiting_for_account_capacity` keepalives and retries selection within the request budget, and a request that exhausts its budget while denied MUST receive HTTP 429 with `error.type` `rate_limit_error` and a `Retry-After` header rather than a 503. The denial message MUST state the key's in-flight count, the fair share, the pool in-flight and capacity, and the active-key count without naming other API keys. + +#### Scenario: Denied request parks and admits after the pool decongests + +- **GIVEN** a heavy key denied by the fair-share gate +- **WHEN** enough streams release for the key to fall under its fair share or the pool to fall below the threshold +- **THEN** a subsequent parked retry admits the request without client intervention + +#### Scenario: Budget exhaustion surfaces 429 with fair-share numbers + +- **GIVEN** a request that remains fair-share denied until its budget is exhausted +- **WHEN** the terminal error is rendered +- **THEN** the status is 429 with `error.type` `rate_limit_error` and a `Retry-After` header +- **AND** the message includes the key in-flight count, fair share, pool in-flight, pool capacity, and active-key count + +### Requirement: Per-API-key stream accounting follows the lease lifecycle + +Every stream lease acquired through account selection MUST record the requesting API key, and the per-account per-key in-flight map MUST be maintained under the runtime lock across acquire, explicit release, and stale reclaim, with map entries removed when a key's count reaches zero and removed together with pruned account runtime state. Account-scoped keys MUST be measured against their scoped candidate accounts only. On the sticky selection path the gate decision MUST be re-validated in the commit lock section before the lease is acquired, so concurrent selections for one key cannot overshoot the share between the filter and commit sections; the unbound path MUST evaluate the gate and acquire the lease in a single lock section. + +#### Scenario: Release and stale reclaim decrement the owning key + +- **GIVEN** a key holding in-flight stream leases +- **WHEN** a lease is released explicitly or reclaimed as stale +- **THEN** that key's in-flight count decreases accordingly and its map entry is removed at zero + +#### Scenario: Scoped key is measured against its scoped pool + +- **GIVEN** a key restricted to a subset of accounts via account assignment scope +- **WHEN** the fair-share gate evaluates its request +- **THEN** pool capacity, pool in-flight, and the key's in-flight count are computed over the scoped candidate accounts only + +#### Scenario: Concurrent sticky selections cannot overshoot the share + +- **GIVEN** a congested pool and one key one stream below its fair share +- **WHEN** two sticky selections for that key pass the filter-phase gate concurrently +- **THEN** at most one acquires a lease and the other is denied at the commit re-check + diff --git a/DECISIONS.md b/openspec/specs/proxy-architecture/context.md similarity index 92% rename from DECISIONS.md rename to openspec/specs/proxy-architecture/context.md index e7ee2697e1..7c710d80ae 100644 --- a/DECISIONS.md +++ b/openspec/specs/proxy-architecture/context.md @@ -1,7 +1,9 @@ -# Architectural Decisions +# Context: proxy-architecture -This file records long-lived architecture decisions for codex-lb. New decisions -are appended and superseded by later entries rather than edited in place. +Normative requirements live in [`spec.md`](./spec.md). This document carries +free-form context for the proxy-architecture capability: architecture decision +records (ADRs) are appended here and superseded by later entries rather than +edited in place. Relocated from the former repository-root `DECISIONS.md`. ## ADR-0001: ProxyService target-architecture cutover refactor diff --git a/openspec/specs/proxy-architecture/spec.md b/openspec/specs/proxy-architecture/spec.md new file mode 100644 index 0000000000..e0f0168804 --- /dev/null +++ b/openspec/specs/proxy-architecture/spec.md @@ -0,0 +1,73 @@ +# proxy-architecture Specification + +## Purpose +Structural fitness gates for the proxy: ProxyService stays a stable façade and internal decomposition (selection orchestration, bridge mixins) cannot drift behavior or re-grow god-modules. +## Requirements +### Requirement: Proxy architecture fitness gates are enforced + +The repository SHALL enforce the accepted proxy architecture thresholds during +the required lint gate. `app/modules/proxy/service.py` SHALL contain no more +than 2,600 lines, `app/modules/proxy/load_balancer.py` SHALL contain no more +than 3,021 lines, and `LoadBalancer.select_account()` SHALL span no more than +527 lines. Implementations SHALL restore or lower these ratchets rather than +increase, bypass, or remove them to make CI pass. + +#### Scenario: Multiple ratchets are violated + +- **WHEN** more than one independent proxy architecture threshold or boundary is violated +- **THEN** one architecture-check run reports every independently evaluable violation in deterministic order +- **AND** the check exits non-zero + +#### Scenario: All architecture gates pass + +- **WHEN** every proxy architecture threshold and boundary is satisfied +- **THEN** the architecture check exits zero +- **AND** it reports that the proxy architecture checks passed + +### Requirement: ProxyService remains a stable façade + +`app.modules.proxy.service.ProxyService` and the required compatibility exports +SHALL remain available to existing consumers. Behavior extracted from +`ProxyService` or `service.py` SHALL be owned by focused private modules under +`app/modules/proxy/_service/`. +Compatibility shims SHALL remain re-export-only and private service domains +SHALL comply with the repository's explicit cross-domain dependency policy. + +#### Scenario: Existing consumers import the proxy façade + +- **WHEN** an existing caller imports `ProxyService` or a required compatibility export from `app.modules.proxy.service` +- **THEN** the import resolves to behavior compatible with the pre-change façade +- **AND** no caller migration is required + +### Requirement: Account selection orchestration is decomposed without behavior drift + +`LoadBalancer.select_account()` SHALL remain the public account-selection entry +point and SHALL delegate cohesive sticky-key retry orchestration and policy to a +private, protocol-typed load-balancer implementation unit. The decomposition +MUST preserve account scope, continuity ownership, security authorization, +exclusions, routing policy, quota and health filtering, concurrency caps, +affinity, stale-state retries, lease cleanup, persistence, result metadata, and +error-code behavior. + +#### Scenario: Selection succeeds with or without stickiness + +- **WHEN** a request is eligible for account selection with either a sticky key or no sticky key +- **THEN** the selected account, lease, persisted runtime state, and result metadata match the pre-change behavior for the same inputs + +#### Scenario: Ownership or capacity prevents selection + +- **WHEN** continuity ownership is ambiguous or conflicting, a hard-affinity owner is unavailable, or account caps are exhausted +- **THEN** selection returns the same fail-closed outcome, error code, and mapping-preservation behavior as before the decomposition + +#### Scenario: Persistence or cancellation interrupts selection + +- **WHEN** persistence fails, a selected row becomes stale, or the selection task is cancelled +- **THEN** acquired leases are released exactly once +- **AND** retries and final errors follow the existing bounded behavior + +#### Scenario: Non-sticky selection observes a cache-generation change + +- **WHEN** non-sticky selection acquires a lease and the selection-input cache generation changes during persistence +- **THEN** the acquired lease is released exactly once +- **AND** non-sticky selection reloads its inputs and retries within the existing bound + diff --git a/openspec/specs/proxy-runtime-observability/context.md b/openspec/specs/proxy-runtime-observability/context.md index 6a01cd1e21..78d23ccee9 100644 --- a/openspec/specs/proxy-runtime-observability/context.md +++ b/openspec/specs/proxy-runtime-observability/context.md @@ -12,9 +12,22 @@ See `openspec/specs/proxy-runtime-observability/spec.md` for normative requireme - **Request tracing is opt-in:** outbound request summary and payload tracing remain configurable because payload logs can be noisy or sensitive. Since issue #1340 phase 1 the switch is the single `CODEX_LB_TRACE` comma-separated channel list (`shape`, `shape_raw_cache_key`, `payload`, `service_tier`, `upstream_summary`, `upstream_payload`); empty default = all off. It is an incident-debugging knob for interactive use only. - **Error logs must be correlated:** request id, endpoint, status, code, and message are the minimum useful fields for debugging 4xx/5xx failures. - **Prewarm observability is outcome-only:** the Codex HTTP-bridge prewarm canary experiment finished, so its bucket/cohort dimensions were retired (issue #1340 phase 4). The `codex_lb_http_bridge_prewarm_total` counter is labelled by `outcome` only, request logs record `prewarm_status` / `prewarm_latency_ms` (statuses: `not_applicable`, `skipped`, `success`, `timeout`, `error` — `canary_miss` no longer occurs), and the legacy `prewarm_canary_bucket` / `prewarm_eligible_reason` request-log columns stay declared but unwritten for one release for rolling-upgrade safety; the Alembic drop revision ships next release (see the next-release queue in `openspec/specs/deployment-installation/context.md`). +- **TTFT datasource selection stays in Grafana:** the Helm chart packages the + TTFT dashboard but does not provision a PostgreSQL datasource or its + credentials. The visible, single-select `DS_SQL` variable keeps + installation-specific datasource UIDs out of chart values while routing all + four SQL panels through one explicit selection. ## Operational Notes - Use request ids to correlate inbound proxy logs, outbound upstream traces, and client-visible failures. - Prefer summary tracing in normal debugging sessions; enable payload tracing only when the exact normalized outbound request matters. - For direct compact `5xx` failures, look for `proxy_compact_failure` alongside `upstream_request_complete`; together they show the compact failure phase, failure detail, exception type, retry metadata, and affinity source. +- After the Grafana sidecar imports the TTFT dashboard, select the ordinary + PostgreSQL datasource that points to the codex-lb database from the visible + **PostgreSQL** dropdown. A datasource registered only as a frontend runtime + plugin is not listed by Grafana's datasource variable. +- Timeout invariant violation logs describe startup `Settings` and imported + constant validation only. They intentionally avoid request-scoped overrides, + runtime-derived effective timeout values, payloads, API keys, access tokens, + raw affinity keys, account emails, and other high-cardinality identifiers. diff --git a/openspec/specs/proxy-runtime-observability/spec.md b/openspec/specs/proxy-runtime-observability/spec.md index 4dab9abd80..b90c63583d 100644 --- a/openspec/specs/proxy-runtime-observability/spec.md +++ b/openspec/specs/proxy-runtime-observability/spec.md @@ -184,13 +184,22 @@ analytics. - **AND** request-log metadata stores `upstream_status_code = null` ### Requirement: Request logs persist prompt-client user-agent metadata -The proxy MUST persist prompt-client user-agent metadata on `request_logs` for both HTTP and WebSocket Responses traffic. Each persisted row MUST store the full inbound `User-Agent` header value when present and a derived `useragent_group` value extracted from the first product token. When the inbound header is missing or blank after trimming, both persisted values MUST be `null`. +The proxy MUST persist prompt-client user-agent metadata on `request_logs` for both HTTP and WebSocket Responses traffic. Each persisted row MUST store the full inbound `User-Agent` header value when present and a derived `useragent_group` value. When the inbound header contains `/`, `useragent_group` MUST be the complete sequence of characters before its first `/`; when it contains no `/`, the existing group extraction behavior MUST remain unchanged. When the inbound header is missing or blank after trimming, both persisted values MUST be `null`. + +#### Scenario: Historical request-log user-agent families are backfilled without normalization +- **WHEN** the user-agent family migration processes historical `request_logs` rows +- **THEN** rows whose `useragent` is non-null and contains `/` MUST have `useragent_group` set to the exact unprocessed full prefix before the first `/` +- **AND** rows whose `useragent` is `null` or contains no `/` MUST remain unchanged #### Scenario: HTTP request log stores user-agent metadata - **WHEN** an HTTP or HTTP/SSE proxy request includes `User-Agent: opencode/1.15.13 ai-sdk/provider-utils/4.0.23 runtime/bun/1.3.14` - **THEN** the persisted `request_logs` row stores `useragent = "opencode/1.15.13 ai-sdk/provider-utils/4.0.23 runtime/bun/1.3.14"` - **AND** the persisted row stores `useragent_group = "opencode"` +#### Scenario: Multi-word product family retains its full prefix +- **WHEN** an HTTP or HTTP/SSE proxy request includes `User-Agent: Codex Desktop/0.142.4` +- **THEN** the persisted `request_logs` row stores `useragent_group = "Codex Desktop"` + #### Scenario: WebSocket request log stores user-agent metadata - **WHEN** a proxied WebSocket Responses session is opened with `User-Agent: opencode/1.15.13 ai-sdk/provider-utils/4.0.23 runtime/bun/1.3.14` - **THEN** the persisted `request_logs` row for that request stores the full header in `useragent` @@ -469,6 +478,13 @@ Operators MUST have an OpenSpec context runbook or dashboard artifact with model/cache ratio, session gap cohort, prompt size cohort, and prewarm status/outcome. +The shipped Grafana TTFT dashboard MUST declare a visible, single-select +runtime datasource variable named `DS_SQL` that is restricted to PostgreSQL. +Every SQL panel MUST bind to the selected UID through a typed PostgreSQL +datasource object. The Helm chart MUST preserve the dashboard in its existing +sidecar-discoverable ConfigMap, and chart documentation MUST tell operators to +select the PostgreSQL datasource in Grafana. + #### Scenario: Operator investigates TTFT regression - **WHEN** an operator needs to inspect the last 24 hours of request-log @@ -476,6 +492,21 @@ status/outcome. - **THEN** the repository provides SQL that reports p50, p90, p95 TTFT and total latency for the requested breakdowns +#### Scenario: Sidecar-provisioned dashboard resolves the selected database + +- **GIVEN** the Helm chart renders the Grafana dashboard ConfigMap +- **AND** Grafana has a PostgreSQL datasource available +- **WHEN** the operator selects that datasource through `DS_SQL` +- **THEN** all four TTFT panels resolve to the selected datasource UID +- **AND** no panel reports `Datasource ${DS_SQL} was not found` + +#### Scenario: Datasource choice remains explicit and deterministic + +- **WHEN** Grafana loads the TTFT dashboard +- **THEN** `DS_SQL` is visible to the operator +- **AND** it permits exactly one PostgreSQL datasource selection +- **AND** it does not offer an all-datasources selection + ### Requirement: Dashboard request logs show generation speed The dashboard request-log table MUST show time to first token and output-token generation speed when the required latency and output-token fields are available. Generation speed MUST use output tokens divided by elapsed generation time after time to first token, not total input plus output tokens and not total request latency including TTFT. @@ -638,3 +669,141 @@ The service MUST expose a Prometheus gauge named `codex_lb_cap_partition_replica - **WHEN** a partition refresh observes and adopts two active members - **THEN** `codex_lb_cap_partition_replicas` reports 2 - **AND** an info-level log records the rebalance from count 1 to count 2 with the replica's rank + +### Requirement: Source-routed requests report upstream-measured generation timings + +The proxy MUST record upstream-reported generation timing on the request log +for source-routed chat/responses/audio-transcription requests when the +OpenAI-compatible source's response body includes a `metrics` object with +`time_to_first_token_ms` and `generation_time_ms`. The proxy MUST set +`latency_first_token_ms` to the reported time-to-first-token and `latency_ms` +to the sum of time-to-first-token and generation time, using the same +request-log fields subscription-backed requests already populate. Sources +that do not return a `metrics` object MUST leave both fields `null`, and +negative or non-numeric values MUST be rejected rather than recorded. +Non-finite numeric values (`NaN`, positive infinity, or negative infinity) +MUST also be rejected rather than failing or interrupting the proxied request. + +#### Scenario: Source metrics populate TTFT and total latency + +- **GIVEN** an OpenAI-compatible source's chat completion response includes + `metrics: {time_to_first_token_ms: 108.83, generation_time_ms: 162.98}` +- **WHEN** the request is logged +- **THEN** the request log's `latency_first_token_ms` is `109` +- **AND** the request log's `latency_ms` is `272` + +#### Scenario: Streamed responses capture metrics from the final frame + +- **GIVEN** a source-routed streaming chat completion whose final SSE frame + carries both `usage` and `metrics` +- **WHEN** the stream completes successfully +- **THEN** the request log records the same `latency_first_token_ms` / + `latency_ms` derived from that frame's `metrics` + +#### Scenario: Missing metrics leaves latency fields null + +- **GIVEN** an OpenAI-compatible source's response includes no `metrics` object +- **WHEN** the request is logged +- **THEN** `latency_ms` and `latency_first_token_ms` remain `null`, unchanged + from prior behavior + +#### Scenario: Dashboard retains generation-only throughput semantics + +- **GIVEN** a source response reports `time_to_first_token_ms: 108.83`, + `generation_time_ms: 162.98`, and `9` output tokens +- **WHEN** the existing dashboard computes tokens per second as output tokens + divided by `latency_ms - latency_first_token_ms` +- **THEN** it reports approximately `55.2` generation tokens per second +- **AND** it does not substitute an upstream `tokens_per_second` value that may + include TTFT + +#### Scenario: Non-finite metrics are ignored safely + +- **GIVEN** a source response contains `NaN` or infinity in either timing field +- **WHEN** the proxy parses the optional metrics +- **THEN** both timing values remain unset +- **AND** the otherwise successful proxied request is not interrupted + +### Requirement: Shipped high-error-rate alert uses aggregate request share + +The shipped `CodexLBHighErrorRate` alert MUST calculate, independently for each +namespace and job, the sum of five-minute 5xx request rates divided by the sum +of all five-minute request rates. Method, path, status, instance, replica, and +other non-scope labels MUST be aggregated before division. The alert MUST +compare the aggregate ratio to 0.05 and MUST require it to remain above that +threshold for five minutes. + +#### Scenario: Mixed success and error series produce their aggregate share + +- **GIVEN** one namespace and job have positive 2xx and 5xx request rates +- **WHEN** the high-error-rate alert expression is evaluated +- **THEN** the ratio equals the sum of 5xx request rates divided by the sum of + all request rates +- **AND** the ratio is not 1 unless all requests in that group are 5xx + +#### Scenario: Alert groups remain isolated + +- **GIVEN** request series exist for more than one namespace or job +- **WHEN** the high-error-rate alert expression is evaluated +- **THEN** each namespace and job pair has an independent aggregate ratio +- **AND** traffic from one pair is not included in another pair + +#### Scenario: Threshold and duration apply to the aggregate ratio + +- **GIVEN** one namespace and job have an aggregate 5xx share above 0.05 +- **WHEN** that aggregate share remains above 0.05 for five minutes +- **THEN** `CodexLBHighErrorRate` fires for that namespace and job pair + +### Requirement: Bundled Grafana 5xx stat uses selected aggregate request share + +The bundled Grafana `Error Rate (5xx)` stat MUST apply the selected namespace +and job filters to both operands, aggregate all remaining request-series labels +before division, and display the resulting 5xx share as one value. When the +selected total request rate is positive but no matching 5xx series exists, the +stat MUST display 0%. + +#### Scenario: Selected mixed traffic produces one aggregate value + +- **GIVEN** the selected namespace and job have positive 2xx and 5xx request + rates across one or more request or replica label combinations +- **WHEN** the Grafana error-rate stat is evaluated +- **THEN** it displays the sum of selected 5xx request rates divided by the sum + of all selected request rates + +#### Scenario: Dashboard selection filters both operands + +- **GIVEN** request series exist inside and outside the selected namespace and + job +- **WHEN** the Grafana error-rate stat is evaluated +- **THEN** both the 5xx numerator and total denominator exclude traffic outside + the selected namespace and job + +#### Scenario: Success-only traffic displays zero + +- **GIVEN** the selected scope has a positive successful-request rate +- **AND** no matching 5xx series exists +- **WHEN** the Grafana error-rate stat is evaluated +- **THEN** the stat displays 0% + +### Requirement: Stream pool congestion is observable + +When Prometheus support is available the service MUST expose a gauge named `codex_lb_stream_pool_capacity` whose value equals the fair-share gate's most recently computed candidate pool capacity and a gauge named `codex_lb_stream_pool_inflight` whose value equals the corresponding pool in-flight stream count, and a counter named `codex_lb_api_key_fair_share_rejections_total` incremented once per fair-share denial. The gauges and the counter MUST NOT carry API-key, account, or request labels. Each fair-share denial MUST log at warning level with the requesting `api_key_id`, the key's in-flight count, the computed fair share, the pool in-flight and capacity, and the active-key count, and MUST NOT include other keys' identifiers, instance secrets, or request payload content. All fair-share metrics MUST degrade to no-ops when the Prometheus client is absent. + +#### Scenario: Pool gauges are exported during gate evaluation + +- **GIVEN** the fair-share gate is enabled and evaluates a stream selection +- **WHEN** metrics are scraped +- **THEN** `codex_lb_stream_pool_capacity` and `codex_lb_stream_pool_inflight` report the evaluated pool values without per-key or per-account labels + +#### Scenario: Denials are counted without key cardinality + +- **GIVEN** repeated fair-share denials for multiple keys +- **WHEN** metrics are scraped +- **THEN** `codex_lb_api_key_fair_share_rejections_total` reflects the total denial count with no per-key label + +#### Scenario: Denial log carries the diagnostic numbers + +- **GIVEN** a fair-share denial +- **WHEN** the warning is logged +- **THEN** it includes the requester's `api_key_id`, key in-flight count, fair share, pool in-flight, pool capacity, and active-key count and no other key's identifier + diff --git a/openspec/specs/proxy-warmup/spec.md b/openspec/specs/proxy-warmup/spec.md index 3bd97584ca..a176bb56fb 100644 --- a/openspec/specs/proxy-warmup/spec.md +++ b/openspec/specs/proxy-warmup/spec.md @@ -4,7 +4,7 @@ TBD - created by archiving change add-v1-warmup-endpoint. Update Purpose after archive. ## Requirements ### Requirement: Warmup endpoint is exposed on the v1 proxy surface -The system SHALL expose `POST /v1/warmup` on the same authenticated proxy surface as other `/v1/*` routes. The endpoint SHALL accept a JSON body with `mode` and SHALL return a structured JSON summary of submitted, skipped, and failed account warmups. +The system SHALL expose `POST /v1/warmup` on the same authenticated proxy surface as other `/v1/*` routes. The endpoint SHALL accept a JSON body with `mode` and SHALL return HTTP 200 with a structured JSON summary of submitted, skipped, and failed account warmups for every valid execution. Per-account `ProxyAuthError` and `ProxyRateLimitError` failures SHALL be represented in the `failed` summary regardless of the number of target accounts. The system SHALL also expose `POST /v1/warmup/{mode}` on the same authenticated proxy surface. That route SHALL not require a request body and SHALL execute the same warmup behavior as the body-based route for the supplied `mode`. @@ -12,6 +12,14 @@ The system SHALL also expose `POST /v1/warmup/{mode}` on the same authenticated - **WHEN** a client calls `POST /v1/warmup` with a valid API key and valid mode - **THEN** the system returns 200 with a per-account warmup result summary +#### Scenario: Single-account authentication failure returns summary +- **WHEN** a valid warmup request targets exactly one account and its submission raises `ProxyAuthError` +- **THEN** the system returns 200 with `total_accounts=1` and one `failed` entry with error code `auth_error` + +#### Scenario: Single-account rate-limit failure returns summary +- **WHEN** a valid warmup request targets exactly one account and its submission raises `ProxyRateLimitError` +- **THEN** the system returns 200 with `total_accounts=1` and one `failed` entry with error code `rate_limit_exceeded` + #### Scenario: Invalid mode is rejected - **WHEN** a client calls `POST /v1/warmup` with an unsupported mode value - **THEN** the system returns a 400 invalid request error diff --git a/openspec/specs/query-caching/spec.md b/openspec/specs/query-caching/spec.md index 4f6abf3ce3..0d4c827ecd 100644 --- a/openspec/specs/query-caching/spec.md +++ b/openspec/specs/query-caching/spec.md @@ -767,3 +767,86 @@ Non-PostgreSQL backends MUST NOT be affected (no visibility map). - **GIVEN** a PostgreSQL deployment where the identical autovacuum settings were already applied manually (the reference deployment's hotfix) - **WHEN** the autovacuum tuning revision is applied - **THEN** the migration MUST complete without error and leave the same settings in place + +### Requirement: Request-log listing totals are cached per filter signature + +The request-log listing MUST NOT execute an exact `COUNT(*)` over the filtered set on every page request; the total MUST be reused from a per-filter-signature cache within a fixed 30-second TTL (an application constant per the `reduce-settings-surface-phase-2` change — not an operator tunable). Cached totals are display-only: page contents themselves MUST remain exact and newest-first. + +#### Scenario: Repeated pages reuse the cached total + +- **GIVEN** two listing requests with the same filters but different offsets within the TTL +- **WHEN** both pages are served +- **THEN** the filtered set is counted once and both responses report the same total + +#### Scenario: Distinct filter signatures count independently + +- **WHEN** a listing request arrives with different filters +- **THEN** its total comes from its own count, not another signature's cache entry + +#### Scenario: Expired entries are recounted + +- **GIVEN** a cached total whose 30-second TTL has elapsed +- **WHEN** a listing request with the same filter signature arrives +- **THEN** an exact count is executed and the cache entry is refreshed + +### Requirement: Upstream-route resolution is invalidation-driven with a TTL backstop + +Proxy hot-path upstream-route resolution MUST be served from a per-account cache of resolver outcomes. Admin mutations of any resolver input (account proxy bindings, proxy pool membership, upstream-proxy dashboard settings, account deletion cascading a binding away) MUST invalidate the cache on the mutating replica before the mutating response returns and durably bump a cache-invalidation namespace so peer replicas converge within one poll interval. If the durable bump write fails (the bump primitive is non-raising), the implementation MUST enqueue the coalesced retry so peers still converge on the first poll cycle after the write path recovers. The cache TTL MUST default to 60 seconds as a backstop for out-of-band database edits, and a TTL of 0 MUST disable caching entirely. + +#### Scenario: Repeat turns skip route re-resolution + +- **GIVEN** an account whose route resolved less than the TTL ago with no intervening route-input mutation +- **WHEN** another proxy request uses that account +- **THEN** the route MUST be served from the cache without opening a database session + +#### Scenario: Binding change invalidates before the response returns + +- **GIVEN** a cached route outcome for an account +- **WHEN** an operator upserts that account's proxy binding +- **THEN** the mutating replica's cache MUST be cleared before the HTTP response returns +- **AND** the `upstream_route` namespace MUST be durably bumped so peers clear their caches via the poller + +#### Scenario: Pool membership change invalidates + +- **GIVEN** a cached route outcome resolved from a pool +- **WHEN** an operator adds a member to any proxy pool +- **THEN** the local cache MUST be cleared and the `upstream_route` namespace durably bumped before the response returns + +#### Scenario: Account deletion invalidates + +- **GIVEN** a cached route outcome for an account +- **WHEN** an operator deletes the account (cascading its proxy binding away) +- **THEN** the local cache MUST be cleared and the `upstream_route` namespace durably bumped before the response returns + +#### Scenario: Peer replicas converge through the poller + +- **GIVEN** a cached route outcome on a replica that did not perform the mutation +- **WHEN** the `upstream_route` or `settings` namespace version advances +- **THEN** that replica's cache-invalidation poller MUST clear its route cache within one poll interval + +#### Scenario: Upstream settings change invalidates + +- **GIVEN** a cached route outcome +- **WHEN** an operator changes `upstream_proxy_routing_enabled` or `upstream_proxy_default_pool_id` +- **THEN** the mutating replica's route cache MUST be cleared and the `upstream_route` namespace durably bumped (with the coalesced retry on write failure) before the response returns +- **AND** peers MUST also clear theirs via the durable `settings` namespace bump + +### Requirement: Aggregated rate-limit reads never run concurrently on a shared session + +Proxy rate-limit header and usage-payload construction MUST NOT execute +multiple statements concurrently on one `AsyncSession`. Repository objects +exposed by the same `ProxyRepositories` context SHALL be treated as sharing that +single-session ownership constraint. + +#### Scenario: Rate-limit header reads execute sequentially + +- **WHEN** the proxy constructs upstream-quota rate-limit headers from primary, secondary, monthly, and credit usage rows +- **THEN** each database read MUST complete before the next read starts on the shared session +- **AND** the returned header names and values remain unchanged for equivalent rows + +#### Scenario: Codex usage payload reads execute sequentially + +- **WHEN** the proxy constructs the aggregate `/api/codex/usage` payload for a request that does not resolve to a codex-lb API key, using usage windows, credits, and additional limits +- **THEN** each database read MUST complete before the next read starts on the shared session +- **AND** the returned payload remains schema- and value-compatible for equivalent rows + diff --git a/openspec/specs/quota-phase-planner/spec.md b/openspec/specs/quota-phase-planner/spec.md index bdcbedb80a..494df4c204 100644 --- a/openspec/specs/quota-phase-planner/spec.md +++ b/openspec/specs/quota-phase-planner/spec.md @@ -65,7 +65,9 @@ produce an `observed`, `known`, or `high` confidence warmup-effect observation. The quota planner SHALL expose authenticated dashboard APIs and UI controls for settings, forecast, decisions, warm-now, and cancellation. Settings changes and scheduler decisions MUST remain auditable, and decision responses SHOULD expose -parsed decision details when stored audit JSON is available. +parsed decision details when stored audit JSON is available. Warm-now reset +eligibility gates MUST compare persisted quota reset epochs against the current +UTC instant regardless of the server process timezone. #### Scenario: Operators can inspect planner decisions @@ -80,6 +82,16 @@ parsed decision details when stored audit JSON is available. execution - **AND** it records a skipped, failed, or executed decision outcome +#### Scenario: Warm-now reset gate is timezone-independent + +- **GIVEN** a short-window usage reset epoch is already due in UTC +- **AND** the server process local timezone is UTC+ +- **WHEN** a dashboard user requests a manual warm-now probe for that account +- **THEN** the reset gate MUST NOT skip with `account_window_already_active` + because of process-local timestamp conversion +- **AND** the warm-now request remains eligible for execution when the other + server-side gates allow it + ### Requirement: Quota planner decisions persist naive UTC instants The quota phase planner SHALL normalize timezone-aware datetimes to naive UTC diff --git a/openspec/specs/rate-limit-reset-credits/spec.md b/openspec/specs/rate-limit-reset-credits/spec.md index e2642244ae..511b9beb60 100644 --- a/openspec/specs/rate-limit-reset-credits/spec.md +++ b/openspec/specs/rate-limit-reset-credits/spec.md @@ -5,7 +5,7 @@ TBD - created by archiving change add-rate-limit-reset-credits. Update Purpose a ## Requirements ### Requirement: Reset credits are polled per account on a fixed cadence -The system SHALL poll upstream `GET /wham/rate-limit-reset-credits` for each eligible account on a configurable cadence that defaults to 60 seconds, using that account's stored OAuth bearer token and `chatgpt-account-id`. The scheduler SHALL always start with the application lifespan. Because snapshots are kept in process-local memory, every running replica SHALL refresh its own snapshot cache instead of relying on leader election, and the scheduler SHALL NOT be leader-gated while snapshots remain process-local. Each replica SHALL apply a randomized startup delay of up to one full interval and randomized per-tick jitter of +/-10% so replica ticks are desynchronized. The aggregate upstream fetch rate scales with the number of running replicas; `rate_limit_reset_credits_refresh_interval_seconds` is the operator control for total upstream load. The poll SHALL skip any account that is paused, requires reauthentication, deactivated, or lacks a usable `chatgpt-account-id`. +The system SHALL poll upstream `GET /wham/rate-limit-reset-credits` for each eligible account on a configurable cadence that defaults to 60 seconds, using that account's stored OAuth bearer token and `chatgpt-account-id`. The scheduler SHALL start with the application lifespan when reset-credit polling is enabled. Because snapshots are kept in process-local memory, every running replica SHALL refresh its own snapshot cache instead of relying on leader election, and the scheduler SHALL NOT be leader-gated while snapshots remain process-local. Each replica SHALL apply a randomized startup delay of up to one full interval and randomized per-tick jitter of +/-10% so replica ticks are desynchronized. The aggregate upstream fetch rate scales with the number of running replicas; `rate_limit_reset_credits_refresh_interval_seconds` is the operator control for total upstream load. The poll SHALL skip any account that is paused, requires reauthentication, deactivated, or lacks a usable `chatgpt-account-id`. #### Scenario: Default cadence polls every 60 seconds - **WHEN** the application starts with default settings @@ -145,13 +145,39 @@ The reset-credits refresh scheduler SHALL NOT transition any account's persisted ### Requirement: Reset credit polling interval is configurable -The system SHALL expose setting `rate_limit_reset_credits_refresh_interval_seconds` (default `60`) to control the polling cadence. The system SHALL NOT expose a separate enable/disable toggle for reset-credit polling. +The system SHALL expose setting `rate_limit_reset_credits_refresh_interval_seconds` (default `60`) to control the polling cadence. The system SHALL expose setting `rate_limit_reset_credits_refresh_enabled` (default `true`) to enable or disable background reset-credit polling. Because the refresh loop is the sole driver of automatic reset-credit redemption, disabling background polling SHALL also disable automatic redemption; when polling is disabled while the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is enabled, the system SHALL log a configuration-conflict warning at startup naming both settings. While polling is disabled, the dashboard settings update SHALL reject a request that newly enables `auto_redeem_reset_credits_before_expiry` with a bad-request error naming the polling toggle; an already-persisted opt-in SHALL remain readable and re-savable so unrelated settings edits are not blocked. #### Scenario: Operator tunes the polling interval - **GIVEN** `rate_limit_reset_credits_refresh_interval_seconds` is set to `120` - **WHEN** the application starts and runs - **THEN** each eligible account's credits are fetched from upstream at most once per 120 seconds +#### Scenario: Operator disables background polling +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **WHEN** the application starts +- **THEN** the reset-credit polling scheduler does not create a background polling task +- **AND** no upstream reset-credits fetches occur + +#### Scenario: Disabled polling conflicts with persisted auto-redeem opt-in +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **AND** the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is `true` +- **WHEN** the application starts +- **THEN** the system logs a configuration-conflict warning naming both settings +- **AND** no automatic reset-credit redemption occurs while polling remains disabled + +#### Scenario: Auto-redeem opt-in is rejected while polling is disabled +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **AND** the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is `false` +- **WHEN** a dashboard settings update sets `auto_redeem_reset_credits_before_expiry` to `true` +- **THEN** the update is rejected with a bad-request error naming the polling toggle +- **AND** the persisted setting remains `false` + +#### Scenario: Persisted auto-redeem does not block unrelated settings edits +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **AND** the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is already `true` +- **WHEN** a full settings payload that keeps the opt-in unchanged is submitted +- **THEN** the update succeeds + ### Requirement: Reset credit redemption is serialized and idempotent across replicas Per-account redemption serialization MUST hold across all replicas and processes sharing one database. On PostgreSQL the system SHALL use `pg_advisory_xact_lock` keyed by the account id on the caller's session. On SQLite the system SHALL acquire a durable claim row via a single atomic conditional upsert (`INSERT ... ON CONFLICT(account_id) DO UPDATE ... WHERE expires_at < now`) with a 30-second lease, a bounded retry loop that surfaces a client-facing conflict on timeout, release on completion, and takeover of expired claims. While the redeem section runs, the claim holder SHALL renew its lease on a heartbeat cadence shorter than the lease (10 seconds) so a redemption that legitimately outlives one lease (e.g. slow upstream fetch/consume) is NOT taken over by a concurrent process; lease expiry without renewal remains the crash-recovery path. A claim-acquisition timeout SHALL surface in the caller surface's native error envelope: the dashboard error envelope on the dashboard consume endpoint and the `/v1/*` OpenAI error envelope (HTTP 409) on `POST /v1/reset-credit`. The system SHALL persist the `(account_id, redeem_request_id) -> credit_id` mapping in the shared database, committed inside the serialized section BEFORE the upstream consume call; a retry carrying the same `redeem_request_id`, served by ANY replica, MUST resolve to the originally selected `credit_id` and MUST NOT consume a different credit. Ledger rows SHALL be retained at least 24 hours (including after a failed consume, so a retry retargets the same credit) and purged opportunistically afterwards. Expired rows for an account SHALL be purged BEFORE a new pin is inserted, so that reusing a `redeem_request_id` after its prior row has aged past the 24h TTL durably re-pins the new attempt to its newly selected `credit_id` instead of silently discarding the new pin because an `ON CONFLICT DO NOTHING` insert collided with the soon-purged expired row. The pin lookup SHALL apply the same 24h TTL on read: a ledger row whose `created_at` is older than the TTL MUST be treated as absent (not returned as a durable pin) so a reused `redeem_request_id` is re-selected against the fresh fetch and re-pinned rather than forwarded for the stale expired `credit_id`; the read TTL and the purge TTL SHALL be the same duration. Both the dashboard consume endpoint and `POST /v1/reset-credit` SHALL redeem inside this cross-replica serialized section. diff --git a/openspec/specs/release-management/spec.md b/openspec/specs/release-management/spec.md index b56cf6f7f8..801a73e143 100644 --- a/openspec/specs/release-management/spec.md +++ b/openspec/specs/release-management/spec.md @@ -88,7 +88,32 @@ The release publishing workflow SHALL accept both stable tags (`vX.Y.Z`) and pre ### Requirement: Stable release promotion remains release-please owned -A beta-tested release train SHALL be promoted by merging the normal release-please stable release PR for the corresponding base version. Stable promotion SHALL rebuild PyPI, Docker, Helm, and GitHub Release artifacts with the stable version instead of retagging prerelease artifacts. +A beta-tested release train SHALL be promoted by merging the normal +release-please stable release PR for the corresponding base version. Stable +promotion SHALL rebuild PyPI, Docker, Helm, and GitHub Release artifacts with +the stable version instead of retagging prerelease artifacts. + +Before the stable release PR for `X.Y.Z` is merged, every change in the +release candidate SHALL either be covered by a `vX.Y.Z-beta.N` prerelease +that has been published and deployed to at least one production-scale +environment for a soak of at least 48 hours without new regressions +attributable to the release train, or fall under the safe-delta exception +below. The unsoaked delta — every change not covered by such a soaked +prerelease, whether because no prerelease of the train completed a soak or +because the change landed after the last soaked prerelease — qualifies for +the exception only when it consists solely of documentation, CI, or +release-tooling changes, an urgent security or outage hotfix, or a +combination of these; otherwise the train SHALL soak (again) as a new +prerelease before stable promotion. When promotion relies on the exception, +the exception and its reason SHALL be recorded on the stable release PR +before merge. + +When the release train contains Alembic revisions, the maintainer SHALL review +the revisions between the previous stable tag and the release candidate +directly for data-backfill migrations and SHALL estimate their startup impact +against a production-scale dataset before merging the stable release PR. +Generated changelog titles SHALL NOT be treated as sufficient evidence that +the train contains no data backfills. #### Scenario: beta train is promoted to stable @@ -99,6 +124,52 @@ A beta-tested release train SHALL be promoted by merging the normal release-plea - **AND** the release publishing workflow publishes stable artifacts for `1.19.0` - **AND** stable Docker aliases `latest`, `1`, and `1.19` are updated only by the stable release +#### Scenario: stable promotion waits for the beta soak + +- **GIVEN** `v1.20.0-beta.1` was published 12 hours ago and is deployed on a + production-scale environment +- **WHEN** a maintainer considers merging the stable release PR for `1.20.0` +- **THEN** promotion waits until the beta has soaked for at least 48 hours + without new regressions attributable to the release train + +#### Scenario: stable promotion without a soaked beta records an exception + +- **GIVEN** no `v1.21.1-beta.N` prerelease has completed a 48-hour soak +- **AND** the entire delta since `v1.21.0` consists of an urgent security + hotfix and CI changes only +- **WHEN** a maintainer merges the stable release PR for `1.21.1` with the + exception and its reason recorded on the PR +- **THEN** the promotion is compliant with this requirement + +#### Scenario: unrelated unsoaked changes cannot ride a hotfix exception + +- **GIVEN** no `v1.22.0-beta.N` prerelease has completed a 48-hour soak +- **AND** the delta since the previous stable release contains an urgent + hotfix alongside unrelated feature or migration changes +- **WHEN** a maintainer considers promoting `1.22.0` directly to stable +- **THEN** the hotfix exception does not apply to the train +- **AND** the train either soaks as a beta or the hotfix is released + separately + +#### Scenario: changes landing after the soaked beta restart the soak + +- **GIVEN** `v1.23.0-beta.1` completed a 48-hour production-scale soak +- **AND** a feature or migration change lands on `main` afterwards, before + the stable release PR for `1.23.0` is merged +- **WHEN** a maintainer considers promoting `1.23.0` to stable +- **THEN** the post-beta change is part of the unsoaked delta +- **AND** because a feature or migration change is not exception-eligible, + promotion requires a new soaked prerelease covering it + +#### Scenario: data backfills are identified from Alembic revisions + +- **GIVEN** the release train adds revisions under `app/db/alembic/versions` + since the previous stable tag +- **WHEN** the maintainer prepares to merge the stable release PR +- **THEN** they review those revisions directly for data-backfill operations +- **AND** changelog titles alone are not treated as evidence that no backfill + is present + ### Requirement: Stable release promotions guard every release-managed version field Stable release promotion pull requests SHALL fail CI unless every release-managed version field agrees on the stable version and every field that previously held the prior release train version advances together. The guarded fields SHALL include `pyproject.toml`, `app/__init__.py`, `frontend/package.json`, both Helm chart version fields, and the editable `codex-lb` entry in `uv.lock`. diff --git a/openspec/specs/replica-operations/context.md b/openspec/specs/replica-operations/context.md index d82acc6070..0559a98da8 100644 --- a/openspec/specs/replica-operations/context.md +++ b/openspec/specs/replica-operations/context.md @@ -116,9 +116,6 @@ programmatic shutdown, instead of returning an unbounded task to asyncio runner ## Known limitations (triaged follow-ups) -- **`file_id` → account pins are process-local best-effort** — file finalize/input_file requests - landing on another replica can route to an account that does not own the file. Follow-up: - `persist-file-account-pins`. - **Concurrent cross-replica usage refresh can transiently tear `additional_usage_history`** — the per-account delete+insert rewrite is non-transactional across refreshers; the tear self-heals within one refresh interval. Documented limitation; no follow-up scheduled. diff --git a/openspec/specs/responses-api-compat/context.md b/openspec/specs/responses-api-compat/context.md index 9a62f360da..b9e4547395 100644 --- a/openspec/specs/responses-api-compat/context.md +++ b/openspec/specs/responses-api-compat/context.md @@ -31,10 +31,13 @@ See `openspec/specs/responses-api-compat/spec.md` for normative requirements. - Compact transport may use bounded same-contract retries only for safe pre-body transport failures and `401 -> refresh -> retry`. - `/v1/responses/compact` is supported only when the upstream implements it. - `prompt_cache_key` affinity on OpenAI-style routes is intentionally bounded by a dashboard-managed freshness window, unlike durable backend `session_id` or dashboard sticky-thread routing. -- Codex-native direct websocket `/backend-api/codex/responses` treats upstream `previous_response_id` as an ephemeral anchor. If that anchor goes stale, the proxy must mask raw `previous_response_not_found` details and emit a sanitized `codex_previous_response_stale` classifier so compatible Codex clients can soft-reset and retry without `previous_response_id`. +- Codex-native direct websocket `/backend-api/codex/responses` treats upstream `previous_response_id` as an ephemeral anchor. If that anchor goes stale, the proxy masks raw upstream details and emits the sanitized canonical `previous_response_not_found` classifier so compatible Codex clients can retry with full local history and no `previous_response_id`. The upstream Codex socket has emitted this condition both with the canonical code and as a parameterless `invalid_request_error` carrying ``Invalid `previous_response_id`.``; both shapes use the same recovery policy. - Upstream Responses WebSockets use transport ping/pong control frames to detect a black-holed connection without confusing valid application-event silence with an idle turn. Direct and routed connections reuse `proxy_downstream_websocket_idle_timeout_seconds` for this zero-config liveness budget. - A post-send liveness timeout is delivery-ambiguous. It remains account-neutral, is never transparently replayed, and retires the affected upstream socket so a client retry opens a fresh route without risking duplicated model work or tool side effects. +- An HTTP SSE first-event `stream_idle_timeout` is also account-neutral for health writes. The request may still exclude that account and fail over, but idle silence must not increment `error_count` or move the account into probe/drain. - HTTP bridge settlement ownership is explicit: `closed` rejects new work but does not imply that a submitter owns existing siblings. Only a liveness-failed send claims whole-deque settlement under the lifecycle lock; otherwise the reader remains responsible for settling pending requests when the transport dies. +- A DRAINING durable row with a live lease is still owned. Foreign `claim_live_session` and local session create must not steal it, including when forced recovery would otherwise run because the owner endpoint is missing; expired or ownerless DRAINING rows remain recoverable. +- Hard-affinity retry-circuit evidence is request-lifecycle evidence: retirement counts only while the bridge still owns an eventless pending request. Idle no-pending retirement remains observable but neutral, so routine socket churn cannot manufacture the first strike for a later real timeout. ## Fast Mode and Service Tiers @@ -72,6 +75,33 @@ Responses request with: Clients that expose Fast Mode as `fast` may keep using that spelling; codex-lb normalizes it to `priority` before forwarding. +### Ultrafast Processing + +The [OpenAI Responses API reference](https://developers.openai.com/api/reference/resources/responses/methods/create) +documents `ultrafast` as an access-controlled processing tier currently +available for `gpt-5.6-sol`. codex-lb forwards this canonical value unchanged; +it does not grant Ultrafast access by itself. + +Account eligibility comes from live or retained per-account upstream catalog +metadata. The bundled bootstrap catalog deliberately does not advertise +Ultrafast. If no account advertises the tier, an explicit Ultrafast request +cannot select an eligible account; API-key enforcement follows the existing +model-capability fallback when the model itself does not advertise the tier. + +Send a Responses request with: + +```json +{ + "model": "gpt-5.6-sol", + "input": "Summarize the change.", + "service_tier": "ultrafast" +} +``` + +After completion, verify that the response reports +`service_tier: "ultrafast"`. Request logs retain `ultrafast` in the requested, +actual, and effective billable tier fields when upstream confirms it. + ### Operator Fast Mode prohibition Operators can enable the Routing setting `prohibitFastMode` when qualified @@ -115,9 +145,10 @@ when upstream reports a different actual tier. - **HTTP bridge session closes or expires:** The next compatible HTTP `/v1/responses` or `/backend-api/codex/responses` request recreates a fresh upstream websocket bridge session; continuity is guaranteed only within the lifetime of one active bridged session. - **Multi-instance routing without bridge owner policy:** if operators do not configure a bridge ring or front-door affinity, continuity can still fragment across replicas. With a configured bridge ring, hard continuity keys landing on a non-owner replica are proxy-forwarded to the owner replica; the proxy fails closed only when the owner endpoint or ring membership cannot be resolved or the forward signature fails authentication. Gateway-safe prompt-cache requests may accept locality misses and continue locally instead of forwarding. - **Codex websocket reconnects:** Reconnect continuity now depends on the client replaying the accepted `x-codex-turn-state`; generated turn-state is emitted on accept for backend Codex routes and echoed back when the client already supplies one. -- **Codex websocket stale previous-response anchors:** Direct backend Codex websocket stale-anchor failures are surfaced as `response.failed` / `codex_previous_response_stale` without the raw upstream code or missing `resp_...` id; OpenAI-compatible `/v1/responses` websocket clients continue to receive generic `stream_incomplete` masking. +- **Codex websocket stale previous-response anchors:** Direct backend Codex websocket stale-anchor failures are either replayed transparently from a self-contained full resend or surfaced as a sanitized `response.failed` whose `response.error.code` is `previous_response_not_found`; the error omits `param`, the raw upstream envelope, and the missing `resp_...` id. A connect-time failure uses the same code directly at `error.code`. This includes the parameterless upstream message ``Invalid `previous_response_id`.``. OpenAI-compatible `/v1/responses` websocket clients continue to receive generic `stream_incomplete` masking. - **Websocket handshake forbidden/not-found:** Auto transport now fails loud on `403` / `404` instead of silently hiding the websocket regression behind HTTP fallback. - **Upstream websocket stops answering pings:** Pending direct-WebSocket and HTTP-bridge work fails with `upstream_websocket_liveness_timeout`; the account remains healthy and the request is not replayed because upstream acceptance is unknown. +- **Repeated eventless bridge failures:** Two consecutive request-affecting pre-response failures can open the hard-key cooldown. A successful terminal response clears the state; an idle close followed by one real timeout remains only one strike. - **Invalid request payloads:** Return 4xx with `invalid_request_error`. ## Error Envelope Mapping (Reference) @@ -150,6 +181,50 @@ Cursor-style model alias request: This forwards upstream as `model: "gpt-5.4-mini"` with `reasoning.effort: "high"`. +Retry-circuit accounting example: an idle bridge closes with `pending=0`, then +the next request times out before `response.created`. The idle close is logged +but contributes no failure; the timeout is the first strike. Only another +consecutive eventless pending failure may open the repeated-failure cooldown. + +Stale-anchor recovery example: a reconnect sends a tool-output delta with a +recent `previous_response_id`, and upstream answers +``{"type":"error","status":400,"error":{"type":"invalid_request_error","message":"Invalid `previous_response_id`."}}``. +Because the delta cannot stand alone, codex-lb returns a sanitized +`previous_response_not_found` signal on the Codex-native route so the client can +retry once with full local history. If the original request already contained a +self-contained full resend, codex-lb instead reconnects and replays that body +without the rejected anchor. + +## Previous-response replay owner fencing + +Removing a stale continuation anchor does not make every retained body +portable. Encrypted reasoning, account-scoped items, file references, and +durable bridge operation identities remain owned by the account that first +received them. The proxy records that dispatch owner and requires it on later +HTTP streaming, HTTP bridge, and direct WebSocket selections. + +For example, if account A first receives encrypted reasoning and then returns a +pre-visible Trusted Access or authentication failure, account B must never +receive the retained ciphertext. One forced token refresh may replay the body +on account A; permanent failure or owner unavailability fails closed. + +Verified recovery installs a replacement body and updates owner state +atomically. A canonical account-neutral replacement clears the owner and may +use normal failover. A verified nonneutral replacement, including a +Responses-Lite full resend, may replay only on the same owner and preserves the +fence. + +HTTP bridge tracing archive IDs do not pin neutral requests. A real durable +`operation_id` does pin the request until an explicit operation-rebind path +replaces that identity. Existing file pins and API-key settlement-before-health +ordering remain independent invariants. + +Streaming selection authorizes owner compatibility before opening upstream, but +persists a new owner only after dispatch is observed. A transport failure that +is positively classified as pre-dispatch therefore leaves the body unowned and +eligible for its first real dispatch on another account. Ambiguous failures +remain owner-bound. + ## Known Client Integrations (Reference) Third-party agents that consume the `/v1` Responses surface documented by this @@ -179,5 +254,6 @@ OpenSpec change first. - When tracing compact incidents, confirm that request logs and upstream logs show direct `/codex/responses/compact` usage without surrogate `/codex/responses` fallback. - Post-deploy: monitor `no_accounts`, `stream_incomplete`, and `upstream_unavailable`. - Post-deploy: monitor `upstream_websocket_liveness_timeout`; recurring failures indicate a host route, VPN, proxy, or intermediary that black-holes established WebSockets. -- Post-deploy: monitor `codex_previous_response_stale` on `/backend-api/codex/responses`; recurring spikes mean clients are still relying on stale upstream anchors and should perform the documented full-context retry without `previous_response_id`. +- Post-deploy: correlate retry-circuit `opened`, `half_open`, and `reset` events with bridge `pending` and `response_events_seen` diagnostics. An idle `pending=0` retirement must not precede an immediate two-failure cooldown. +- Post-deploy: monitor `previous_response_not_found` on `/backend-api/codex/responses`; recurring spikes show repeated continuity failures, which may come from malformed client identifiers, server-side invalidation, or connection lifecycle. Clients should perform the documented full-context retry without `previous_response_id`. Investigate socket-lifecycle remediation only when a separate close-reason, reconnect, or transport diagnostic correlates with the failures. - Websocket/Codex CLI tier verification runbook: `openspec/specs/responses-api-compat/ops.md` diff --git a/openspec/specs/responses-api-compat/spec.md b/openspec/specs/responses-api-compat/spec.md index aace091bfe..1e5472c160 100644 --- a/openspec/specs/responses-api-compat/spec.md +++ b/openspec/specs/responses-api-compat/spec.md @@ -67,22 +67,227 @@ When `upstream_stream_transport` is `"auto"` and the serialized request payload ### Requirement: Clean upstream close before any response event fails fast -When the HTTP responses bridge observes an upstream websocket close with `close_code = 1000` before any `response.*` event has been surfaced for the pending request, the proxy MUST classify the close as rejected input, surface HTTP 502 `upstream_rejected_input`, and MUST NOT trigger `retry_precreated` or `retry_fresh_upstream`. +When the HTTP Responses bridge observes an upstream WebSocket close with +`close_code = 1000` before any `response.*` event has been surfaced for the +pending request, the proxy MUST preserve its existing pre-visible replay +guards. If the request has already used exactly one eligible pre-visible +replay and the replacement upstream WebSocket also closes cleanly before any +response event, the proxy MAY perform exactly one additional replay. The +additional replay MUST be hard-capped at one per request, and the configured +maximum MUST NOT raise that cap. + +The proxy MUST NOT replay after downstream-visible output, after a terminal +response event, or when continuity-sensitive request state makes replay unsafe. +Before the additional replay, the proxy MAY sleep for bounded configured +jitter. The proxy MUST emit a dedicated low-cardinality diagnostic event for +the additional replay. + +When a downstream HTTP stream task initiates pre-response recovery while the +upstream reader is blocked on the superseded socket, the proxy MUST cancel and +await that reader before locally closing the socket. It MUST then start exactly +one reader for the replacement socket. A close caused by replacing the socket +MUST NOT be recorded as an upstream clean-close failure, MUST NOT increment the +retry circuit, and MUST NOT retire pending work moved to the replacement. The +cancelled reader's socket-generation finalizer MUST NOT leave the shared session +marked closed while the replacement socket is being selected or opened, so idle +pruning MUST NOT evict the handoff in progress. + +The default pre-response idle-recovery window MUST leave bounded headroom +before the downstream client's request timeout. With the default ten-second +keepalive interval, the proxy MUST initiate eligible recovery after no more +than six silent intervals so replacement connection and first output can occur +before a 120-second client deadline. + +The stuck pre-response watchdog MUST judge staleness using elapsed time since +the last upstream activity and the absence of a response identifier or +`response.created` latency, not admission flags alone. A request with a prior +continuity anchor MUST receive at most two retire-thresholds of grace before +being considered stale. When the watchdog skips a candidate, it MUST emit a +low-cardinality diagnostic containing the session-closed state, candidate +count, and pending-state verdicts. #### Scenario: clean close before response.created is not retried -- **WHEN** upstream closes the HTTP responses bridge with `close_code = 1000` before any `response.*` event for the pending request +- **WHEN** the initial upstream HTTP responses bridge closes with `close_code = 1000` before any `response.*` event for the pending request - **THEN** the proxy returns HTTP 502 with `error.code = "upstream_rejected_input"` - **AND** does not transparently replay the pre-created request +#### Scenario: clean close before response output receives one bounded additional replay + +- **GIVEN** an HTTP bridge request has no surfaced `response.*` events +- **AND** its first pre-visible replay has already been used +- **WHEN** the replacement upstream WebSocket closes with code `1000` +- **THEN** the proxy performs one additional pre-visible replay +- **AND** the request replay count increases by one +- **AND** the proxy emits a `retry_precreated_clean_close` diagnostic event + +#### Scenario: repeated clean closes do not create an unbounded replay loop + +- **GIVEN** the additional clean-close replay has already been used +- **WHEN** another upstream WebSocket closes cleanly before response output +- **THEN** the proxy does not replay the request again +- **AND** the existing terminal or circuit handling is used + +#### Scenario: visible output still prevents clean-close replay + +- **GIVEN** the pending request has surfaced any response event downstream +- **WHEN** the upstream WebSocket closes with code `1000` +- **THEN** the proxy does not replay the request + +#### Scenario: clean-close retry jitter is bounded + +- **GIVEN** clean-close retry jitter is configured +- **WHEN** the additional clean-close replay is scheduled +- **THEN** the delay is no greater than the configured jitter maximum +- **AND** the hard replay cap remains one regardless of the configured value + +#### Scenario: downstream idle recovery transfers reader ownership + +- **GIVEN** the upstream reader is blocked on the current bridge socket +- **AND** the downstream HTTP stream task initiates eligible pre-response recovery +- **WHEN** the bridge replaces the upstream socket +- **THEN** the old reader is cancelled and awaited before its socket is closed +- **AND** the shared session remains live while the replacement socket opens +- **AND** idle pruning retains the registered session while the handoff is in progress +- **AND** exactly one reader owns the replacement socket +- **AND** the local close does not open or increment the retry circuit +- **AND** pending work remains attached to the replacement session + +#### Scenario: silent pre-response recovery precedes the client timeout + +- **GIVEN** the upstream has produced no response event +- **AND** the default ten-second keepalive interval is active +- **WHEN** six silent intervals elapse +- **THEN** the proxy initiates eligible pre-response recovery +- **AND** at least sixty seconds remain before a 120-second client request timeout + +#### Scenario: anchored stuck-gate grace is bounded + +- **GIVEN** a pending HTTP bridge request has a prior continuity anchor +- **AND** no response identifier or `response.created` latency has been recorded +- **WHEN** less than two retire thresholds have elapsed since the gate began waiting +- **THEN** the watchdog does not classify the request as stale +- **WHEN** two retire thresholds elapse without upstream activity +- **THEN** the watchdog may classify the request as stale + +#### Scenario: upstream activity resolves admission-flag ambiguity + +- **GIVEN** a pending request has not acquired the response-created gate +- **AND** upstream activity has not produced a response identifier or `response.created` +- **WHEN** the staleness threshold elapses +- **THEN** the watchdog classifies the request as stale +- **AND** emits pending-state verdict inputs when it skips a watchdog pass + +### Requirement: Durable retry-circuit state protects repeated hard-affinity failures + +For a hard-affinity bridge key, the proxy MUST scope retry-circuit state by +affinity kind, affinity key, and API-key scope (using a stable anonymous scope +when no API key is present). The proxy MUST record only the documented +pre-response failure classes (`stream_incomplete`, `clean_close`, and +`stream_idle_timeout`). + +A bridge retirement MUST record one of those failures only when the retiring +session still owns at least one pending request and no response event has been +observed for that request lifecycle. Retiring an idle upstream bridge with no +pending request MUST NOT advance the circuit or cause a later request to be +treated as a repeated failure. A pending request that has already emitted a +response event MUST remain excluded from this pre-response circuit. + +The default circuit MUST open after two consecutive recorded failures. Once +open, it MUST suppress pre-created replay until the persisted cooldown expires, +using exponential backoff from sixty seconds up to ten minutes. Clean-close +failures MUST cap their cooldown at thirty seconds. The proxy MUST persist +failure count, cooldown deadline, last failure detail, and update time in the +`http_bridge_retry_circuits` table and MUST merge conflict updates so concurrent +replicas cannot shorten an existing cooldown. + +The clean-close retry jitter maximum MUST be read from the +`http_responses_session_bridge_clean_close_retry_jitter_max_seconds` runtime +setting and MUST be bounded to the inclusive range 0–30 seconds. + +The proxy MUST evict process-local circuit entries and their loaded/persisted +markers after one hour without use, independently of durable-row cleanup, so +one-shot hard-affinity keys cannot grow the worker's memory without bound. + +Before every hard-affinity retry decision, the proxy MUST refresh the durable +row so a cooldown opened by another replica is observed even when this process +has already loaded the key. A durable lookup or persistence failure MUST NOT +crash the request; the proxy MUST continue using available local state and +record the failure for observability. Rows older than one hour MUST be treated +as expired and removed. A successful terminal response MUST clear the local +and durable circuit state. + +#### Scenario: idle bridge retirement does not consume a circuit strike + +- **GIVEN** a hard-affinity HTTP bridge has no pending requests +- **WHEN** its upstream WebSocket closes and the idle bridge is retired +- **THEN** the retry-circuit failure count for that key remains unchanged +- **AND** a later request is not placed in cooldown because of the idle close + +#### Scenario: eventless pending retirement consumes exactly one strike + +- **GIVEN** a hard-affinity HTTP bridge owns a pending request with no observed response event +- **WHEN** the bridge retires because the upstream fails before acknowledging the request +- **THEN** the retry circuit records exactly one failure for that request lifecycle + +#### Scenario: midstream retirement does not consume a pre-response strike + +- **GIVEN** a hard-affinity HTTP bridge owns a pending request with an observed response event +- **WHEN** the bridge retires before completion +- **THEN** the pre-response retry-circuit failure count remains unchanged + +#### Scenario: the second hard-key failure opens a durable circuit + +- **GIVEN** a hard-affinity key has one recorded pre-response failure +- **WHEN** a second eligible failure is recorded +- **THEN** the proxy opens the retry circuit +- **AND** persists at least two consecutive failures and a cooldown deadline +- **AND** subsequent pre-created replay is suppressed until that deadline + +#### Scenario: retry decisions observe a cooldown opened by another replica + +- **GIVEN** this replica previously looked up a hard-affinity key with no row +- **AND** another replica persists an open cooldown for that same key and API-key scope +- **WHEN** this replica evaluates the next pre-created retry +- **THEN** it refreshes durable state before deciding +- **AND** suppresses the retry for the persisted cooldown + +#### Scenario: circuit state remains isolated by key and API-key scope + +- **GIVEN** one hard-affinity key has an open circuit +- **WHEN** a different affinity key or API-key scope evaluates a retry +- **THEN** that request is not suppressed by the first key's circuit + +#### Scenario: durable circuit lookup failure does not fail the request + +- **GIVEN** durable retry-circuit lookup or persistence is unavailable +- **WHEN** the proxy evaluates or records a retry-circuit event +- **THEN** the request continues using any available local circuit state +- **AND** the failure is logged and exposed through retry-circuit observability + ### Requirement: Long Codex websocket turns tolerate extended upstream silence -The default compact request budget MUST be at least 180 seconds, and the default upstream stream idle timeout MUST be at least 600 seconds, so long-running Codex turns can survive expensive compaction or tool execution without a local proxy watchdog ending the turn prematurely. +The default compact request budget MUST be at least 180 seconds, and the default upstream stream idle timeout MUST be at least 600 seconds, so long-running Codex turns can survive expensive compaction or tool execution without a local proxy watchdog ending the turn prematurely. Responses streams over both HTTP and WebSocket transports MUST use `http_responses_stream_request_budget_seconds` when it is configured; they MUST fall back to `proxy_request_budget_seconds` only when no stream-specific budget is available. #### Scenario: compact and stream watchdog defaults leave room for long turns - **WHEN** the service starts with default configuration - **THEN** `compact_request_budget_seconds` is at least 180 seconds - **AND** `stream_idle_timeout_seconds` is at least 600 seconds +#### Scenario: WebSocket Responses stream uses the stream-specific request budget +- **GIVEN** `proxy_request_budget_seconds = 600` +- **AND** `http_responses_stream_request_budget_seconds = 7200` +- **WHEN** a native WebSocket Responses stream computes its request deadline +- **THEN** the stream budget is 7200 seconds +- **AND** the generic 600 second proxy request budget does not terminate the turn + +#### Scenario: WebSocket reconnect keeps the stream-specific deadline +- **GIVEN** `proxy_request_budget_seconds = 600` +- **AND** `http_responses_stream_request_budget_seconds = 7200` +- **AND** a native WebSocket Responses request needs to reconnect after more than 600 seconds but less than 7200 seconds +- **WHEN** the reconnect performs account selection and opens its replacement upstream WebSocket +- **THEN** both operations remain bounded by the original 7200-second stream deadline +- **AND** the reconnect does not fail solely because the generic 600-second budget elapsed + ### Requirement: Responses upstream websocket liveness is bounded The proxy MUST configure direct and routed upstream Responses WebSocket transports with finite ping/pong liveness detection derived from `proxy_downstream_websocket_idle_timeout_seconds`. When an established Responses WebSocket is terminated because its transport did not receive the required pong, the adapter MUST classify the failure as `upstream_websocket_liveness_timeout`. Direct WebSocket and HTTP bridge relay owners MUST treat that failure as account neutral, MUST NOT transparently replay a pending request whose delivery is ambiguous, MUST finalize its pending request ownership exactly once, and MUST retire the affected upstream socket so a later client retry opens a fresh connection. An HTTP bridge reader MUST suppress its own pending-deque settlement only when a concurrent submitter explicitly claimed liveness-settlement ownership under the session lifecycle lock; `session.closed` alone MUST NOT suppress settlement. @@ -128,7 +333,22 @@ The proxy MUST configure direct and routed upstream Responses WebSocket transpor - **AND** the submitter cancellation is preserved after settlement completes ### Requirement: Upstream websocket drops penalize affected accounts -When an upstream websocket closes while one or more streamed response requests are pending and have not reached a terminal event, the proxy MUST record a transient upstream error for the account before signaling failure for those pending requests, except when the close carries a classified process-wide network failure or upstream WebSocket liveness timeout. A classified process-wide network failure or upstream WebSocket liveness timeout MUST remain account neutral and use its classified error code. For other closes, the proxy MUST surface `stream_incomplete` to affected pending requests except when a direct Responses WebSocket request has already successfully emitted a finite integer `sequence_number`. For that sequenced direct-WebSocket case, the proxy MUST record the request outcome as `stream_incomplete` without emitting a synthetic terminal frame under the active response id, then MUST close the downstream WebSocket with code 1011. +When an upstream websocket closes while one or more streamed response requests +are pending and have not reached a terminal event, the proxy MUST record a +transient upstream error for the account before signaling failure for those +pending requests, except when the close carries a classified process-wide +network failure or upstream WebSocket liveness timeout, is a clean close +(`close_code = 1000`) before any `response.*` event, or carries the classified +per-socket `upstream_keepalive_timeout` transport error. Clean pre-response +closes, keepalive timeouts, process-wide network failures, and liveness +timeouts MUST remain account-neutral and use their classified error and bounded +retry or retry-circuit handling. For other closes, the proxy MUST surface +`stream_incomplete` to affected pending requests except when a direct Responses +WebSocket request has already successfully emitted a finite integer +`sequence_number`. For that sequenced direct-WebSocket case, the proxy MUST +record the request outcome as `stream_incomplete` without emitting a synthetic +terminal frame under the active response id, then MUST close the downstream +WebSocket with code 1011. #### Scenario: websocket closes before pending responses complete @@ -157,8 +377,29 @@ When an upstream websocket closes while one or more streamed response requests a - **AND** the account receives no failure-health signal - **AND** the request is not transparently replayed +#### Scenario: clean pre-response close does not penalize the account + +- **GIVEN** a hard-affinity HTTP bridge request is pending with no surfaced response event +- **WHEN** the upstream websocket closes cleanly before response output +- **THEN** the proxy records the clean-close retry-circuit outcome +- **AND** the selected account is not penalized + +### Requirement: HTTP SSE stream idle timeouts remain account-neutral + +When an HTTP SSE Responses stream's first upstream event is `response.failed` with `code=stream_idle_timeout`, the proxy MUST exclude that account from the remainder of the same request and MAY fail over to another account. It MUST NOT write account error-health (`record_error`, rate-limit, quota, or permanent failure) for that idle timeout. Request logs MUST still record `stream_idle_timeout` on the idle attempt. + +#### Scenario: First-event stream idle timeout failovers without health penalty + +- **GIVEN** an HTTP SSE Responses stream whose first upstream event is `response.failed` with `code=stream_idle_timeout` +- **AND** another healthy account is available +- **WHEN** the proxy retries the request +- **THEN** the idle account is excluded from the remainder of this request +- **AND** the idle account receives no error-health write +- **AND** the client receives the later account's successful stream +- **AND** the idle attempt's request log still uses `error_code=stream_idle_timeout` + ### Requirement: Single HTTP bridge previous-response misses recover or fail closed -When an HTTP bridge session receives an anonymous upstream `previous_response_not_found` error for a single pending follow-up request, the service MUST treat the error as an internal continuity-loss signal. The same treatment MUST apply when upstream returns `code=invalid_request_error` with the exact canonical message `Invalid previous_response_id.` (allowing only quote/backtick and terminal-period variations) and either omits `param` or sets `param=previous_response_id`. It MUST either recover through the existing previous-response rebind path or rewrite the error to a retryable continuity failure instead of forwarding the raw upstream invalid-request error. Other `invalid_request_error` messages or conflicting `param` values MUST retain their ordinary request-error classification. +When an HTTP bridge session receives an anonymous upstream `previous_response_not_found` error for a single pending follow-up request, the service MUST treat the error as an internal continuity-loss signal. It MUST either recover through the existing previous-response rebind path or rewrite the error to a retryable continuity failure instead of forwarding the raw upstream invalid-request error. #### Scenario: single pending HTTP bridge follow-up loses previous-response continuity - **WHEN** an HTTP `/v1/responses` or `/backend-api/codex/responses` bridge session has exactly one pending request with `previous_response_id` @@ -167,18 +408,6 @@ When an HTTP bridge session receives an anonymous upstream `previous_response_no - **AND** if recovery is unavailable, it emits a retryable continuity failure for that request - **AND** the downstream error code is not `previous_response_not_found` -#### Scenario: canonical invalid previous-response anchor omits param -- **WHEN** an HTTP `/v1/responses` or `/backend-api/codex/responses` bridge session has a pending request with `previous_response_id` -- **AND** upstream emits `code=invalid_request_error`, no `param`, and the exact message `Invalid previous_response_id.` before `response.created` -- **THEN** the service classifies the event as previous-response continuity loss -- **AND** it uses the same proof-gated recovery or fail-closed path as `previous_response_not_found` -- **AND** it does not forward the raw invalid-request response - -#### Scenario: unrelated invalid request remains a request error -- **WHEN** upstream emits `code=invalid_request_error` -- **AND** its `param` conflicts with `previous_response_id` or its message contains additional request-validation text -- **THEN** the service MUST NOT classify that error as previous-response continuity loss - ### Requirement: WebSocket full-resend previous-response misses retry without stale anchor When a direct WebSocket `response.create` request includes both `previous_response_id` and a self-contained full resend payload, the service MUST retain a safe replay body without `previous_response_id`. If upstream rejects the anchor with `previous_response_not_found` before `response.created`, the service MUST reconnect and replay the retained full payload as a fresh turn instead of forwarding the raw upstream invalid-request error. A payload that only carries incremental tool outputs for tool calls that are not also present in the same request is not self-contained and MUST NOT be replayed as a fresh turn without `previous_response_id`. @@ -197,13 +426,42 @@ When a direct WebSocket `response.create` request includes both `previous_respon - **THEN** the service MUST NOT replay that payload as a fresh turn without `previous_response_id` - **AND** the downstream client receives a retryable continuity failure rather than a fabricated fresh turn +### Requirement: Parameterless invalid previous-response errors use continuity recovery + +When an upstream Responses WebSocket rejects an anchored request with `type = "invalid_request_error"`, no `code` or `param`, and the normalized message ``Invalid `previous_response_id``` with or without one trailing period, the service MUST classify the frame as a previous-response continuity miss. It MUST apply the same replay, masking, ownership, and account-health rules as the canonical `previous_response_not_found` error and MUST NOT relay the raw invalid-request frame downstream. A different named parameter or any other trailing punctuation MUST NOT match this error shape. + +#### Scenario: Codex-native delta continuation receives the canonical recovery signal + +- **GIVEN** a Codex-native `/backend-api/codex/responses` request carries `previous_response_id` and delta-only tool output that cannot be replayed safely without its anchor +- **WHEN** upstream returns the parameterless ``Invalid `previous_response_id`.`` error before `response.created` +- **THEN** the downstream client receives a sanitized error with `code = "previous_response_not_found"` +- **AND** the raw upstream envelope and previous response id are not exposed + +#### Scenario: Self-contained full resend is replayed without the rejected anchor + +- **GIVEN** an anchored direct WebSocket request retains a self-contained full-resend body that is safe to replay without `previous_response_id` +- **WHEN** upstream returns the parameterless ``Invalid `previous_response_id`.`` error before `response.created` +- **THEN** the service reconnects and replays the retained body without `previous_response_id` +- **AND** the raw upstream error is not sent downstream + +#### Scenario: Public WebSocket retains generic continuity masking + +- **GIVEN** a public `/v1/responses` WebSocket request carries `previous_response_id` but cannot be replayed safely without its anchor +- **WHEN** upstream returns the parameterless ``Invalid `previous_response_id`.`` error +- **THEN** the downstream client receives the existing sanitized `stream_incomplete` continuity failure +- **AND** neither `previous_response_not_found` nor the raw upstream envelope is exposed + +#### Scenario: Unrelated invalid requests retain their original classification + +- **WHEN** upstream returns `invalid_request_error` with a different message or names a parameter other than `previous_response_id` +- **THEN** the service MUST NOT classify that error as a previous-response continuity miss + ### Requirement: Public Responses errors mask previous-response misses Public Responses endpoints MUST NOT return an OpenAI-shaped `previous_response_not_found` error to clients. If a lower layer still raises or collects that error, the API layer MUST rewrite it to a retryable `stream_incomplete` continuity failure and remove the missing response id from the public payload. #### Scenario: API layer receives an upstream previous-response miss - **WHEN** a public `/responses`, `/v1/responses`, `/responses/compact`, or `/v1/responses/compact` handler receives an error with `code=previous_response_not_found` - **OR** it receives `code=invalid_request_error` with `param=previous_response_id` and a message saying the previous response was not found -- **OR** it receives `code=invalid_request_error` with an absent or matching `param` and the exact canonical message `Invalid previous_response_id.` - **THEN** the response status is retryable - **AND** the public error code is `stream_incomplete` - **AND** the missing `previous_response_id` is not exposed in the response body @@ -393,6 +651,44 @@ When a Responses follow-up depends on previously established continuity state, t - **AND** the takeover retry carries the fresh durable lookup as its continuity anchor even when the turn-state alias registration was lost - **AND** a fresh durable lookup showing a live lease held by another instance — even for a DRAINING row — still fails closed with the retryable `bridge_owner_unreachable` error +### Requirement: Live DRAINING durable leases reject foreign claims + +When a durable HTTP-bridge session is `DRAINING` and another instance still holds an unexpired lease, a foreign `claim_live_session` MUST leave the current owner and lease unchanged even when `allow_takeover` is true. Local session create MUST use the same live-owner predicate as turn-state takeover and MUST NOT treat `DRAINING` alone, or a forced recovery after a missing ring endpoint, as permission to steal a live `DRAINING` lease. The locked claim row, not a stale pre-claim lookup, MUST be the source of the `DRAINING` decision. Expired, released, or `CLOSED` rows MUST remain takeover-eligible. + +#### Scenario: Foreign claim refuses a live DRAINING lease + +- **GIVEN** instance A owns a durable session whose state is `DRAINING` +- **AND** A's lease is still unexpired +- **WHEN** instance B claims the same key with `allow_takeover` false +- **THEN** the row owner remains A +- **AND** the row stays `DRAINING` +- **AND** A's lease expiry is unchanged + +#### Scenario: Forced claim still refuses after an ACTIVE lookup becomes live DRAINING + +- **GIVEN** instance A owns a durable session whose lookup snapshot is still `ACTIVE` +- **AND** instance B would force takeover because A's endpoint is missing +- **AND** A marks the row `DRAINING` with a live lease before B's claim lock +- **WHEN** B claims the same key with `allow_takeover` true +- **THEN** the row owner remains A +- **AND** the row stays `DRAINING` + +#### Scenario: Missing owner endpoint does not force-steal a live DRAINING lease + +- **GIVEN** instance A owns a durable session whose state is `DRAINING` +- **AND** A's lease is still unexpired +- **AND** the ring cannot resolve A's endpoint +- **WHEN** instance B creates a local HTTP-bridge session for the same key +- **THEN** the durable claim is issued with `allow_takeover` false +- **AND** A's owner and lease remain unchanged + +#### Scenario: Expired DRAINING row remains takeover-eligible + +- **GIVEN** a `DRAINING` durable session whose lease is expired or whose owner is released +- **WHEN** another instance claims the same key +- **THEN** that instance becomes the owner +- **AND** the row becomes `ACTIVE` + ### Requirement: Hard continuity owner lookup fails closed When a request depends on hard continuity ownership, the service MUST fail @@ -465,14 +761,42 @@ authoritative actual tier even when it differs from the requested tier. ### Requirement: API key service tier enforcement applies to upstream Responses requests -When an API key carries an enforced service tier, the proxy MUST override any incoming Responses request service tier with that enforced value before forwarding upstream. The legacy alias `fast` MUST be treated as `priority`. +When an API key carries an enforced service tier, the proxy MUST override any +incoming Responses request service tier with that enforced value before route +selection. The omit-equivalent client values `auto` and `default` MUST count as +an omitted tier when tracking whether the enforced value supplied the request's +tier. The legacy alias `fast` MUST be treated as `priority`. + +For a subscription-account route, when an authoritative account catalog says +the selected model never advertises the enforced tier, the proxy MUST remove +that tier from the effective request before account selection and upstream +forwarding. The resulting effective tier MUST survive internal owner +forwarding unchanged. This fallback MUST NOT remove an explicit non-default +client tier, MUST NOT alter a request routed through an external model source, +and MUST NOT apply when the account catalog has no authoritative answer for the +model. #### Scenario: Enforced service tier overrides the request payload +- **GIVEN** the selected account model advertises the `priority` service tier - **WHEN** an API key is configured with `enforcedServiceTier: "priority"` - **AND** an incoming Responses request asks for `service_tier: "default"` - **THEN** the forwarded upstream payload uses `service_tier: "priority"` +#### Scenario: Omit-equivalent request permits account-catalog fallback + +- **GIVEN** an account model authoritatively advertises no `priority` service tier +- **WHEN** an API key is configured with `enforcedServiceTier: "priority"` +- **AND** an incoming Responses request omits `service_tier` or supplies `auto` or `default` +- **THEN** the account-routed upstream payload omits `service_tier` +- **AND** an internal owner forward preserves that effective omission + +#### Scenario: Explicit non-default tier is not downgraded + +- **GIVEN** an account model authoritatively advertises no `priority` service tier +- **WHEN** a client explicitly requests `service_tier: "priority"` or the equivalent `fast` alias +- **THEN** API-key enforcement does not make the tier eligible for account-catalog fallback + #### Scenario: Fast alias is applied as priority - **WHEN** an API key is configured with `enforcedServiceTier: "fast"` @@ -659,14 +983,22 @@ The system SHALL accept `input_file` content items that reference an upload by ` ### Requirement: Responses requests with input_file.file_id route to the upload's account -A `/v1/responses`, `/backend-api/codex/responses`, or `/responses/compact` request that references an `{type: "input_file", file_id}` content item SHALL be routed to the upstream account that registered the file via `POST /backend-api/files` when an in-memory pin for that `file_id` is still live. A live file pin is hard ownership evidence: it MUST override prompt-cache or bare process-session locality and MUST agree with independently resolved turn-state, previous-response, bridge, or other hard ownership. +A `/v1/responses`, `/backend-api/codex/responses`, or `/responses/compact` request that references an `{type: "input_file", file_id}` content item SHALL be routed to the upstream account that registered the file via `POST /backend-api/files` when a durable, unexpired pin for that `file_id` exists. The pin MUST be visible to every replica that shares the application database. A live file pin is hard ownership evidence: it MUST override prompt-cache or bare process-session locality and MUST agree with independently resolved turn-state, previous-response, bridge, or other hard ownership. + +When multiple `file_id`s are referenced, all live pins MUST resolve to the same account. If at least one ID has a live pin and another ID has no live pin, the request MUST fail with `file_owner_unavailable`; if live pins resolve to different accounts, it MUST fail with `continuity_owner_conflict`. If none of the referenced IDs has a live pin, the proxy MUST preserve compatibility with files registered directly upstream or before durable ownership was observed by forwarding the opaque IDs verbatim under ordinary unpinned routing. -When multiple `file_id`s are referenced, all live pins MUST resolve to the same account. If at least one ID has a live pin and another ID has no live pin, the request MUST fail with `file_owner_unavailable`; if live pins resolve to different accounts, it MUST fail with `continuity_owner_conflict`. If none of the referenced IDs has a live pin, the proxy MUST preserve compatibility with files registered directly upstream or before the current process observed the upload by forwarding the opaque IDs verbatim under ordinary unpinned routing. +A live durable pin MUST NOT be reassigned to another account. Repeating the claim for the same account MUST be idempotent and MAY renew its expiry; an expired identifier MAY be claimed by a later upload. + +Every hard file-owner decision MUST read the shared database and MUST NOT rely on a process-local owner cache. Authenticated inter-replica forwarding metadata MAY corroborate the freshly resolved durable owner but MUST NOT replace the receiver's database read. A missing or conflicting receiver-side durable owner MUST fail closed before account selection or upstream invocation. Pin expiry, reclaim, and cleanup MUST use database-authoritative statement time rather than a replica's application clock. + +For a streaming Responses request whose durable file-owner lookup runs in the stream service, any API-key usage reservation acquired before that lookup MUST have exactly one cleanup owner if resolution fails or the request is cancelled. Within one replica, the API layer MUST own cleanup until the direct stream service enters its settlement-guarded `try/finally` or the local HTTP-bridge service successfully submits the request and installs its request-state finalizer. The service finalizer MUST own cleanup after that explicit boundary so those layers cannot both release the reservation. Merely completing the durable lookup MUST NOT transfer cleanup before a service finalizer is active, and an initial SSE heartbeat MUST NOT transfer ownership to the client. + +When an authenticated HTTP-bridge origin forwards that reservation to another replica, the receiver MUST delay its successful HTTP 200 response until its service finalizer is active. That 200 response MUST be the cleanup-handoff acknowledgement that transfers ownership from the origin to the receiver. The origin MUST distinguish a request that has not been dispatched, a dispatch with no observed response status, a successful HTTP 200 acknowledgement, and a definitive non-200 rejection. Before dispatch or after a definitive non-200, receiver-side owner-revalidation failure or cancellation MUST propagate with cleanup remaining at the origin. After dispatch when no response status can be observed, the origin MUST NOT actively release or replay the reservation because the receiver may already own settlement; receiver settlement or bounded stale-reservation cleanup MUST resolve that ambiguity. After the acknowledgement, the receiver service finalizer MUST remain authoritative even if no upstream event has arrived. If a bounded startup probe hands pending preflight work to the response body and the body closes first, the active owner MUST cancel and await that work before scheduling one cancellation-safe release attempt. If that persistence write fails, the same cleanup owner MUST schedule one follow-up release attempt instead of abandoning the reservation. An SSE heartbeat or another frame MUST NOT transfer cleanup ownership. Compact service settlement MUST likewise suppress a second API-layer release after its single settlement attempt. Once a forwarded compact service has made that settlement attempt, including when both the primary finalize and the fallback release fail, a later receiver-side output validation failure or a `usage_settlement_failed` error MUST preserve HTTP 200 as the cleanup-handoff acknowledgement and surface a terminal `response.failed` event with the stable error code; it MUST NOT become a non-200 rejection that permits origin release or replay. A client disconnect after the initial SSE heartbeat MUST close the service stream even when the startup probe already completed. A cleanup-store failure MUST NOT replace a stable owner-resolution error. A cleanup-store failure MUST NOT mask the original stable owner error or cancellation. Owner-lookup failure or cancellation MUST NOT trigger account failover or another upstream attempt. #### Scenario: file_id pin drives routing for an input_file response -- **GIVEN** a `POST /backend-api/files` registered `file_xyz` through `account_a` -- **WHEN** a `/v1/responses` request references `{"type": "input_file", "file_id": "file_xyz"}` +- **GIVEN** a `POST /backend-api/files` registered `file_xyz` through `account_a` on one replica +- **WHEN** a `/v1/responses` request references `{"type": "input_file", "file_id": "file_xyz"}` on another replica - **THEN** the proxy MUST route the request to `account_a` #### Scenario: file_id pin overrides prompt-cache locality @@ -677,11 +1009,189 @@ When multiple `file_id`s are referenced, all live pins MUST resolve to the same #### Scenario: opaque file_id without a live pin remains compatible -- **GIVEN** a request references a `file_id` registered directly upstream or before the current process observed its upload -- **AND** no referenced file has a live in-memory pin +- **GIVEN** a request references a `file_id` registered directly upstream or before the system durably observed its upload +- **AND** no referenced file has a live durable pin - **WHEN** the request is routed - **THEN** the proxy MUST forward the `file_id` verbatim under ordinary unpinned routing -- **AND** it MUST NOT reject the request solely because local owner metadata is absent +- **AND** it MUST NOT reject the request solely because owner metadata is absent + +#### Scenario: file finalize resolves ownership across replicas + +- **GIVEN** one replica registered `file_xyz` through `account_a` +- **WHEN** another replica handles `POST /backend-api/files/file_xyz/uploaded` +- **THEN** the proxy MUST finalize the file through `account_a` +- **AND** it MUST NOT fall back to a different eligible account + +#### Scenario: concurrent live ownership claims do not overwrite + +- **GIVEN** `file_xyz` has a live durable pin to `account_a` +- **WHEN** another replica attempts to pin `file_xyz` to `account_b` +- **THEN** the claim MUST fail with `continuity_owner_conflict` +- **AND** subsequent routing MUST still resolve `file_xyz` to `account_a` + +#### Scenario: a replica observes an expired pin reclaimed by another replica + +- **GIVEN** a replica previously resolved `file_xyz` to `account_a` +- **AND** the durable pin expires and another replica claims `file_xyz` for `account_b` +- **WHEN** the first replica resolves `file_xyz` again +- **THEN** it MUST read the durable owner and return `account_b` +- **AND** it MUST NOT return `account_a` from process-local state + +#### Scenario: durable owner lookup failure fails closed + +- **GIVEN** a request references a file whose owner decision requires the shared database +- **WHEN** the durable owner lookup fails +- **THEN** the request MUST fail before selecting or invoking an unpinned fallback account + +#### Scenario: cancellation during owner lookup releases admission state + +- **GIVEN** a request has acquired an API-key usage reservation before durable file-owner resolution completes +- **WHEN** the request is cancelled while the owner lookup is pending +- **THEN** exactly one cleanup owner MUST attempt to release or settle the reservation +- **AND** no account selection, upstream invocation, retry, or failover may occur + +#### Scenario: delayed owner failure after stream handoff releases admission state + +- **GIVEN** the streaming startup probe expires while durable file-owner resolution is still pending +- **WHEN** the lookup later fails or the response body is closed +- **THEN** the origin API MUST cancel and await any still-pending lookup +- **AND** the origin API MUST make exactly one release attempt +- **AND** a lookup failure MUST be represented by the stable `file_owner_unavailable` error + +#### Scenario: failed reservation release is retried + +- **GIVEN** a startup or disconnect cleanup owns an API-key reservation +- **WHEN** the first persistence release fails +- **THEN** the cleanup owner MUST schedule one follow-up release attempt +- **AND** it MUST NOT leave the reservation reserved with no later cleanup path + +#### Scenario: forwarded owner metadata is revalidated against durable ownership + +- **GIVEN** a replica receives authenticated forwarding metadata that identifies `account_a` as a referenced file's owner +- **WHEN** the receiver's fresh durable lookup has no live owner or identifies a different owner +- **THEN** the receiver MUST fail closed +- **AND** it MUST NOT route using the forwarded value alone +- **AND** it MUST propagate the preflight failure to the origin without releasing the origin reservation +- **AND** the originating request path MUST remain the sole cleanup owner because no successful handoff acknowledgement was sent + +#### Scenario: forwarded stream acknowledges cleanup ownership before HTTP 200 + +- **GIVEN** the origin forwards a file-pinned streaming request and its API-key reservation to the authenticated owner replica +- **WHEN** the receiver completes durable owner revalidation and installs its service settlement finalizer +- **THEN** the receiver MAY return HTTP 200 as the cleanup-handoff acknowledgement +- **AND** the origin MUST stop releasing the reservation after receiving that acknowledgement +- **AND** cancellation before the first upstream event MUST invoke only the receiver's service finalizer + +#### Scenario: ambiguous owner dispatch defers active origin cleanup + +- **GIVEN** the origin has begun dispatching a signed forwarded request carrying its reservation +- **WHEN** the transport fails before the origin can observe an HTTP status +- **THEN** the origin MUST NOT actively release or replay the reservation +- **AND** receiver settlement or stale-reservation cleanup MUST remain the only recovery paths + +#### Scenario: definitive owner rejection retains origin cleanup + +- **GIVEN** the origin dispatches a signed forwarded request carrying its reservation +- **WHEN** the receiver returns a non-200 response without acknowledging cleanup handoff +- **THEN** the origin MUST make exactly one cancellation-safe release attempt +- **AND** the receiver MUST NOT settle the origin reservation + +#### Scenario: owner non-200 remains a rejection after body-read failure + +- **GIVEN** the origin has observed a non-200 owner-forward status +- **WHEN** reading the rejection body then fails +- **THEN** the origin MUST treat the outcome as a definitive rejection +- **AND** it MUST NOT reclassify the dispatch as ambiguous + +#### Scenario: compact service settlement is not released twice + +- **GIVEN** terminal or direct compaction receives an API-key usage reservation +- **WHEN** the compact service makes its single settlement or release attempt +- **THEN** the API layer MUST NOT issue another release for that reservation +- **AND** a pre-service failure MUST still leave exactly one release attempt at the API layer + +#### Scenario: malformed compact output after settlement preserves handoff + +- **GIVEN** a forwarded terminal compact request whose receiver service has made its single settlement attempt +- **WHEN** the settled response lacks a valid compaction output item +- **THEN** the receiver MUST return HTTP 200 as the cleanup-handoff acknowledgement +- **AND** it MUST emit a terminal `response.failed` event +- **AND** the origin MUST NOT release or replay the reservation + +#### Scenario: compact settlement failure after fallback preserves handoff + +- **GIVEN** a forwarded terminal compact request whose receiver service has made its single settlement attempt +- **WHEN** usage settlement fails after a successful fallback release +- **THEN** the receiver MUST return HTTP 200 as the cleanup-handoff acknowledgement +- **AND** it MUST emit a terminal `response.failed` event with code `usage_settlement_failed` +- **AND** the origin MUST NOT release or replay the reservation + +#### Scenario: compact settlement attempt preserves handoff when both writes fail + +- **GIVEN** a forwarded terminal compact request whose receiver service attempts settlement +- **WHEN** both reservation finalization and the fallback release fail +- **THEN** the receiver MUST still return HTTP 200 as the cleanup-handoff acknowledgement +- **AND** it MUST emit a terminal `response.failed` event with code `usage_settlement_failed` +- **AND** the origin MUST NOT release or replay the reservation + +#### Scenario: completed startup probe still closes the service stream + +- **GIVEN** the streaming startup probe already obtained the first service event +- **WHEN** the client disconnects after the initial SSE heartbeat +- **THEN** the origin MUST close the service stream +- **AND** reservation cleanup MUST still run if ownership has not transferred + +### Requirement: Soft HTTP-bridge 1011 reconnect keeps a live file-pin owner + +A still-unsubmitted HTTP-bridge reconnect MUST keep a live `input_file.file_id` +pin as a required owner after a soft session closes with `1011`. +When an HTTP-bridge session is soft (prompt-cache or request locality) and +upstream closed it with `1011`, a still-unsubmitted request that carries a +live `input_file.file_id` pin MUST keep that pin account as a required +reconnect owner. The proxy MUST NOT exclude that account solely because the +close code was `1011`, and MUST NOT fall back to another account while the +pin is live. If the required pin account is already excluded or cannot be +reconnected, the proxy MUST fail closed with the existing required-owner +unavailable error. A soft `1011` reconnect that has no live file pin and no +other required owner MAY still skip the closed account. + +#### Scenario: Soft 1011 reconnect keeps the file-pin account required + +- **GIVEN** a live in-memory pin `file_xyz -> account_a` +- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz` +- **WHEN** the proxy reconnects that session +- **THEN** account selection MUST treat `account_a` as the required owner +- **AND** it MUST NOT add `account_a` to the excluded-account set solely because of `1011` +- **AND** it MUST NOT enable preferred-account fallback to another account + +#### Scenario: Soft 1011 reconnect without a file pin may skip the closed account + +- **GIVEN** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the still-unsubmitted request has no live file pin and no other required owner +- **WHEN** the proxy reconnects that session +- **THEN** account selection MAY exclude `account_a` and choose another eligible account + +#### Scenario: Soft 1011 file-pin reconnect fails closed when the required owner cannot be selected + +- **GIVEN** a live in-memory pin `file_xyz -> account_a` +- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz` +- **AND** account selection cannot return `account_a` +- **WHEN** the proxy reconnects that session +- **THEN** the proxy MUST fail closed with the existing required-owner unavailable error +- **AND** it MUST NOT replace that envelope with a generic selection failure + +#### Scenario: Soft 1011 file-pin reconnect fails closed when the required owner cannot be connected + +- **GIVEN** a live in-memory pin `file_xyz -> account_a` +- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz` +- **AND** account selection returns `account_a` +- **AND** opening a replacement upstream for `account_a` fails +- **WHEN** the proxy reconnects that session on submit +- **THEN** the client-visible error MUST be the existing required-owner unavailable error +- **AND** it MUST NOT be replaced with a generic `upstream_unavailable` envelope ### Requirement: Codex backend session_id preserves account affinity When a backend Codex Responses or compact request includes a non-empty accepted session header, the service MUST use that value as the routing affinity key for upstream account selection unless the client supplied a non-empty `x-codex-turn-state` header. If the request lacks a client-supplied `prompt_cache_key`, the service MUST derive and attach a stable `prompt_cache_key` before upstream forwarding so account affinity and upstream prompt-cache routing can coexist. Accepted session headers are `session_id`, `session-id`, `x-codex-session-id`, `x-codex-conversation-id`, and `thread-id`, in that priority order. @@ -1344,7 +1854,7 @@ the existing pre-visible forced-refresh and eligible-account failover behavior. #### Scenario: file-pinned compact request fails closed on refresh transport failure -- **GIVEN** `file_pinned` was uploaded through `account_a` and its in-memory pin is live +- **GIVEN** `file_pinned` was uploaded through `account_a` and its durable pin is live - **AND** a compact request references `{"type": "input_file", "file_id": "file_pinned"}` - **WHEN** `account_a` fails token refresh with a pre-visible transport or connection error - **THEN** the proxy returns an upstream-unavailable error for that compact request @@ -1761,8 +2271,9 @@ terminal `response.failed` SSE event with error code `stream_incomplete`, record the request-log row as an upstream `stream_incomplete` error, and apply the normal transient upstream account-health signal. If the downstream client cancels or disconnects before a terminal event, the proxy MUST record the -request-log row as a downstream `client_disconnected` error and MUST NOT -penalize the upstream account. +request-log row with status `cancelled`, downstream error code +`client_disconnected`, and downstream failure metadata, and MUST NOT penalize +the upstream account. #### Scenario: Raw stream upstream EOF is not successful @@ -1781,7 +2292,7 @@ penalize the upstream account. - **GIVEN** a raw HTTP streaming Responses request has not observed a terminal SSE event - **WHEN** the downstream client cancels or disconnects from the stream -- **THEN** the request log stores status `error`, error code +- **THEN** the request log stores status `cancelled`, error code `client_disconnected`, and downstream failure metadata - **AND** the selected account is not penalized for the client-side close @@ -1922,34 +2433,6 @@ When a direct Responses WebSocket request has a prepared retry-safe fresh upstre - **THEN** the service reconnects and replays the prepared no-anchor request - **AND** it does not rewrite the turn to `previous_response_owner_unavailable` -### Requirement: Exact HTTP store-context trims recover rejected proxy anchors from the retained request -When an HTTP Responses bridge request arrives without `previous_response_id`, -contains complete conversation context, and has an input prefix that exactly -matches the current live session checkpoint, the service MAY trim that prefix -and inject the checkpoint anchor. Before doing so, it MUST seal a request-local -proof binding the logical session key, owning account, response anchor, stored -input count and fingerprint, pending tool-call manifest, and full request -fingerprint. If upstream rejects the proxy-injected anchor before any response -event, the service MAY retry the retained complete request exactly once without -the anchor on the same account. It MUST preserve the prior durable checkpoint -until replacement completion and MUST keep incomplete inputs, changed proof -inputs, client-supplied anchors, post-event failures, and account changes fail -closed. - -#### Scenario: verified store-context trim recovers in the same HTTP request -- **GIVEN** a live HTTP bridge checkpoint has an owning account, response anchor, and exact stored input fingerprint -- **AND** an unanchored follow-up exactly matches that prefix and retains prior response or pending tool context -- **WHEN** the bridge trims the prefix, injects the anchor, and upstream rejects it before any response event -- **THEN** the service reconnects once on the same account -- **AND** sends the sealed complete request once without `previous_response_id` -- **AND** replacement completion atomically supersedes the old checkpoint - -#### Scenario: store-context recovery cannot broaden replay authority -- **GIVEN** the input is incomplete, the exact prefix or proof binding changed, the anchor was supplied by the client, an event was observed, or the replacement send is ambiguous -- **WHEN** recovery is evaluated -- **THEN** the service does not dispatch a request-local unanchored replay -- **AND** it does not switch accounts or consume another replay attempt - ### Requirement: Codex WebSocket prewarm completions are classified separately For a direct Responses WebSocket, the service MUST treat Codex turn metadata received on the HTTP handshake as connection-scoped metadata rather than applying its `request_kind` to every `response.create` frame. The service MUST classify an individual turn as `prewarm` when the connection metadata is `prewarm` and either that turn carries `generate: false` or its completed usage reports zero output tokens. Other turns on the same connection MUST be classified as `normal`. @@ -3371,6 +3854,7 @@ and object key order. - **WHEN** two requests differ only in tool array order or tool object key order - **THEN** their tools affinity/observability hash is identical + ### Requirement: Streaming events are parsed once and re-serialized only when modified Within each streaming layer (core client consumer, streaming mixin, bridge upstream reader, /v1 normalizers), an SSE event's JSON payload MUST be parsed at most once and reused by that layer's consumers, and an event that no consumer modified MUST NOT be re-serialized by the /v1 normalizers. Event framing, payload contents, dedupe/rewrite semantics, and error normalization MUST be unchanged. @@ -3503,3 +3987,1470 @@ exist. The default threshold MUST be no greater than seven failures. - **WHEN** the dead-owner classifier evaluates lease liveness against the application's naive-UTC clock - **THEN** both timestamps MUST be normalized to naive UTC before comparison - **AND** the anchored-lookup path MUST NOT raise on mixed-awareness datetimes + +### Requirement: HTTP bridge model-transition isolation is single-pass + +When an HTTP bridge request cannot reuse the session selected by its incoming affinity because that session uses an incompatible model, the service MUST preserve the resulting internal model-parallel key until bridge creation or reuse completes. It MUST NOT reapply the original session-header or turn-state fallback to the same request after selecting that fork. + +#### Scenario: Fresh turn state falls back to a session on another model + +- **GIVEN** a request carries a fresh generated turn-state header and a session header whose active bridge uses an incompatible model +- **WHEN** lookup isolates the request with an internal model-parallel key +- **THEN** lookup emits at most one model-transition fork for that request scope +- **AND** bridge creation continues under the internal key without closing or reusing the incompatible session + +#### Scenario: Follow-up fallback has no previous-response lookup + +- **GIVEN** a request carries a fresh generated turn-state header, a `previous_response_id` without a local or durable lookup, and a session header whose active bridge uses an incompatible model +- **WHEN** lookup isolates the request with an internal model-parallel key +- **THEN** the session-header fallback remains an anchored continuation for the rest of that lookup/create operation +- **AND** bridge creation continues under the internal key without a `continuity_lost` error + +#### Scenario: Full cache preserves the incompatible parent + +- **GIVEN** the HTTP bridge cache is at its session limit and a model transition isolates a session-header fallback into a child key +- **WHEN** creation needs to evict an idle session +- **THEN** the incompatible session-header parent MUST NOT be selected for that eviction +- **AND** ordinary LRU eviction remains eligible for other idle sessions + +#### Scenario: In-flight parent completes before model isolation + +- **GIVEN** a request waits for an in-flight session-header parent whose completed bridge uses an incompatible model +- **WHEN** the request isolates itself with an internal model-parallel key after that wait +- **THEN** the completed parent MUST receive the same capacity-eviction protection as an immediately available parent + +#### Scenario: Compatible session fallback remains reusable + +- **GIVEN** a request carries a fresh generated turn-state header and a session header whose active bridge uses a compatible model +- **WHEN** lookup applies the session-header fallback +- **THEN** the compatible bridge remains eligible for normal reuse + +### Requirement: Standalone Codex web search is forwarded faithfully + +The proxy SHALL expose `POST /backend-api/codex/alpha/search` through the same +proxy-authenticated Codex control-request path used by other unary Codex control +endpoints. The proxy MUST preserve the inbound request body and query parameters, +MUST apply the existing API-key scope, account selection, token refresh, session +affinity, failover, and upstream-route policies, and MUST forward the request to +the upstream `POST /codex/alpha/search` path. Successful downstream responses +MUST preserve the upstream status and body and MUST include only response +headers allowed by the existing Codex control-response policy. Final non-2xx +responses MUST preserve their status while using the existing Codex control +OpenAI error-envelope normalization. The proxy MUST NOT parse, normalize, or +invent a local schema for successful search requests or responses. + +#### Scenario: authenticated standalone search reaches the upstream Codex path + +- **GIVEN** a valid proxy API key and at least one eligible ChatGPT account +- **WHEN** Codex sends `POST /backend-api/codex/alpha/search` with a JSON body and + query parameters +- **THEN** the proxy forwards the unchanged body and query parameters to + `POST /codex/alpha/search` using the selected account credentials +- **AND** the downstream client receives the upstream status and body + +#### Scenario: unsafe upstream response headers are not exposed + +- **WHEN** the upstream search response includes both allowlisted metadata and + a response header outside the Codex control-response allowlist +- **THEN** the proxy returns the allowlisted metadata +- **AND** it omits the non-allowlisted response header + +#### Scenario: final upstream search failures use the control error contract + +- **WHEN** upstream search failure handling finishes with a non-2xx response +- **THEN** the proxy preserves the final HTTP status +- **AND** it returns the failure through the existing OpenAI error envelope +- **AND** existing account refresh, health, and failover handling remains active + +#### Scenario: unsupported methods do not enter search forwarding + +- **WHEN** a client sends a non-POST request to + `/backend-api/codex/alpha/search` +- **THEN** the request does not enter the upstream search forwarding path + +### Requirement: Pre-acceptance account-model rejections fail over safely + +When upstream rejects a Responses request with `invalid_request_error` and the exact message `The '' model is not supported when using Codex with a ChatGPT account.` before accepting the response, the proxy MUST classify the failure internally as `account_model_unsupported`. The quoted model MUST match +the requested model. For native WebSocket, HTTP responses bridge, and raw +HTTP/SSE transports, the proxy MUST make at most one transparent attempt on a +different account that advertises the same model, provided the request can move +without violating continuation or uploaded-file ownership. The proxy MUST +exclude the rejecting account only for that request and MUST NOT record an +account-health penalty for this rejection. + +The proxy MUST NOT replay after any response id recognized in an upstream payload, +including a `response.failed` payload that carries `response.id` even when +`response.created` was not observed or an `error` payload with top-level +`response_id`, a nonterminal `response.*` +event, downstream sequence/output, another pending request on the shared +socket, or an earlier replay. If no compatible replacement is available, or +the request is account-bound, the proxy MUST preserve the original upstream +400 error instead of replacing it with `no_accounts`, `stream_incomplete`, or +another proxy-generated failure. + +#### Scenario: stale model route retries another advertising account + +- **GIVEN** two accounts advertise the requested model in the current routing snapshot +- **AND** upstream rejects the first account with the exact account/model unsupported envelope before `response.created` +- **WHEN** the request has no hard account or uploaded-file binding +- **THEN** the proxy excludes the first account for this request and retries once on the second account +- **AND** it forwards only the replacement attempt's response events downstream +- **AND** it does not penalize the first account's global health + +#### Scenario: no replacement preserves the upstream rejection + +- **GIVEN** upstream rejects a pre-acceptance request with the exact account/model unsupported envelope +- **AND** no other compatible account is available +- **WHEN** transparent failover cannot select a replacement +- **THEN** the client receives the original HTTP 400 `invalid_request_error` +- **AND** the error is not rewritten to `no_accounts`, `stream_incomplete`, or HTTP 502 + +#### Scenario: selected replacement failure is surfaced + +- **GIVEN** upstream rejects a pre-acceptance request with the exact account/model unsupported envelope +- **AND** the proxy selects a different compatible replacement account +- **WHEN** that replacement attempt fails before acceptance +- **THEN** the client receives the replacement attempt's failure +- **AND** the skipped account's original HTTP 400 is not used as a fallback +- **AND** the proxy does not select a third account after a retryable replacement + refresh, transport, or server failure + +#### Scenario: failed bridge replacement retires without restoring rejected metadata + +- **GIVEN** an HTTP responses bridge reconnect has selected and installed a + replacement account after an account/model rejection +- **WHEN** replacement response-create lease acquisition or request send fails +- **THEN** the proxy forwards the replacement failure and retires that bridge + session after draining the rejected request +- **AND** it does not restore the rejected account's turn state or headers onto + the replacement socket + +#### Scenario: accepted or visible request is never replayed + +- **WHEN** the account/model unsupported envelope arrives after a response id, a nonterminal response event, downstream sequence/output, or an earlier replay +- **THEN** the proxy does not transparently replay the request on another account + +#### Scenario: account-bound request is never migrated + +- **WHEN** a rejected request depends on an account-scoped uploaded file or an owner-bound continuation without a verified self-contained fresh replay body +- **THEN** the proxy does not move the request to another account +- **AND** it preserves the original upstream rejection + +### Requirement: Model-capacity messages are retryable transient failures + +When upstream returns a temporary model-capacity failure whose message says that the selected model is at capacity, the proxy MUST treat the failure as retryable transient even if the upstream error code or HTTP status would otherwise look non-retryable. + +#### Scenario: Selected model capacity with invalid request code is retryable + +- **WHEN** upstream returns an error envelope with `error.message = "Selected model is at capacity. Please try a different model."` +- **AND** the normalized error code is `invalid_request_error` +- **AND** the HTTP status is `400` +- **THEN** `classify_upstream_failure` returns `failure_class = "retryable_transient"` +- **AND** pre-visible streaming/websocket paths are eligible to retry or fail over instead of surfacing a terminal client error. + +#### Scenario: Serialized selected-model capacity event surfaces without replay + +- **WHEN** a streaming Responses request receives a first upstream `response.failed` or `error` event whose message says the selected model is at capacity +- **AND** no downstream-visible output has been emitted +- **THEN** the proxy MUST surface that terminal event without transparently re-POSTing the request +- **AND** the absence of an upstream response id MUST NOT by itself prove the POST was safe to replay. + +#### Scenario: Post-connect body-read disconnect is not replayed as capacity retry + +- **WHEN** a streaming Responses request fails while reading the upstream stream body after the upstream request has been dispatched +- **AND** the failure is an `aiohttp` client error, timeout, EOF, or other transport/body-read close without typed pre-dispatch provenance +- **THEN** the proxy MUST surface the stream failure to the downstream client +- **AND** the proxy MUST NOT transparently re-POST the request as a model-capacity retry. + +#### Scenario: Websocket connect failure retries before request dispatch + +- **WHEN** an upstream websocket handshake raises a typed connector failure or connect timeout before the `response.create` frame is sent +- **THEN** the proxy MUST preserve typed pre-dispatch provenance and MAY retry or fail over before any downstream-visible output +- **AND** a websocket transport selection MUST NOT turn that failure into a terminal serialized SSE event. + +#### Scenario: Direct HTTP TLS verification failure is not retried + +- **WHEN** a direct HTTP stream raises a certificate or TLS connector failure before request dispatch +- **THEN** the proxy MUST surface the TLS failure without transparently retrying or failing over +- **AND** pre-dispatch provenance MUST NOT classify the non-transient TLS failure as retryable. + +#### Scenario: Quota and rate-limit codes retain their stronger classification + +- **WHEN** upstream returns a quota or rate-limit error code +- **THEN** the proxy MUST keep classifying it as quota or rate-limit before applying message-based model-capacity detection. + +#### Scenario: Post-refresh transient exhaustion preserves every health signal + +- **WHEN** one or more accounts each exhaust multiple same-account post-refresh transient retries before the request succeeds or terminates +- **THEN** the proxy MUST settle API-key usage before recording any deferred account-health failure +- **AND** each exhausted account MUST receive exactly one classified health failure plus one additional failure for every remaining exhausted retry +- **AND** selecting or exhausting a later account MUST NOT replace, lose, or duplicate an earlier account's deferred failures. + +#### Scenario: Classified quota failures still use the model-capacity replay wait + +- **WHEN** a replayable pre-created HTTP bridge request receives the selected-model capacity message with a quota or + rate-limit error code +- **THEN** the proxy MUST preserve that quota or rate-limit classification for account health handling +- **AND** the proxy MUST still apply the model-capacity wait before replaying the request. + +### Requirement: HTTP bridge model-capacity retry waits preserve stream contracts + +The proxy MUST wait before replaying a pre-created HTTP bridge request with a selected-model capacity failure only +when the failure happened before any downstream-visible response event and the request is still replayable as a fresh +request. + +#### Scenario: Public propagated-error streams do not receive pre-retry keepalives + +- **WHEN** a `/v1/responses`-compatible HTTP bridge stream is configured to propagate startup HTTP errors +- **AND** upstream returns a selected-model capacity error before `response.created` +- **THEN** the proxy MUST NOT emit `codex.keepalive` or account-capacity wait events before the retry completes. + +#### Scenario: Replay waits remain bounded by the original bridge deadline + +- **WHEN** the selected-model capacity error arrives near or after the original bridge request deadline +- **THEN** the proxy MUST NOT start a fresh upstream replay after that deadline is exhausted. + +#### Scenario: Only fresh replayable bridge requests wait + +- **WHEN** the selected-model capacity error belongs to an anchored request that cannot be replayed without + `previous_response_id` +- **THEN** the proxy MUST forward the terminal error promptly without sleeping for the model-capacity retry delay. + +#### Scenario: Retry-safe injected anchors still wait + +- **WHEN** the proxy injected `previous_response_id` and retained a fresh request body that is safe to replay without + that anchor +- **AND** upstream returns a selected-model capacity error before visible output +- **THEN** the proxy MUST apply the model-capacity wait before stripping the injected anchor and replaying the fresh + request. + +#### Scenario: Remote-owner relay preserves the hidden startup wait + +- **WHEN** an origin replica forwards a bridge request to its remote owner +- **THEN** the origin MUST keep its startup probe pending until the owner relay returns response headers or a terminal + startup error +- **AND** a selected-model capacity wait on the owner MUST NOT cause the origin to commit HTTP 200 before that wait + completes. + +#### Scenario: Waiting keeps the retry tied to the pending request + +- **WHEN** the proxy waits before replaying a selected-model capacity failure +- **THEN** the request MUST remain reserved in the bridge pending queue while it waits +- **AND** the proxy MUST retain the session response-create gate so a younger request cannot enter while the sole + upstream reader is sleeping +- **AND** the proxy MUST release account-level and shared response-create capacity during the wait +- **AND** the proxy MUST reacquire both capacity leases before sending the replay +- **AND** the proxy MUST skip the replay if that queued request detaches before the wait completes. + +### Requirement: WebSocket stale-anchor failures include diagnostic metadata +When a direct Responses WebSocket request fails closed because upstream rejects `previous_response_id` with `previous_response_not_found`, the service MUST emit stale-anchor diagnostic metadata in operator logs and request-log failure metadata. The metadata MUST distinguish `previous_response_source` (`client_supplied`, `proxy_injected`, or `unknown`), whether a fresh no-anchor replay body was available, owner lookup outcome/source, whether the matched previous response belongs to the same Codex session when known, and the previous-response age in seconds when known. The metadata MUST NOT expose raw `previous_response_id` values or request payload content. + +#### Scenario: client-supplied stale anchor is classifiable +- **GIVEN** a direct WebSocket request arrives with a client-supplied `previous_response_id` +- **AND** upstream rejects that anchor with `previous_response_not_found` +- **THEN** the continuity failure log and request-log failure metadata identify `previous_response_source=client_supplied` +- **AND** they include owner lookup and replay-availability metadata without raw response ids + +#### Scenario: proxy-injected stale anchor is classifiable +- **GIVEN** codex-lb injects a session-continuity `previous_response_id` into a direct WebSocket request +- **AND** upstream rejects that anchor with `previous_response_not_found` +- **THEN** the continuity failure log and request-log failure metadata identify `previous_response_source=proxy_injected` +- **AND** they state whether a retry-safe fresh no-anchor replay body was available +- **AND** owner lookup, age, and same-session fields remain explicit as `unknown` when unavailable rather than being omitted + +#### Scenario: stale anchor owner hit records age and session relationship +- **GIVEN** owner lookup finds a previous response row for the rejected anchor +- **WHEN** the direct WebSocket request fails closed with `previous_response_not_found` +- **THEN** the stale-anchor diagnostics include the owner lookup source +- **AND** include previous-response age seconds and same-session status when those values can be derived + +#### Scenario: account-only cache hits do not guess owner session metadata +- **GIVEN** owner resolution hits a request cache entry that retains the account id but not the matched request-log row +- **WHEN** the direct WebSocket request fails closed with `previous_response_not_found` +- **THEN** the stale-anchor diagnostics identify the owner lookup source as the request cache +- **AND** leave previous-response age and same-session status unknown rather than inferring them from the current request scope + +### Requirement: Responses HTTP ingress uses the expanded bounded budget + +HTTP requests to `/v1/responses` and `/backend-api/codex/responses`, including trailing-slash variants, MUST use the larger of `max_decompressed_body_bytes` and `max_decompressed_responses_body_bytes` as both the raw-body and decompressed-body ingress budget. The Responses-specific default MUST remain 128 MiB. + +The trailing-slash variants MUST be hidden aliases of the canonical HTTP handlers rather than redirects, so streamed bodies receive the same admission, authorization, and route behavior. + +If either representation exceeds that budget, the service MUST stop before route logic or upstream forwarding and return HTTP 413 with an OpenAI-compatible error envelope carrying `error.code = payload_too_large` and `error.type = invalid_request_error`. + +This transport-ingress 413 applies before parsing and is distinct from the existing application-level oversized-`response.create` guard. A request that fits the 128 MiB transport budget but still exceeds the upstream websocket budget after historical slimming MUST retain the existing HTTP 400 `payload_too_large` behavior and `param = input`. + +#### Scenario: Larger Responses request fits both ingress checks + +- **WHEN** a Responses HTTP request is larger than the general budget but no larger than the Responses budget in either raw or decompressed form +- **THEN** the ingress guards allow the request to continue to Responses route handling + +#### Scenario: Trailing-slash Responses request is admitted without redirect + +- **WHEN** a client sends a chunked HTTP request to `/v1/responses/` or `/backend-api/codex/responses/` +- **THEN** the service applies the same ingress budget and handler as the corresponding canonical path +- **AND** it does not return a trailing-slash redirect before consuming the guarded body + +#### Scenario: Responses raw body exceeds its budget + +- **WHEN** a Responses HTTP request's raw body exceeds the Responses budget +- **THEN** the service returns HTTP 413 with `error.code = payload_too_large` and `error.type = invalid_request_error` +- **AND** the service does not invoke Responses route logic or forward the request upstream + +#### Scenario: Responses expanded body exceeds its budget + +- **WHEN** an encoded Responses HTTP request fits the raw budget but expands beyond the Responses budget +- **THEN** the service returns HTTP 413 with `error.code = payload_too_large` and `error.type = invalid_request_error` +- **AND** the service does not invoke Responses route logic or forward the request upstream + +#### Scenario: Post-slimming application rejection remains 400 + +- **WHEN** a Responses HTTP request fits the raw and decompressed transport-ingress budget +- **AND** its serialized `response.create` still exceeds the upstream websocket budget after historical slimming +- **THEN** the existing application-level guard returns HTTP 400 with `error.code = payload_too_large`, `error.type = invalid_request_error`, and `error.param = input` + +### Requirement: Thread-goal OpenAPI operations have unique stable identifiers +The generated OpenAPI document MUST assign a unique `operationId` to every documented HTTP operation. The GET and POST operations at `/backend-api/codex/thread/goal/get` MUST remain available through the same runtime behavior and MUST expose the deterministic identifiers `thread_goal_get_backend_api_codex_thread_goal_get_get` and `thread_goal_get_backend_api_codex_thread_goal_get_post`, respectively. Correcting this schema metadata MUST NOT change either method's authentication, dependency, request forwarding, upstream operation, response status, or response payload behavior. + +#### Scenario: Full OpenAPI schema has unique operation identifiers +- **WHEN** an unauthenticated client requests `GET /openapi.json` +- **THEN** every documented HTTP operation has an `operationId` +- **AND** no two documented HTTP operations share an `operationId` + +#### Scenario: Thread-goal methods publish deterministic identifiers +- **WHEN** an unauthenticated client inspects `/openapi.json` +- **THEN** `GET /backend-api/codex/thread/goal/get` has `operationId` `thread_goal_get_backend_api_codex_thread_goal_get_get` +- **AND** `POST /backend-api/codex/thread/goal/get` has `operationId` `thread_goal_get_backend_api_codex_thread_goal_get_post` + +#### Scenario: Thread-goal runtime forwarding remains compatible +- **WHEN** a client invokes either GET or POST `/backend-api/codex/thread/goal/get` with valid existing dependencies +- **THEN** the request is forwarded through the existing thread-goal handler using the original request method +- **AND** the upstream operation, response status, and response payload remain unchanged + +### Requirement: Public synthetic Responses failures carry numeric sequences + +Public streaming `POST /v1/responses` MUST emit every terminal +`response.failed` with a finite integer `sequence_number` so +strict OpenAI SDK Responses parsers recognize the terminal failure. If the +upstream or proxy-generated event omits a finite integer sequence, the public +normalizer MUST assign the next sequence after all finite integer sequences it +has observed in the same downstream stream. If it also synthesizes a leading +`response.created` from that failure, the created event MUST consume the next +sequence and the failure MUST use the following sequence so both events have +distinct values. Otherwise, if no finite integer sequence has been observed, +failure numbering MUST begin at zero. + +The public normalizer MUST preserve an existing finite integer +`sequence_number` and advance its next-sequence watermark accordingly. This +repair MUST NOT change Codex-private backend stream shapes. + +#### Scenario: Bridge failure after reasoning remains parseable + +- **GIVEN** public `/v1/responses` has emitted sequenced reasoning events +- **WHEN** the upstream bridge closes before a terminal response +- **THEN** the downstream terminal `response.failed` carries the next numeric + `sequence_number` +- **AND** a strict OpenAI SDK parser recognizes it as a terminal failure + +#### Scenario: Leading failure follows synthesized created sequence + +- **GIVEN** public `/v1/responses` has not emitted a finite integer sequence +- **WHEN** an unsequenced leading `response.failed` requires a synthesized + `response.created` +- **THEN** the created event carries `sequence_number = 0` +- **AND** the terminal failure carries `sequence_number = 1` + +#### Scenario: Failure after an unsequenced created event starts at zero + +- **GIVEN** public `/v1/responses` has emitted an unsequenced + `response.created` and no finite integer sequence +- **WHEN** the proxy synthesizes a terminal `response.failed` +- **THEN** the terminal event carries `sequence_number = 0` + +#### Scenario: Valid upstream failure sequence remains unchanged + +- **GIVEN** an upstream terminal `response.failed` carries a finite integer + `sequence_number` +- **WHEN** the public normalizer forwards the event +- **THEN** it preserves that sequence number unchanged +- **AND** if it must synthesize a leading `response.created`, that event uses + the immediately preceding integer sequence + +#### Scenario: Backend Codex stream shape remains unchanged + +- **GIVEN** a Codex-private backend Responses stream carries an unsequenced + terminal failure +- **WHEN** the stream is served without the public OpenAI SDK contract +- **THEN** the proxy does not add a public compatibility sequence + +### Requirement: Direct WebSocket capability intent is trusted and private + +A direct Responses WebSocket MUST recognize the exact internal marker +`X-Codex-LB-Required-Capability: trusted_cyber` only after successful existing +proxy API-key authentication. It MUST accept one marker from either the +handshake headers or the current `response.create.client_metadata`. Duplicate, +conflicting, non-string, unknown, malformed, or unauthenticated signals MUST +fail before account selection. Raw duplicate JSON keys or duplicate +`client_metadata` containers MUST NOT collapse into an ordinary request. The +marker MUST be rejected on every downstream frame type other than +`response.create`. + +The proxy MUST remove the capability header and the exact consumed metadata +key before upstream dispatch, request archival, diagnostics, and logging. +Unrelated client metadata MUST remain unchanged. + +#### Scenario: Per-frame intent routes before upstream open +- **WHEN** an authenticated frame carries the exact metadata marker on a + downstream socket opened without the header +- **THEN** the proxy establishes REQUIRED before opening or reusing an upstream + socket + +#### Scenario: Ambiguous or untrusted signal fails closed +- **WHEN** a signal is duplicated, malformed, unknown, or lacks an authenticated + proxy API-key principal +- **THEN** the proxy returns a typed error before account or model-source + dispatch + +#### Scenario: Duplicate JSON cannot erase intent +- **WHEN** raw JSON repeats the capability key or repeats `client_metadata` + around a capability marker +- **THEN** the proxy returns the typed unsupported-capability error before + selection + +#### Scenario: Capability metadata on another frame is rejected +- **WHEN** a downstream frame other than `response.create` contains the + capability metadata key +- **THEN** the proxy returns a typed error without forwarding or archiving that + frame upstream +- **AND** malformed JSON text is rejected rather than passed through an already + open upstream socket +- **AND** binary downstream frames are rejected before parsing, archiving, or + upstream forwarding + +#### Scenario: Internal metadata is not forwarded or archived +- **WHEN** a valid capability-bearing frame is dispatched and archived +- **THEN** neither capability carrier appears in upstream headers, upstream + payload, archive payload, diagnostics, or logs + +### Requirement: A late capability cannot reuse an ordinary upstream socket + +A later REQUIRED frame MUST NOT reuse an upstream socket selected for an +ordinary request on the same downstream WebSocket. An idle ordinary socket +MUST be retired before capable +reselection. If another frame is still pending, the proxy MUST fail closed +rather than change the account requirement beneath in-flight work. The socket's +selection contract, not whether its account happened to have the capability +grant, MUST determine whether it was selected as ordinary. Before reusing a +REQUIRED-selected socket, the proxy MUST revalidate the pinned account and its +current capability grant through the canonical selector. + +#### Scenario: Idle ordinary socket is replaced +- **WHEN** an idle downstream session previously selected an ordinary account + and a later frame establishes REQUIRED +- **THEN** the ordinary upstream is retired before the frame is sent +- **AND** the replacement selection requires a security-work-authorized account + +#### Scenario: Pending ordinary work blocks a requirement change +- **WHEN** ordinary work is still pending and a later frame establishes REQUIRED +- **THEN** the later frame fails before upstream send +- **AND** the pending frame's account and request state are not rewritten + +#### Scenario: Revoked capability grant prevents socket reuse +- **WHEN** a socket was selected for REQUIRED but its pinned account's grant is + no longer valid at canonical revalidation +- **THEN** the stale socket does not receive the next REQUIRED frame +- **AND** an idle socket is retired before constrained reselection + +#### Scenario: Revalidation uncertainty fails closed +- **WHEN** canonical account revalidation cannot complete for a REQUIRED socket +- **THEN** the frame receives a typed capability-routing-unavailable error +- **AND** its reservation is settled without forwarding the frame upstream + +### Requirement: Proof-gated recovery attempts are durably fenced + +When an HTTP bridge request has a verified, account-neutral, unanchored full +resend body, the proxy MUST record that request fingerprint in the durable +recovery journal before dispatching it upstream. The record MUST be owned by +the current durable session owner epoch and MUST start in `unknown` state. +Requests without that replay-safety proof MUST NOT create a recovery-journal +record. + +#### Scenario: Safe resend is journaled before dispatch + +- **GIVEN** a request has a verified full-resend body that is safe to replay + without `previous_response_id` +- **WHEN** the proxy admits the request for upstream dispatch +- **THEN** the durable journal contains one `unknown` record for its session + and request fingerprint before `response.create` is sent + +#### Scenario: Suppressed request is not journaled + +- **GIVEN** a hard session retry circuit is cooling down +- **WHEN** the request is rejected before upstream dispatch +- **THEN** no recovery-journal record is created or refreshed + +### Requirement: Durable replay is limited to ambiguous transport outcomes + +The proxy MUST consume an `unknown` recovery-journal record for a fresh +account-neutral replay only after an ambiguous transport outcome, represented +by `stream_incomplete`, `stream_idle_timeout`, or +`upstream_request_timeout`, and only before any response event or downstream +output. Explicit deterministic `response.failed` errors MUST settle normally +and MUST NOT trigger a cross-account replay or consume the recovery fence. + +#### Scenario: Transport ambiguity permits one replay + +- **GIVEN** an `unknown` proof-gated journal record exists +- **AND** the upstream closes or times out before any response event +- **WHEN** the bridge handles the ambiguous transport failure +- **THEN** the record is atomically claimed and the request is replayed once + on a fresh account-neutral upstream session + +#### Scenario: Deterministic failure is not replayed + +- **GIVEN** an `unknown` proof-gated journal record exists +- **AND** upstream emits an explicit pre-output `response.failed` such as an + invalid request or quota rejection +- **WHEN** the bridge handles that terminal event +- **THEN** it forwards the terminal failure +- **AND** it leaves the journal available for settlement without replaying on + another account + +### Requirement: Recovery journal settlement is owner-fenced and idempotent + +After a replayed request reaches `response.completed`, the proxy MUST mark its +journal record `replayed` only through the current durable owner epoch and +MUST retain the downstream response id when available. Repeated settlement, +stale owners, and concurrent claim attempts MUST NOT produce a second replay. +The migration MUST be on the current Alembic head and startup schema checks +MUST require the journal table. + +#### Scenario: Completed replay settles once + +- **GIVEN** a replayed request completes successfully +- **WHEN** the completion event is processed +- **THEN** the matching journal record becomes `replayed` +- **AND** a later retry cannot claim it again + +#### Scenario: Stale owner cannot settle or replay + +- **GIVEN** a journal record belongs to a newer durable owner epoch +- **WHEN** an old replica attempts settlement or replay +- **THEN** the operation is rejected without changing the record state + +### Requirement: Claimed HTTP bridge completed queues remain deliverable + +When HTTP bridge processing of `response.completed` removes a request from +pending ownership, it MUST retain the request's downstream event queue for the +remainder of that completed operation. Later asynchronous bookkeeping or +request detachment MUST NOT revoke that claimed queue before the completed +operation's selected terminal event and end-of-stream marker are enqueued. If +fail-closed bookkeeping replaces the upstream completion with a terminal +failure, that selected failure event is the terminal event governed by this +requirement. + +While the claimed completed-delivery operation remains active, ordinary stream +idle accounting MUST NOT replace the upstream completion with a synthetic idle +failure, and the stream MUST continue emitting its existing liveness frames. +The completed-queue claim and the terminal idle-timeout decision MUST be +serialized under the bridge pending lock. If completed processing wins that +serialization and claims a live queue, the timeout MUST be suppressed. If the +terminal event and end-of-stream marker are already queued when a concurrent +timeout finishes awaited recovery work, the completed claim MUST remain +authoritative until the stream consumes that queued delivery. If the +terminal idle timeout wins while no completed delivery is active, it MUST +revoke the request's mutable event queue before releasing the pending lock so a +later completed event cannot claim an orphaned queue. + +The first idle-timeout suppression for one completed-delivery operation MUST +emit one bounded diagnostic containing the request ID, downstream response ID, +and elapsed seconds. Further liveness intervals for that same operation MUST +NOT repeat the diagnostic. + +When that operation returns, raises, or is cancelled before delivery, idle +timeout behavior MUST resume. + +If detachment removes the request from pending ownership first, existing +client-disconnect and drain behavior MUST remain unchanged. + +#### Scenario: Completed processing claims the request before detachment + +- **GIVEN** an HTTP bridge stream is waiting on its request event queue +- **AND** an upstream `response.completed` event removes that request from pending ownership +- **WHEN** request detachment overlaps later completed-event bookkeeping +- **THEN** the stream receives the terminal event selected for downstream delivery exactly once +- **AND** the stream receives its end-of-stream marker + +#### Scenario: Completed bookkeeping exceeds the idle window + +- **GIVEN** completed-event processing has claimed a live request queue +- **WHEN** later completed bookkeeping exceeds the configured stream idle window +- **THEN** the stream continues emitting liveness frames +- **AND** it does not emit a synthetic idle failure while that operation remains active +- **AND** it logs the suppression once with request, response, and elapsed-time context + +#### Scenario: Terminal idle timeout wins before completed processing + +- **GIVEN** an HTTP bridge stream has exhausted its configured idle window +- **AND** no completed-delivery operation has claimed its queue +- **WHEN** the stream acquires the bridge pending lock before a concurrent completed event +- **THEN** it revokes the mutable event queue while still holding that lock +- **AND** it emits the existing synthetic idle failure +- **AND** later completed processing does not deliver to the revoked queue + +#### Scenario: Completed delivery finishes during timeout recovery + +- **GIVEN** an HTTP bridge timeout path is awaiting pre-response recovery work +- **AND** completed processing claims the live queue and enqueues its terminal event and end-of-stream marker +- **WHEN** completed processing returns before the timeout path rechecks ownership +- **THEN** the completed claim remains authoritative +- **AND** the stream consumes the queued completion without emitting a synthetic idle failure + +#### Scenario: Completed bookkeeping aborts + +- **GIVEN** completed-event processing has claimed a live request queue +- **WHEN** that completed-delivery operation exits without enqueueing its terminal event +- **THEN** idle timeout suppression ends +- **AND** the existing idle-timeout failure behavior resumes + +#### Scenario: Detachment claims the request first + +- **GIVEN** an HTTP bridge request is still pending +- **WHEN** detachment removes downstream queue ownership before completed-event matching +- **THEN** existing client-disconnect and upstream-drain behavior is preserved +- **AND** no completed event is delivered to another request + +### Requirement: Replayed tool-call namespace metadata is local-only on upstream input + +For standard and compact Responses requests, the proxy MUST omit `namespace` from every replayed `input` item whose `type` is `function_call`, `custom_tool_call`, or `apply_patch_call` before forwarding the request upstream. The proxy MUST preserve all other fields on that item, MUST retain the original namespace metadata for local call-identity and replay-deduplication processing, and MUST NOT alter client-provided top-level tool entries as part of this normalization. + +#### Scenario: Standard Responses replay omits tool-call namespaces upstream + +- **WHEN** a standard Responses request replays `function_call` and `custom_tool_call` input items with `namespace` +- **THEN** the upstream payload omits only those items' `namespace` +- **AND** preserves their remaining call fields +- **AND** the local request input retains the namespace metadata + +#### Scenario: Compact Responses replay omits tool-call namespace upstream + +- **WHEN** `/v1/responses/compact` replays a recognized tool-call input item with a namespace +- **THEN** its upstream payload omits the input item's `namespace` +- **AND** preserves the remaining tool-call fields + +#### Scenario: WebSocket response.create omits tool-call namespaces upstream + +- **WHEN** a Responses WebSocket request replays namespaced `function_call` and `custom_tool_call` input items +- **THEN** the upstream `response.create` frame omits only those items' `namespace` +- **AND** preserves their remaining call fields + +#### Scenario: Configured Responses model source omits tool-call namespaces upstream + +- **WHEN** `/v1/responses` routes a replayed namespaced tool call to a configured OpenAI-compatible Responses model source +- **THEN** the source payload omits only the call item's `namespace` +- **AND** preserves source-compatible request fields that the Codex upstream path does not support + +#### Scenario: Account-neutral replay classification retains namespace identity + +- **WHEN** an HTTP bridge evaluates a namespaced tool-call history for cross-account replay safety +- **THEN** the classifier input retains the namespace metadata +- **AND** the request fails closed rather than becoming account-neutral because of wire normalization + +#### Scenario: Malformed replay item type does not fail serialization + +- **WHEN** a permissively parsed input item has a non-string `type` and a `namespace` +- **THEN** outbound serialization does not raise an internal type error +- **AND** does not treat the item as a recognized replayed tool call + +#### Scenario: Top-level namespace tool remains byte-preserved + +- **WHEN** the client includes a top-level tool entry whose `type` is `namespace` +- **THEN** standard Responses serialization forwards that tool entry byte-identically + +### Requirement: Responses-Lite replay proof tolerates only verified developer interleaving + +When a fresh durable HTTP bridge classifies a client-unanchored Responses-Lite +full resend whose `additional_tools` bundle preserves developer messages inline, +the replay proof MUST tolerate a developer message only in the historical and +fresh positions defined below. Every other developer position or shape MUST +remain fail-closed. + +A tolerated fresh developer message MUST have `type` omitted or equal to `message`, +MUST have role `developer`, MUST have no non-empty response-owned ID or phase, +MUST have no status or a `completed` status, MUST contain exact account-neutral +metadata with one nonblank `turn_id`, MUST contain exactly one self-contained +`input_text` content part, and MUST contain no unknown or account-scoped fields. +Explicit null or malformed item types MUST fail closed. + +Classification MUST retain response-owned developer-message ID evidence until +these checks have completed, even when other response-owned IDs are projected +out. It MUST retain developer-role items before applying projection rules that +normally omit their declared item type, so a malformed developer item cannot +disappear before validation. A canonical Lite-prefix developer instruction MAY +appear immediately after the `additional_tools` bundle when it passes the same +account-neutral item checks as historical interleaving and has no response-owned +ID. A developer message in the stored prefix outside that canonical position or +the verified pending-call/matching-output interleave MUST fail closed. Non-Lite +`input` or `messages` forms whose instruction-role messages are normalized into +top-level `instructions` remain outside this requirement. + +#### Scenario: Canonical Responses-Lite prefix remains transparent + +- **GIVEN** a fingerprint-verified stored prefix begins with an `additional_tools` bundle +- **AND** a valid account-neutral developer instruction appears immediately after that bundle +- **WHEN** exact manifest or retained-output replay proof validates the stored prefix +- **THEN** the canonical developer instruction is transparent +- **AND** the original full input remains eligible for account-neutral replay + +#### Scenario: Verified historical Responses-Lite developer message is transparent + +- **GIVEN** a Responses-Lite input contains an `additional_tools` bundle +- **AND** its fingerprint-verified stored prefix contains a supported direct call +- **AND** a valid developer message appears before that call's matching output +- **AND** the fresh suffix exactly settles the durable pending-tool manifest +- **WHEN** the HTTP bridge opens a replacement session on the durable owner +- **THEN** it sends the original full input without injecting `previous_response_id` +- **AND** it sends the request once + +#### Scenario: Other historical messages remain fail-closed + +- **GIVEN** a supported direct call is pending in the verified stored prefix +- **WHEN** a user, assistant, system, malformed developer, or response-owned message appears before its output +- **THEN** exact manifest proof fails + +#### Scenario: Other stored developer positions remain fail-closed + +- **GIVEN** a fingerprint-verified stored prefix has no pending direct call +- **WHEN** a developer message appears outside the canonical adjacent Lite-prefix position +- **OR** the adjacent message has a response-owned ID +- **THEN** exact manifest and retained-output proofs fail + +#### Scenario: Projection-omitted developer type remains visible to validation + +- **GIVEN** a developer-role item declares a type normally omitted by replay projection +- **WHEN** account-neutral replay classification projects the full resend +- **THEN** the malformed developer item remains visible to replay proof +- **AND** replay classification fails closed + +#### Scenario: Historical output remains mandatory + +- **GIVEN** a valid developer message follows a supported historical call +- **WHEN** the matching output is missing or has another call ID or type +- **THEN** exact manifest proof fails + +#### Scenario: Historical developer interleaving is bounded to one call and one message + +- **GIVEN** a fingerprint-verified stored prefix opens a pending direct-call window +- **WHEN** that window holds more than one outstanding call at any point before the developer message +- **OR** a further call opens in that window after it has consumed a developer message +- **OR** a second developer message appears while the same window is still open +- **THEN** exact manifest proof fails +- **AND** a later window that holds exactly one outstanding call may still interleave one developer message + +#### Scenario: Fresh developer suffix bounds are measured on the projected input + +- **GIVEN** account-neutral replay classification projects the full resend +- **WHEN** the projection omits reasoning or completed bookkeeping items from the fresh suffix +- **THEN** the fresh developer suffix and terminality bounds are evaluated on the projected positions +- **AND** the accepted width is limited to shapes whose projected suffix satisfies those bounds + +#### Scenario: Bounded fresh custom-tool developer interleave is transparent + +- **GIVEN** the fingerprint-verified stored prefix is followed by a fresh suffix +- **AND** the durable pending-tool manifest contains exactly one `custom_tool_call` +- **WHEN** the entire suffix is exactly that custom call, one valid developer message, and its matching custom-tool output +- **THEN** exact manifest proof passes +- **AND** the original full input is sent once without injecting `previous_response_id` + +#### Scenario: Other fresh tool-loop developer positions remain fail-closed + +- **GIVEN** a durable pending-tool manifest +- **WHEN** a fresh developer message is used with a function or apply-patch call, appears in a parallel batch, is duplicated, lacks exact metadata, contains malformed or account-scoped content, or has leading or trailing suffix items +- **THEN** exact manifest proof fails + +#### Scenario: Bounded retained-output developer follow-up is transparent + +- **GIVEN** the fingerprint-verified stored prefix is followed by a completed assistant `final_answer` +- **AND** exactly one explicit user message follows that retained output +- **WHEN** one valid developer message is the terminal suffix item +- **THEN** retained-output proof passes +- **AND** the original full input is sent once without injecting `previous_response_id` + +#### Scenario: Unproven retained-output developer follow-up remains fail-closed + +- **GIVEN** a retained-output full resend +- **WHEN** the latest assistant output is not `final_answer`, the developer message is not terminal, the fresh input is raw or contains multiple user items, the developer metadata or content is not account-neutral, or the stored prefix contains historical developer interleaving +- **THEN** retained-output proof fails + +### Requirement: Aborted terminal bookkeeping settles claimed reservations exactly once + +The HTTP bridge MUST settle a request's API-key reservation exactly once even +when terminal-event bookkeeping aborts after removing the request from pending +ownership; that bookkeeping continuation exclusively owns the settlement. If +the continuation raises or is cancelled before finalization transfers that +settlement, the abort path MUST settle every request it still owns: the +reservation heartbeat MUST be cancelled, the +reservation MUST be released, and the downstream waiter SHOULD be unblocked +with an end-of-stream marker instead of waiting for its idle timeout. The +abort settlement MUST run to completion under cancellation (shielded), MUST +apply to the grouped previous-response error path's not-yet-finalized +remainder, and MUST NOT settle requests that a retry branch restored to +pending ownership. Settlement MUST remain idempotent so an abort overlapping +an already-transferred finalization cannot double-account usage. + +If the abort settlement itself fails, the claim MUST be marked abandoned and +request detachment MUST be allowed to reclaim that settlement even though the +request is no longer in pending ownership. Detachment MUST NOT settle a live +claim whose bookkeeping continuation is still running. + +#### Scenario: Completed bookkeeping raises after the pending pop + +- **GIVEN** an upstream `response.completed` event has removed a request with an API-key reservation from pending ownership +- **WHEN** later completed bookkeeping raises before finalization +- **THEN** the reservation heartbeat task finishes +- **AND** the API-key reservation is released exactly once +- **AND** no reservation heartbeat touch runs afterward + +#### Scenario: Completed bookkeeping is cancelled after the pending pop + +- **GIVEN** an upstream `response.completed` event has removed a request with an API-key reservation from pending ownership +- **WHEN** the bookkeeping continuation is cancelled before finalization +- **THEN** the shielded abort settlement still cancels the heartbeat and releases the reservation +- **AND** the cancellation is re-raised after settlement + +#### Scenario: Grouped previous-response finalization aborts mid-loop + +- **GIVEN** a grouped previous-response error has removed multiple requests from pending ownership +- **WHEN** finalization aborts after settling only a prefix of those requests +- **THEN** every not-yet-finalized request in the group has its heartbeat cancelled and its reservation released + +#### Scenario: Detachment reclaims an abandoned claim + +- **GIVEN** terminal bookkeeping claimed a request out of pending ownership, aborted, and its abort settlement failed +- **WHEN** the downstream stream detaches that request +- **THEN** detachment cancels the heartbeat and releases the reservation even though the request is not in pending ownership + +#### Scenario: Detachment leaves a live claim to its owner + +- **GIVEN** terminal bookkeeping has claimed a request out of pending ownership and is still running +- **WHEN** the downstream stream detaches that request +- **THEN** detachment does not release the reservation out from under the in-flight finalization + +### Requirement: Pool usage exhaustion is reported as a usage-limit error + +The proxy MUST report pool-wide Responses usage exhaustion as a usage-limit +error. When every account eligible for a Responses request is exhausted by known +usage windows, the proxy MUST reject the request with HTTP `429` and an +OpenAI-style error envelope whose `error.code` and `error.type` are both +`usage_limit_reached`. If account selection has an authoritative upstream reset +timestamp for the exhausted pool, the response envelope MUST include that +timestamp as `error.resets_at`; the proxy MUST NOT expose the capped +human-facing retry hint or a synthesized fallback as `error.resets_at`. The +proxy MUST NOT collapse this condition into generic `no_accounts`, +`server_error`, or HTTP `503` semantics. Exhaustion classification MUST be +based on structured account state after the same eligibility filtering as +ordinary selection, and MUST NOT reclassify local capacity or overload codes +(account caps, admission gates, fair-share throttles) as usage exhaustion. + +#### Scenario: Public Responses request exhausts the eligible usage pool + +- **WHEN** account selection for a public `/v1/responses` or + `/backend-api/codex/responses` request finds only usage-exhausted eligible + accounts +- **THEN** the response status is HTTP `429` +- **AND** the response body has `error.code = "usage_limit_reached"` +- **AND** the response body has `error.type = "usage_limit_reached"` +- **AND** any selected pool reset timestamp is surfaced as `error.resets_at` + +#### Scenario: Streaming selection failure preserves usage-limit semantics + +- **WHEN** a streaming Responses request cannot select an account because every + eligible account is usage-exhausted before downstream-visible output +- **THEN** the terminal error event uses `usage_limit_reached` +- **AND** clients do not receive a generic no-account/server-unavailable error + +#### Scenario: Usage-limit selection failures are terminal, not waitable + +- **WHEN** account selection fails with `usage_limit_reached` on a streaming, + HTTP-bridge, or WebSocket Responses path +- **THEN** the proxy reports the structured usage-limit failure immediately +- **AND** it does not enter an account-capacity recovery wait for the + remaining request budget before reporting it + +#### Scenario: Local capacity codes keep their rate-limit contract + +- **WHEN** account selection fails with a local capacity or overload code such + as `account_stream_cap` or `account_response_create_cap` +- **THEN** the response keeps HTTP `429` with `error.type = "rate_limit_error"` + and the stable local error code +- **AND** the response is not reported as `usage_limit_reached` + +#### Scenario: Unusable non-exhausted pools keep existing semantics + +- **WHEN** every account is paused, deactivated, or requires re-authentication + and no eligible account is exhausted by a known usage window +- **THEN** the pre-existing `no_accounts` failure semantics are preserved + +#### Scenario: Owner-scoped exhaustion preserves continuity semantics + +- **WHEN** a request is pinned to a previous-response or file owner account and + only that owner is usage-exhausted while the wider eligible pool is usable +- **THEN** the proxy keeps the existing continuity-owner failure semantics +- **AND** it does not report pool-wide `usage_limit_reached` + +### Requirement: Silent HTTP bridge sessions are quarantined from re-attach and reuse + +When an HTTP bridge session proves silent/wedged, the proxy MUST quarantine its session key for a bounded window so later requests stop attaching to it. A session proves silent/wedged when either (a) a pending request being failed or retired carried a proxy-injected `previous_response_id`, had sent `response.create`, observed upstream response events, and never had `response.created` assigned, or (b) the session key hits two consecutive eventless `missing_response_created_timeout` retires. This holds for every path that fails or retires the request — partial stale-holder cleanup, the reader-failure funnel, and direct all-stale session retirement alike. The quarantine MUST be evaluated only when a request is already being failed or its session retired — never against a live owned turn — so a stream whose `response.created` was observed (including deferred-reasoning streams with long event gaps) MUST NOT be quarantined, and mere event silence during an owned live turn MUST NOT trigger quarantine by itself. + +While a session key is quarantined: an existing session under that key MUST NOT be selected for reuse (a new request detaches it and proceeds on a fresh session), and for durable-anchor selection a quarantined session that is still open MUST count as absent, exactly as if it were already gone. The quarantine registry verdict is authoritative for the key: any session under the key while the quarantine window is active — including a freshly created replacement whose own completion has not yet cleared the quarantine — is equally excluded from reuse and equally absent for anchor selection. A fresh reattach whose incoming payload already looks like a full conversation resend MUST NOT receive a proxy-injected durable anchor through any injection point — the fresh-reattach injection, session-state hydration of the durable anchor, or the session-level injection — so the dispatch goes upstream genuinely unanchored with the client's own untrimmed payload. A payload that does not look like a full resend (a genuine delta-only continuation) MUST still receive the durable anchor, because it has no other way to convey prior conversation state. + +Quarantine state MUST be bounded and self-recovering: it is in-memory and session-scoped, expires by TTL (a live session that outlives its quarantine window MUST become reusable again), is cleared when a response completes on the same session key, and MUST NOT write account health or alter account selection. + +#### Scenario: Reattach streams events but response.created is never assigned (#1534) + +- **GIVEN** a durable HTTP bridge session with a stored anchor whose fresh reattach injected a proxy-owned `previous_response_id` +- **AND** the reattached upstream stream delivers response events but `response.created` is never assigned +- **WHEN** the stream fails or the session is retired with that request still pending +- **THEN** the request fails terminally as before +- **AND** the session key is quarantined with reason `reattach_missing_response_created` + +#### Scenario: All-stale direct retirement still quarantines the key + +- **GIVEN** a wedged reattach (proxy-injected `previous_response_id`, `response.create` sent, response events observed, `response.created` never assigned) that is the ONLY stale pending request on its session +- **WHEN** the stuck-gate watchdog retires the session directly instead of failing the stale holder individually +- **THEN** the session key is quarantined with reason `reattach_missing_response_created` +- **AND** the next request takes the fresh no-anchor path instead of rebuilding the identical anchored reattach + +#### Scenario: Next request after the wedge completes on the fresh path + +- **GIVEN** a session key quarantined after a reattach that streamed events without `response.created` +- **WHEN** a later request arrives for the same key with a full-conversation-resend payload and no client `previous_response_id` +- **THEN** the proxy does not inject the durable anchor for that request +- **AND** the request is sent upstream unanchored with the client's own full payload +- **AND** the request can complete normally instead of rebuilding the identical wedged reattach + +#### Scenario: Suppressed anchor does not come back through session state + +- **GIVEN** a quarantined session key and a full-conversation-resend payload whose stored durable prefix is trimmable but whose fresh suffix does not retain the prior output +- **WHEN** the fresh-reattach durable-anchor injection is skipped because of the quarantine +- **THEN** the durable anchor is not rehydrated into the fresh session's completed-response state +- **AND** the session-level injection does not re-add the same anchor or trim the stored prefix +- **AND** the dispatch goes upstream genuinely unanchored with the client's untrimmed payload +- **AND** the suppression applies even when the fresh-reattach injection was already ineligible for other reasons (for example a conversation-scoped payload, a live alias session, or an active-owner forward that falls back to a local rebind) + +#### Scenario: Quarantined session is excluded from reuse selection + +- **GIVEN** a session marked quarantined that is still live or retained for admission handoff +- **WHEN** a new request looks up that session key +- **THEN** the session is not considered reusable +- **AND** the request proceeds on a fresh session instead +- **AND** a replacement session created under the same still-quarantined key is likewise not reusable until a completion or the TTL clears the quarantine + +#### Scenario: Repeated eventless timeouts quarantine the key + +- **GIVEN** a session key whose pending request already retired once with the eventless `missing_response_created_timeout` +- **WHEN** a subsequent attach on the same key retires with the same eventless timeout before any response completes on the key +- **THEN** the session key is quarantined with reason `repeated_eventless_timeout` +- **AND** the first timeout alone does not quarantine the key + +#### Scenario: Deferred-reasoning live turn is never quarantined + +- **GIVEN** an owned live turn whose `response.created` was observed and whose events flow with long gaps (deferred reasoning) +- **WHEN** its stream later fails or its session is retired +- **THEN** the session key is not quarantined +- **AND** later requests keep the existing reuse and anchor-injection behavior + +#### Scenario: Delta-only payloads keep their anchor while quarantined + +- **GIVEN** a quarantined session key — including one whose quarantined session is still open with other active requests +- **WHEN** a later request arrives whose payload does not look like a full conversation resend +- **THEN** the still-open quarantined session counts as absent for durable-anchor selection +- **AND** the durable anchor is still injected for that request, preserving the client's only way to convey prior context + +#### Scenario: Quarantine is bounded and self-clearing + +- **GIVEN** a quarantined session key +- **WHEN** a response completes on that session key, or the quarantine TTL elapses +- **THEN** the quarantine (and its eventless strike counter) is cleared +- **AND** a session that survived the quarantine window is reusable again instead of staying rejected forever +- **AND** no durable row, janitor work, or account-health write was involved at any point + +### Requirement: Scoped operation identity + +The system MUST include the normalized API-key scope in every durable HTTP +bridge operation fingerprint and MUST apply that scope to fingerprint and +completed-operation lookups. + +#### Scenario: Equal requests from different keys remain isolated + +- **WHEN** two API keys submit the same logical request +- **THEN** each key receives an independent durable operation identity + +### Requirement: Recoverable startup takeover + +Startup cleanup MUST retain sessions that own submitted, acknowledged, or +unknown operations and MUST detach ownership before a replacement instance +takes over. + +#### Scenario: Restart preserves an in-flight operation + +- **WHEN** an instance restarts while an operation is nonterminal +- **THEN** cleanup detaches the old owner without deleting the operation spool + +### Requirement: Fresh retry transcript + +When an explicit failed operation is rebound, the system MUST atomically remove +the prior operation events and reset event-byte/spool state before accepting new +events. + +#### Scenario: Failed retry cannot replay stale failure output + +- **WHEN** a failed operation is retried and later completes +- **THEN** replay contains only the new attempt's events + +### Requirement: Proof-gated sibling anchoring + +The system MUST advance a continuation to a completed sibling response only +when the sibling has the same parent and logical request fingerprint in the +same API-key scope. + +#### Scenario: Distinct sibling input keeps its requested parent + +- **WHEN** a request reuses a parent with a different fingerprint +- **THEN** the service does not silently anchor it to another child response + +### Requirement: Single migration head + +The Alembic graph MUST converge the durable operation revisions with the current +release head and MUST expose one canonical head after upgrade. + +#### Scenario: Upgrade resolves one head + +- **WHEN** migrations are upgraded to the release tip +- **THEN** Alembic reports one canonical head + +### Requirement: Conservative spool defaults + +New operation rows MUST start with an incomplete event spool on SQLite and +PostgreSQL. A transcript MUST become replayable only after terminal event drain +and explicit finalization. + +#### Scenario: Nonterminal spool is not replayable + +- **WHEN** an operation has events but no finalized terminal event +- **THEN** recovery does not replay its transcript as complete + +### Requirement: Retain completed recovery transcripts + +Startup ownership cleanup MUST retain sessions with operation transcripts that +remain inside the configured operation retention window, including completed +operations, and MUST let normal spool retention remove the operation rows. + +#### Scenario: Recent completed transcript survives takeover + +- **WHEN** startup cleanup sees a recent completed transcript +- **THEN** it retains the session until normal retention expires it + +### Requirement: Continuous transcript retention + +Operation transcript cleanup MUST run periodically in a leader-gated scheduler +and MUST drain all eligible batches during each pass. Disabling the existing +sticky-session mapping cleanup switch MUST NOT disable operation transcript +retention; that switch MAY skip sticky mapping maintenance while durable +operation retention continues. + +#### Scenario: Retention drains all eligible batches + +- **WHEN** more rows are eligible than one deletion batch +- **THEN** one scheduler pass removes every eligible batch + +#### Scenario: Sticky cleanup toggle does not disable transcript retention + +- **WHEN** sticky-session cleanup is disabled and the durable bridge schema is + available +- **THEN** the leader-gated scheduler still drains expired operation transcript + rows while skipping sticky mapping cleanup + +### Requirement: Fresh indefinite-recovery spool + +Before dispatching a server-owned retry for a nonterminal operation, the system +MUST atomically clear any partial event spool under the durable owner fence. + +#### Scenario: Retry starts with a clean transcript + +- **WHEN** an anchored retry is dispatched after partial persistence +- **THEN** old events and byte counts are cleared before new output is accepted + +### Requirement: Ordered deferred reasoning persistence + +Deferred reasoning events released before a visible event MUST be persisted in +the same order in which they are delivered downstream, before the visible +event is persisted. + +#### Scenario: Deferred events preserve downstream order + +- **WHEN** buffered reasoning is released before visible output +- **THEN** the durable spool stores the reasoning blocks before that output + +### Requirement: Per-operation disconnect classification + +When a shared bridge websocket closes, each pending operation MUST be +classified from that operation's own observed response-event count. Activity +from a sibling request MUST NOT make an eventless operation safely retryable. + +#### Scenario: Sibling output does not acknowledge an eventless request + +- **WHEN** one pending request emitted output and another emitted none +- **THEN** the two operations receive different disconnect classifications + +### Requirement: Abandoned operation retention + +Operation retention MUST expire stale submitted and acknowledged rows in +addition to terminal and ambiguous rows, so a crashed or abandoned operation +cannot retain raw request data indefinitely. + +#### Scenario: Stale abandoned request is purged + +- **WHEN** a submitted operation exceeds retention age +- **THEN** its request data and event spool are removed + +### Requirement: Acknowledged alias persistence failure + +If upstream has acknowledged a response but local continuity-alias persistence +fails, the downstream error MUST NOT transition the durable operation to a +retryable failed state. The operation MUST remain acknowledged/ambiguous so an +identical retry cannot dispatch a duplicate upstream turn. + +#### Scenario: Alias write failure remains fail-closed + +- **WHEN** an acknowledged response cannot publish its continuity alias +- **THEN** the operation remains non-retryable and the client receives a terminal error + +### Requirement: Cross-session nonterminal handoff + +When a scoped operation fingerprint is found under a different durable +session, a nonterminal operation MUST be atomically rebound to the currently +owned session before its event spool is reset or a recovery attempt is sent. +Completed replayable operations MUST remain attached to their original session. +The handoff MUST be refused while the prior session has an unexpired owner +lease, preventing concurrent owners from dispatching the same turn. + +#### Scenario: Active prior owner fences handoff + +- **WHEN** a duplicate request finds a nonterminal operation under another session +- **AND** that session still has an unexpired owner lease +- **THEN** the operation remains with the prior session and no concurrent retry is dispatched + +#### Scenario: Expired prior owner permits handoff + +- **WHEN** the prior session lease is absent or expired +- **THEN** the operation can be atomically rebound before recovery + +### Requirement: Fenced one-shot recovery dispatch + +The durable recovery journal MUST persist a one-shot replay budget for every +recovery-safe request. The budget MUST be consumed atomically when a replay is +claimed for dispatch, and a caller that proves the replay never reached the +upstream send boundary MUST restore that claim under the same session owner +fence. A replacement session MUST retain or transfer a fenced origin owner +until the claim is rolled back or settled; selecting a replacement or failing +preflight MUST NOT permanently consume an unsent replay. + +#### Scenario: Concurrent reconnects consume one replay + +- **WHEN** concurrent reconnects observe the same ambiguous operation +- **THEN** exactly one owner atomically claims the persisted replay budget and + other reconnects fail closed without dispatching a duplicate + +#### Scenario: Pre-dispatch replacement failure restores the budget + +- **WHEN** a replay claim is made but replacement admission or preflight fails + before the exact upstream frame is sent +- **THEN** the claim returns to the available state and the fenced origin + owner is released only after that rollback succeeds + +#### Scenario: Successful replacement settles the origin journal + +- **WHEN** a replacement session dispatches the claimed replay and receives a + terminal response event +- **THEN** settlement uses the retained origin owner fence before releasing it + and the replay budget cannot be claimed again + +### Requirement: Lease-aware operation retention + +Retention MUST NOT delete stale submitted or acknowledged operations while +their session is actively owned with an unexpired lease. The owner/lease +predicate MUST be rechecked in the deletion transaction. + +#### Scenario: Active lease protects stale operation + +- **WHEN** a stale operation belongs to a session with a live lease +- **THEN** retention leaves it intact + +### Requirement: Anchored indefinite recovery gate + +The server-indefinite recovery loop MUST be installed only for an eventless +anchored continuation with a durable parent operation. Fresh first-turn +requests and streams that already emitted downstream response events MUST +terminate normally rather than being resent indefinitely. + +#### Scenario: Fresh request is not held indefinitely + +- **WHEN** a first-turn request loses its upstream connection +- **THEN** the proxy returns its normal error path without an indefinite loop + +### Requirement: Retry reservation terminalization + +If reacquiring API-key usage limits for a recovery attempt fails, the proxy +MUST settle the prior reservation and emit a terminal `response.failed` SSE +event instead of aborting the already-started stream. + +#### Scenario: Quota failure produces terminal SSE + +- **WHEN** a recovery retry cannot reacquire its usage reservation +- **THEN** the client receives `response.failed` and the prior reservation is settled + +#### Scenario: Unexpected admission failure produces terminal SSE + +- **WHEN** recovery admission raises an unexpected infrastructure error before + a replacement stream starts +- **THEN** the client receives `response.failed` and the prior reservation is + settled instead of receiving a truncated stream + +### Requirement: Failure spool/state ordering + +For an explicit deterministic failure, the proxy MUST persist the terminal SSE +block before exposing the durable operation as failed. The event append and +failed-state transition MUST use the same owner fence and transaction when the +durable repository supports it. + +#### Scenario: Concurrent retry cannot reset an unspooled failure + +- **WHEN** a response failure is being settled while an identical reconnect is + admitted +- **THEN** the reconnect observes the terminal operation fence and cannot reset + or mix the previous failure into a new transcript + +### Requirement: Partial disconnect acknowledgement + +When a bridge disconnects after an operation has emitted any response event but +before a terminal event, the durable operation MUST remain acknowledged or +ambiguous. It MUST NOT be classified as retryable failed solely because the +disconnect was non-terminal. + +#### Scenario: Partial output is never resent as a fresh turn + +- **WHEN** the upstream closes after `response.created` but before completion +- **THEN** the operation remains non-retryable + +### Requirement: Retry output stops indefinite recovery + +An indefinite recovery attempt MUST stop retrying once that attempt emits any +downstream response event, even if the attempt later fails with a retryable +transport error. + +#### Scenario: Retry output prevents a second attempt + +- **WHEN** a retry emits a data event and then times out +- **THEN** the server stops the indefinite loop instead of appending another response + +### Requirement: Preserve repeated event occurrences + +The durable event spool MUST preserve repeated identical SSE blocks as distinct +ordered occurrences. Event identity MUST include its operation-local sequence +position rather than content alone. + +#### Scenario: Identical deltas replay twice + +- **WHEN** two consecutive SSE blocks have identical text +- **THEN** both occurrences are present in the replay transcript + +### Requirement: Stop event persistence during shutdown + +Proxy shutdown MUST close the HTTP bridge event batcher and cancel its +background flusher before the process exits. + +#### Scenario: Shutdown cancels the flusher + +- **WHEN** the proxy service begins shutdown after queueing an event +- **THEN** the batcher's background task is cancelled and awaited + +### Requirement: Classify response.incomplete as terminal + +An anchored `response.incomplete` event MUST transition the durable operation to +an explicit terminal state and finalize its transcript so it is not left in an +unknown in-flight state. + +#### Scenario: Incomplete response is replayable as terminal + +- **WHEN** upstream emits `response.incomplete` +- **THEN** the operation is terminalized and its drained transcript is eligible for replay + +### Requirement: Settle reservations before timeout health + +When an eventless timeout retires a keyed bridge, the proxy MUST settle all +pending request reservations before recording the account timeout health signal. +If settlement fails, the health signal MUST NOT claim that cleanup completed. + +#### Scenario: Failed reservation release does not poison health state + +- **WHEN** the timeout cleanup cannot release a pending reservation +- **THEN** the account timeout signal is not recorded before that failure is surfaced + +### Requirement: Replay finalized incomplete operations + +A finalized `incomplete` operation transcript MUST be replayed for an identical +request and MUST NOT be reset or treated as an unknown in-flight operation. + +#### Scenario: Reconnect receives stored incomplete transcript + +- **WHEN** an identical request finds a finalized incomplete operation +- **THEN** the stored terminal transcript is delivered without a new upstream dispatch + +### Requirement: Validate final response.create size + +After adding durable operation metadata, the proxy MUST revalidate the exact +serialized `response.create` frame against the upstream size limit before +sending it. + +#### Scenario: Metadata cannot create an oversized frame + +- **WHEN** operation metadata makes the final frame exceed the configured limit +- **THEN** the request is rejected or slimmed before any upstream send + +### Requirement: Fence same-session active operations + +Server-indefinite recovery MUST NOT reset or redispatch a nonterminal operation +when another pending request in the same durable session still references that +operation. Submitted and acknowledged operations MUST remain fail-closed; +only an inactive `unknown` operation may enter a fresh recovery attempt. + +#### Scenario: Active same-session operation is not duplicated + +- **WHEN** a duplicate request finds a submitted operation still referenced by another pending request +- **THEN** the proxy refuses a second dispatch and preserves the existing spool + +### Requirement: Responses routes preserve the Ultrafast service tier + +Responses-compatible routes MUST accept the canonical `ultrafast` service tier and MUST forward it unchanged. When upstream reports the actual response tier, request logging MUST preserve `ultrafast` using the existing requested, actual, and billable tier contract. + +#### Scenario: Explicit Ultrafast request is forwarded + +- **WHEN** a client sends a Responses request with `service_tier: "ultrafast"` +- **THEN** the forwarded upstream payload contains `service_tier: "ultrafast"` + +#### Scenario: Upstream confirms Ultrafast processing + +- **WHEN** upstream completes a request with `response.service_tier: "ultrafast"` +- **THEN** the actual and billable request-log tiers are `ultrafast` + +### Requirement: Account-bound retries remain on their dispatch owner + +The proxy MUST bind a Responses request body that is not a canonical +account-neutral fresh replay to the account that first receives that exact +body. Every later selection for that request MUST treat the dispatch owner as a +strict required account across HTTP streaming, HTTP bridge, and direct +WebSocket transports. + +The proxy MUST NOT exclude the dispatch owner and send the retained body to a +different account during stale-anchor recovery, retryable account failure, +Trusted Access migration or degradation, bridge reconnect, or WebSocket account +switching. If the required owner is unavailable, the proxy MUST fail closed +without dispatching the retained body to another account. + +The proxy MAY perform one forced authentication refresh and replay a retained +account-bound body on the same dispatch owner. It MUST NOT use that refresh to +exclude the owner or migrate the body to another account, and a permanent +authentication failure MUST remain terminal for the bound body. + +The proxy MAY clear the dispatch-owner binding only after verified recovery +replaces the exact wire body and the replacement passes the canonical +account-neutral-fresh-replay predicate. Removing `previous_response_id` alone +MUST NOT make retained account-scoped input portable. + +Proxy-owned operation metadata that will be added at the send boundary MUST +remain bound to the current account unless an explicit operation-rebind path +replaces that identity before account selection. Installing a verified fresh +body and clearing its dispatch-owner binding MUST occur as one state +transition. + +#### Scenario: Encrypted reasoning remains on its first dispatch account + +- **GIVEN** account A first receives a Responses request containing encrypted + reasoning or another account-scoped retained item +- **WHEN** a pre-visible retry excludes account A or requests a differently + authorized account +- **THEN** the proxy does not dispatch the retained body to account B +- **AND** the retry fails closed when account A is unavailable + +#### Scenario: Verified account-neutral fresh replay may change accounts + +- **GIVEN** verified recovery removes a stale continuation anchor +- **AND** the exact replacement body contains only canonical account-neutral + fresh input +- **WHEN** normal retry selection chooses account B +- **THEN** the proxy may dispatch the replacement body to account B + +#### Scenario: Confirmed pre-dispatch failure does not create an owner + +- **GIVEN** account A is selected for a nonportable Responses body +- **WHEN** transport evidence confirms the request failed before any upstream + bytes were dispatched +- **THEN** the proxy does not record account A as the dispatch owner +- **AND** normal retry selection may dispatch the body first on account B + +#### Scenario: HTTP bridge preserves payload ownership + +- **GIVEN** an HTTP bridge request has already dispatched a nonportable body to + account A +- **WHEN** pre-created recovery or reconnect selection excludes account A +- **THEN** the bridge does not submit that body on account B + +#### Scenario: Direct WebSocket preserves payload ownership + +- **GIVEN** a direct WebSocket request has already dispatched a nonportable body + to account A +- **WHEN** retry handling prepares an account switch +- **THEN** the proxy rejects the switch unless the exact replacement body is a + canonical account-neutral fresh replay + +#### Scenario: Bound authentication refresh stays on the owner + +- **GIVEN** a nonportable body is bound to account A +- **WHEN** account A reports a refreshable authentication failure before + visible output +- **THEN** the proxy may refresh and replay once on account A +- **AND** it does not dispatch the retained body to account B + +#### Scenario: HTTP bridge operation identity remains on its owner + +- **GIVEN** an HTTP bridge retry retains a proxy-owned operation identity +- **AND** no explicit operation rebind has replaced that identity +- **WHEN** retry selection evaluates another account +- **THEN** the bridge requires the current operation owner + +#### Scenario: Existing settlement ordering is unchanged + +- **GIVEN** an API-key reservation requires settlement during the failed retry +- **WHEN** account health is updated +- **THEN** required settlement still completes before deferred health writes + +### Requirement: Compact terminal SSE errors preserve top-level error type + +When the compact Responses upstream terminates with a top-level SSE `type=error` frame, the proxy MUST preserve a supplied non-blank `error_type` in the emitted OpenAI error envelope. If `error_type` is absent, non-string, or blank, the proxy MUST use `server_error`. The proxy MUST preserve existing status, code, message, and parameter mapping, and MUST NOT alter nested OpenAI-style error-envelope behavior. + +#### Scenario: Top-level invalid request type is preserved + +- **WHEN** compact upstream terminates with a top-level `type=error` frame whose `error_type` is `invalid_request_error` +- **THEN** the proxy returns HTTP 400 with `error.type=invalid_request_error` +- **AND** preserves the frame's code, message, and parameter + +#### Scenario: Missing or blank top-level type uses compatibility fallback + +- **WHEN** compact upstream terminates with a top-level `type=error` frame whose `error_type` is absent or blank +- **THEN** the emitted OpenAI error envelope uses `error.type=server_error` +- **AND** existing status, code, message, and parameter mapping remains unchanged + +#### Scenario: Nested compact error envelope remains unchanged + +- **WHEN** compact upstream terminates with a nested OpenAI-style error envelope +- **THEN** the proxy preserves the nested type and all other mapped fields using the existing parser diff --git a/openspec/specs/runtime-portability/spec.md b/openspec/specs/runtime-portability/spec.md index 957a866c8d..bd790b4d94 100644 --- a/openspec/specs/runtime-portability/spec.md +++ b/openspec/specs/runtime-portability/spec.md @@ -53,3 +53,41 @@ The `codex-lb` CLI SHALL provide a `codex-sessions retag` subcommand that rewrit - **THEN** the command uses that path as the Codex data directory - **AND** otherwise it falls back to `CODEX_HOME`, `/codex-home` in containers, a discoverable WSL Windows profile Codex directory, or `~/.codex` +### Requirement: Server CLI validates the main listener port before startup + +The `codex-lb` server CLI SHALL accept integer main-listener ports in the inclusive range `0..65535` when supplied through `--port` or `PORT`, and an explicit `--port` SHALL continue to take precedence over `PORT`. The CLI SHALL reject non-integer values and integers outside that range before loading Uvicorn, importing or starting the ASGI application, running its lifespan or migrations, or creating runtime data. A rejection MUST identify `--port/PORT`, state the supported range, and include the invalid value. + +#### Scenario: Out-of-range command-line port is rejected before startup + +- **WHEN** an operator supplies `--port` with an integer below `0` or above `65535` +- **THEN** the CLI exits with an error that identifies `--port/PORT`, the invalid value, and the supported range `0..65535` +- **AND** Uvicorn is not loaded +- **AND** the ASGI lifespan, migrations, and runtime data creation do not run + +#### Scenario: Out-of-range environment port is rejected before startup + +- **WHEN** `PORT` contains an integer below `0` or above `65535` +- **AND** no `--port` flag is supplied +- **THEN** the CLI exits with an error that identifies `--port/PORT`, the invalid value, and the supported range `0..65535` +- **AND** Uvicorn is not loaded +- **AND** the ASGI lifespan, migrations, and runtime data creation do not run + +#### Scenario: Non-integer listener port is rejected before startup + +- **WHEN** the selected `--port` or `PORT` value is not an integer +- **THEN** the CLI exits with an error that identifies `--port/PORT` and the invalid value +- **AND** Uvicorn is not loaded + +#### Scenario: Inclusive listener-port boundaries are forwarded + +- **WHEN** the selected `--port` or `PORT` value is `0` or `65535` +- **THEN** the CLI forwards the same integer to Uvicorn +- **AND** port `0` retains Uvicorn's ephemeral-listener behavior + +#### Scenario: Command-line port retains precedence over the environment + +- **WHEN** `PORT` contains any value +- **AND** the operator supplies an in-range `--port` value +- **THEN** the CLI validates and forwards the flag value +- **AND** the environment value does not replace it + diff --git a/openspec/specs/sticky-session-operations/context.md b/openspec/specs/sticky-session-operations/context.md index 676bad1712..545d76556b 100644 --- a/openspec/specs/sticky-session-operations/context.md +++ b/openspec/specs/sticky-session-operations/context.md @@ -12,7 +12,13 @@ See `openspec/specs/sticky-session-operations/spec.md` for normative requirement - Bare process-session headers use a header-inaccessible, source-separated storage key and are soft only for self-contained pre-visible work. - Account-cap spillover is request-local: it selects an alternate without deleting or rebinding the process-session row. - Raw and legacy Codex rows remain hard during rolling upgrades because they may represent explicit turn-state ownership. -- Live file pins, responses, conversations, live/durable bridges, replay, and reattach sources are independent hard evidence; conflicting evidence fails closed instead of using source precedence. Opaque file IDs with no live pin remain unpinned for compatibility with uploads that occurred outside the current process. +- A raw legacy Codex owner can be abandoned only for an explicit goal-continuation restart whose canonical upstream payload passes the account-neutral fresh-replay proof, and only while that owner has a persisted unavailable status. Canonicalization keeps accepted compatibility fields and transport envelopes from changing classification. The compare-and-set marker is scoped to `session_header`, so an explicit turn-state lookup with colliding raw text retains the stored owner; a concurrent rebind or owner recovery still wins. The scoped marker deliberately leaves the historical global-tombstone timestamp empty, so replicas that do not understand scope continue to fail closed on the retained owner. Current Codex also sends `thread-id`; that locality source does not block the process-session exception. The raw compatibility lookup stays a `session_header` interpretation so a scoped tombstone cannot revive the retired owner on later thread-id turns. +- Restart mutation authority is the authenticated account-assignment and security-policy scope before model and service-tier eligibility. Model filtering constrains only replacement selection. +- Goal-restart retirement is an account-selection capability. An existing HTTP bridge cannot consume the request first through local reuse, durable-owner promotion, or forwarding. The retired owner is excluded from stale account snapshots for the remainder of the request, including when another selector wrote the scoped marker and this selector discovers it after losing the compare-and-set. +- Canonical bridge replacement preserves request-owned pre-submit admission on the detached predecessor, but that predecessor cannot publish new continuity aliases under the replacement's key. Every detached generation remains lifecycle-owned and capacity-counted until resource closure ends, including an idle predecessor already marked closed for admission. +- Drain status counts unsettled pending or queued work after detachment closes a generation for admission. If an idle predecessor alone fills the cap, the verified restart owns its bounded close synchronously and rechecks capacity before opening a replacement. +- Resource close is single-flight across reader retirement, account invalidation, and shutdown. Capacity is released only after resource finalization, not after detachment or a bounded-close timeout. Close finalization defers caller cancellation until owned resources are released, while shutdown starts all snapshotted closes before propagating cancellation and retains failed generations for a later close pass. Durable claims are fenced per websocket generation as well as per replica, so a replacement for a row still owned by the same configured replica advances the owner epoch before serving work even when model-transition isolation no longer uses that row for routing. Security-authorized rebind keeps typed continuity provenance so a source-qualified session-header tombstone cannot reappear as an untyped hard owner. +- Durable file pins, responses, conversations, live/durable bridges, replay, and reattach sources are independent hard evidence; conflicting evidence fails closed instead of using source precedence. Opaque file IDs with no live durable pin remain unpinned for compatibility with uploads that occurred outside the current process. A resolved file-pin owner bypasses the current-Codex thread PROMPT_CACHE row the same way it bypasses process-session locality, so an upload does not rebind later unpinned turns on that thread. - Dashboard prompt-cache TTL is persisted in settings so operators can adjust it without restart. - Background cleanup removes stale prompt-cache rows proactively, while manual delete and purge endpoints provide operator override. @@ -22,20 +28,49 @@ See `openspec/specs/sticky-session-operations/spec.md` for normative requirement - Durable `codex_session` and `sticky_thread` mappings are never deleted by automatic cleanup. - HTTP forbids CR/LF in headers and affinity parsing strips surrounding whitespace, while database text preserves LF. The internal soft-key sentinel therefore cannot be reproduced by a normalized client turn-state header. - Every transport resolves live and durable turn-state aliases; an existing route or socket is not itself proof that a newly supplied conversation belongs to that account. -- File owner indexes are process-local. Cross-replica bridge forwarding authenticates the origin-resolved owner rather than requiring a duplicate index on the remote owner. +- File owner pins live in the shared application database. Cross-replica bridge forwarding still authenticates the origin-resolved owner, but the receiving replica must revalidate that owner against a fresh durable lookup. ## Failure Modes - Cleanup failures are logged and retried on the next interval; request handling continues. - Manual purge and delete operations are dashboard-auth protected and return normal dashboard API errors on invalid input or missing keys. - Mixed-version replicas may temporarily produce both raw and namespaced rows. The raw row wins conservatively, which may reduce spillover but cannot weaken continuity. +- A raw process-session value may collide with an explicit turn-state value. Source-qualified abandonment lets the process session recover without making the retained raw account disappear from turn-state lookup. - Partial file-pin coverage or conflicting hard-owner metadata returns a stable fail-closed error before upstream dispatch; zero file-pin coverage preserves the established opaque-ID forwarding path. - A turn-state token learned from a retired WebSocket is discarded before a movable bare-session request connects to another account. +- An ordinary same-session request, or a goal-marked request that still carries previous-response, conversation, account-scoped file/image, or unresolved tool state, remains fail-closed on an unavailable raw owner. +- Local caps, retry exclusions, transient runtime health, and budget pressure never authorize legacy hard-owner abandonment. +- A live bridge may retain a detached ACTIVE account object after the database owner becomes unavailable. A verified goal restart bypasses that stale bridge and reaches guarded account selection; otherwise bridge reuse would strand the restart on the old owner. +- Selection inputs can predate the guarded retirement transaction. Once retirement succeeds, or once a compare-and-set loser rereads the winner's scoped marker, the old owner is filtered from those inputs so the request cannot immediately recreate namespaced affinity on it. +- A rolling older replica ignores the scope column. The scoped marker therefore leaves the historical timestamp tombstone empty, causing the older reader to keep the retained hard owner instead of treating the row as globally ownerless. +- Repeated restart replacement cannot hide visible or idle predecessor generations from the session cap or shutdown merely because a newer generation occupies the canonical key or the predecessor is admission-closed. +- Admission-closed pending work remains restart-blocking, while an idle predecessor that alone fills the cap is synchronously closed before replacement capacity is enforced. +- Account invalidation treats an owned/successful resource-close task, not the admission-only `closed` flag, as evidence that detached teardown is covered. +- A nested stream finalizer may clear a detached reservation before outer request cleanup runs, so final cleanup sweeps every detached generation for newly drainable ownership instead of relying on the mutable marker transition alone. +- A predecessor close may release after its same-replica replacement is created, including after a model transition clears the durable lookup from routing; generation-specific owner epochs prevent that stale release from closing the replacement lease. ## Example A process session is mapped to account A, but A is locally capped. A self-contained request may run on account B while the process-session row continues to point to A. If B produces `resp_123`, a follow-up carrying `previous_response_id=resp_123` follows B's response-owner index. If the same follow-up also references a file pinned to A, it fails with `continuity_owner_conflict` rather than choosing either source. By contrast, a first-turn request carrying only an opaque `file_external` ID that has no live codex-lb pin remains eligible for ordinary routing and is forwarded verbatim. +For an explicit restart example, session `thread-1` has a raw legacy mapping to +quota-exceeded account A while account B is active. Codex resends the complete +account-neutral thread under `thread-1`, without previous-response or +conversation continuity, and includes its goal-continuation marker. The proxy +marks the still-current A mapping abandoned for process-session interpretation, +selects B, and records subsequent session/response continuity on B. An explicit +turn-state request using the same raw text still resolves A. If A recovers or +the row is rebound before the marker commits, the compare-and-set misses and +the request remains fail-closed. + ## Operational Notes No schema or setting migration is required for bare-session spillover. Namespaced rows appear lazily, and old raw rows age out only through existing operational controls. Rollback simply removes the spillover capability and leaves both row forms readable. + +Goal-restart recovery adds no setting. Its nullable abandonment-scope migration +requires no backfill: a historical non-null timestamp with NULL scope remains +global, while `session_header` preserves an equal explicit turn-state owner. +Source-qualified markers leave the legacy timestamp NULL, so an older binary +safely restores conservative hard ownership during rollback. Dropping the scope +column loses only restart-recovery state; it does not make the retained owner +mobile. diff --git a/openspec/specs/sticky-session-operations/spec.md b/openspec/specs/sticky-session-operations/spec.md index 8091fe5fb4..718e1b31c4 100644 --- a/openspec/specs/sticky-session-operations/spec.md +++ b/openspec/specs/sticky-session-operations/spec.md @@ -7,6 +7,10 @@ Define sticky-session operation contracts so durable sessions, dashboard affinit ### Requirement: Sticky sessions are explicitly typed The system SHALL persist each sticky-session mapping with an explicit kind so durable Codex backend affinity, durable dashboard sticky-thread routing, and bounded prompt-cache affinity can be managed independently. Budget-pressure reallocation MUST apply only to mappings whose kind/source is soft. A raw or legacy `codex_session` mapping MUST remain owner-bound because it may represent explicit turn-state continuity; budget pressure MUST NOT delete or rebind it. +An explicit Codex goal-continuation restart MAY abandon a raw legacy `codex_session` owner only when the complete Responses payload is account-neutral and self-contained: it MUST have no nonblank `previous_response_id`, no nonblank `conversation`, no account-scoped input file or image reference, and no unresolved or orphan tool state. Classification MUST use the canonical upstream request form so accepted compatibility controls and transport-envelope fields do not make equivalent requests disagree. The owner MUST be persisted as `PAUSED`, `RATE_LIMITED`, or `QUOTA_EXCEEDED` and MUST belong to the authenticated request's account-assignment and security-policy scope computed before model and service-tier eligibility; local capacity, model eligibility, retry exclusions, runtime health, budget pressure, and an out-of-scope owner MUST NOT determine mutation authority. The retirement write MUST compare the current mapping owner and unavailable account status atomically, MUST preserve a concurrently changed mapping or recovered owner, and on success MUST let normal selection establish affinity to the replacement account. Because a raw key's persisted source is ambiguous, goal-restart abandonment MUST apply only to `session_header` interpretation and MUST retain the stored account as hard ownership for an explicit `turn_state` lookup using the same text. During a rolling deployment or rollback, replicas that do not understand source-qualified abandonment MUST continue treating that retained account as hard ownership. A selector that observes source-qualified abandonment initially or after losing the retirement compare-and-set MUST exclude the retained retired owner until replacement affinity is persisted, even if its account inputs predate retirement. Restart authority MUST remain scoped to the classified request and MUST NOT persist on a reusable bridge for later requests. A live or durable HTTP bridge for the same process session MUST NOT bypass this guarded selection through local reuse, owner forwarding, or preferred-owner promotion. Canonical replacement MUST preserve an already reserved predecessor request's authority to submit on its detached draining generation after queue publication clears the mutable reservation marker. A detached predecessor MAY finish its admitted response but MUST NOT publish new turn-state or previous-response aliases under the replacement generation's canonical key. Every detached generation, including an idle generation already marked closed for admission, MUST remain owned by the bridge lifecycle, MUST count against the configured session cap until resource closure completes, and MUST be closed during service shutdown, account invalidation, or drained reservation cleanup. The admission-only closed state MUST NOT be treated as proof that the socket and leases have a close owner. Resource teardown MUST be single-flight, and all close paths MUST release detached-generation ownership only after resource closure finishes, even when a close caller is cancelled. Shutdown MUST schedule and await every snapshotted generation before propagating cancellation. A new local bridge generation that replaces durable ownership under the same replica identity MUST advance the durable owner epoch before serving requests so a predecessor's late release cannot close the replacement lease, including when model-transition isolation discards the durable lookup as a routing input. + +A later security-authorized bridge replacement that revalidates a raw legacy row MUST preserve the request's typed continuity source. A source-scoped session-header abandonment MUST remain ownerless for that replacement while an explicit turn-state lookup of the same raw value remains owner-bound. A planned capacity eviction MUST NOT stop counting a detached generation merely because its bounded close wait timed out. Shutdown MUST retain any generation whose resource close fails so a later shutdown pass can retry finalization. Drain status MUST count pending or queued work on a detached generation even after it is closed for admission. When a verified restart replaces an idle predecessor that fills the configured session cap, admission MUST give that predecessor synchronous bounded-close ownership and MUST recheck actual lifecycle capacity before opening the replacement. + #### Scenario: Soft sticky reallocation uses split primary and secondary pressure thresholds - **WHEN** a request resolves an existing prompt-cache, sticky-thread, or other explicitly soft mapping - **AND** the pinned account is otherwise eligible to serve traffic @@ -40,10 +44,239 @@ The system SHALL persist each sticky-session mapping with an explicit kind so du - **GIVEN** a raw `codex_session` mapping points to account A - **AND** account A is temporarily quota-exceeded or otherwise unusable - **AND** account B is healthy -- **WHEN** hard-owner selection fails +- **WHEN** an ordinary request or an unsafe restart-shaped request requires the mapping - **THEN** the request fails closed instead of selecting account B - **AND** the raw mapping is neither deleted nor rebound +#### Scenario: Self-contained goal restart abandons unavailable legacy owner + +- **GIVEN** a process-session identifier has a raw legacy `codex_session` mapping to account A +- **AND** account A is paused, rate-limited, or quota-exceeded +- **AND** account B is eligible +- **WHEN** Codex sends the recognized goal-continuation marker with an account-neutral self-contained full resend and no other continuity dependency +- **THEN** the proxy marks the still-current raw mapping to account A abandoned only for process-session interpretation +- **AND** it routes the restarted turn to account B +- **AND** subsequent session or response continuity remains on account B + +#### Scenario: Goal restart cannot erase colliding explicit turn-state ownership + +- **GIVEN** a raw legacy `codex_session` row was written as explicit turn-state ownership for account A +- **AND** a process-session header later uses the same client-controlled text +- **WHEN** a marked self-contained goal restart abandons that text for process-session interpretation +- **THEN** the process-session restart may select account B +- **AND** an explicit turn-state lookup of the same text remains hard-bound to account A + +#### Scenario: Goal restart with process session and thread-id abandons the unavailable raw owner + +- **GIVEN** a process-session identifier has a raw legacy `codex_session` mapping to account A +- **AND** account A is paused, rate-limited, or quota-exceeded +- **AND** account B is eligible +- **AND** the request also carries a distinct `thread-id` +- **WHEN** Codex sends the recognized goal-continuation marker with an account-neutral self-contained full resend and no other continuity dependency +- **THEN** the proxy marks the still-current raw mapping to account A abandoned only for process-session interpretation +- **AND** it routes the restarted turn to account B +- **AND** subsequent same-thread continuity remains on account B + +#### Scenario: Thread-id on a goal restart cannot erase colliding explicit turn-state ownership + +- **GIVEN** a raw legacy `codex_session` row was written as explicit turn-state ownership for account A +- **AND** a later request carries the same text as a process-session header plus a distinct `thread-id` +- **WHEN** a marked self-contained goal restart abandons that text for process-session interpretation +- **THEN** the restart may select account B +- **AND** an explicit turn-state lookup of the same text remains hard-bound to account A + +#### Scenario: Account-dependent thread-scoped restart stays fail-closed + +- **GIVEN** a process-session identifier has a raw legacy mapping to unavailable account A +- **AND** the request carries a distinct `thread-id` +- **AND** the body has a previous response, conversation, file pin, or unresolved tool state +- **WHEN** the request is selected +- **THEN** the request fails closed on account A +- **AND** the raw mapping is neither deleted nor rebound + +#### Scenario: Source-qualified retirement fails closed on an older replica + +- **GIVEN** a current replica marks a raw account A mapping abandoned only for `session_header` interpretation +- **WHEN** a replica that does not understand abandonment scope reads the same raw mapping +- **THEN** it continues to resolve account A as hard ownership +- **AND** it cannot re-pin a colliding explicit turn state to another account + +#### Scenario: Model eligibility does not narrow retirement authority + +- **GIVEN** unavailable account A is inside the authenticated account-assignment and security-policy scope +- **AND** account A cannot serve the restart's requested model while account B can +- **WHEN** a marked self-contained goal restart evaluates the raw mapping owned by account A +- **THEN** account A remains authorized for the guarded abandonment mutation +- **AND** model and service-tier eligibility apply only when selecting the replacement + +#### Scenario: Equivalent request forms receive the same restart classification + +- **GIVEN** two marked self-contained goal restarts differ only by accepted compatibility controls or a transport-only response-create envelope +- **WHEN** the proxy classifies their account-neutral replay safety +- **THEN** it evaluates the same canonical upstream request fields for both forms +- **AND** neither form remains pinned merely because its accepted input representation differs + +#### Scenario: Goal restart bypasses a stale live HTTP bridge owner + +- **GIVEN** a live HTTP bridge and raw legacy mapping both identify account A for a process session +- **AND** the bridge's detached account snapshot still reports account A active +- **AND** account A is now persisted as paused, rate-limited, or quota-exceeded +- **WHEN** a marked account-neutral self-contained goal restart arrives for that process session +- **THEN** the proxy does not reuse or forward to account A's bridge +- **AND** guarded selection retires the raw owner before a replacement bridge is created on eligible account B + +#### Scenario: Restart authority does not outlive its request + +- **GIVEN** a marked self-contained goal restart creates a reusable HTTP bridge while legacy owner account A is healthy +- **WHEN** a later ordinary request reuses that bridge and account A has become unavailable +- **THEN** the ordinary request fails closed instead of inheriting the earlier restart's retirement authority +- **AND** the raw mapping to account A is neither tombstoned nor rebound + +#### Scenario: Reserved predecessor submits after canonical replacement + +- **GIVEN** an unanchored request has reserved the canonical session-header bridge before submit +- **AND** a verified goal restart replaces that canonical bridge while the reserved request is preparing its payload +- **WHEN** the reserved request publishes queued activity and clears its mutable reservation marker +- **THEN** the request submits exactly once on its detached predecessor generation +- **AND** canonical replacement does not reject that request as unregistered or replaced + +#### Scenario: Detached restart generations remain capacity bounded + +- **GIVEN** repeated verified restarts replace canonical bridges that still own visible or reserved requests +- **WHEN** the number of canonical, detached-live, and in-flight generations reaches the configured session cap +- **THEN** the service refuses another generation with its bounded local-capacity error +- **AND** detached sockets, readers, durable leases, and account leases are not omitted from capacity accounting + +#### Scenario: Idle detached predecessor remains capacity owned while closing + +- **GIVEN** a verified restart replaces an idle canonical bridge and its resource close is still running +- **WHEN** another restart would exceed the configured session cap +- **THEN** the admission-closed predecessor still counts as a detached generation +- **AND** the service either closes an evictable canonical generation before replacement creation or refuses the new generation + +#### Scenario: Closed detached request settlement blocks restart + +- **GIVEN** canonical replacement marked a detached predecessor closed for admission +- **AND** that predecessor still has pending or queued request settlement +- **WHEN** the service reports HTTP bridge drain status +- **THEN** the bridge remains active and restart-blocking +- **AND** it stops blocking only after the unsettled work reaches zero + +#### Scenario: One-session restart closes its idle predecessor before cap enforcement + +- **GIVEN** the bridge session cap is one and an idle canonical predecessor occupies that generation +- **WHEN** a verified goal restart forces canonical replacement +- **THEN** admission detaches the predecessor and gives it synchronous bounded-close ownership +- **AND** it opens the replacement only after close finalization releases the slot, otherwise it returns the bounded capacity refusal + +#### Scenario: Timed-out LRU close does not manufacture capacity + +- **GIVEN** admission detaches an idle LRU generation and reserves an in-flight replacement slot +- **AND** the bounded close wait returns before that generation's resource finalizer completes +- **WHEN** admission rechecks the configured session cap before opening the replacement socket +- **THEN** the detached generation still consumes capacity +- **AND** the service refuses replacement creation rather than exceeding the cap + +#### Scenario: Shutdown closes detached bridge generations + +- **GIVEN** canonical replacement detached an older generation whose request is still draining +- **WHEN** the service closes all HTTP bridge sessions +- **THEN** it closes both canonical and detached generations +- **AND** no detached socket, reader, durable lease, or account lease escapes shutdown ownership + +#### Scenario: Shutdown cancellation does not orphan later generations + +- **GIVEN** shutdown snapshots multiple canonical or detached bridge generations +- **AND** one generation has a slow resource close +- **WHEN** the shutdown caller is cancelled +- **THEN** every snapshotted generation receives a close owner before cancellation is propagated +- **AND** shutdown awaits all of those closes through resource finalization + +#### Scenario: Failed shutdown close remains retryable + +- **GIVEN** shutdown removes a canonical generation from routing and starts its resource close +- **WHEN** pending settlement or another resource finalizer fails +- **THEN** the generation remains in detached lifecycle ownership +- **AND** a later shutdown pass retries its close instead of losing the socket or leases + +#### Scenario: Detached predecessor cannot publish replacement continuity + +- **GIVEN** canonical replacement detaches an older generation while its admitted response is still draining +- **WHEN** that predecessor receives a new turn-state or previous-response alias +- **THEN** it does not publish the alias under the canonical key now occupied by the replacement +- **AND** the predecessor may still finish delivering its already admitted response + +#### Scenario: Detached generation closes after its final reservation ends + +- **GIVEN** a detached predecessor is retained only by an unsubmitted request reservation +- **WHEN** request finalization releases that reservation without submitting +- **THEN** the service closes the drained predecessor and releases its capacity ownership after resource closure finishes + +#### Scenario: Account invalidation includes detached generations + +- **GIVEN** an account owns both canonical and detached bridge generations +- **WHEN** the account is deactivated, requires reauthentication, or changes proxy binding +- **THEN** the service closes every generation authenticated to that account +- **AND** no detached socket remains routed through the invalid account binding + +#### Scenario: Admission-closed detached generation is still invalidated + +- **GIVEN** a detached generation is marked closed for admission but has no resource-close owner +- **WHEN** its account is invalidated +- **THEN** the service schedules resource teardown for that generation +- **AND** an already owned or successfully finalized close is not scheduled twice + +#### Scenario: Same-replica model replacement advances the durable epoch + +- **GIVEN** a durable bridge row names the current replica and an older model +- **WHEN** model-transition isolation creates a replacement generation and stops using that row for routing +- **THEN** the replacement claim still advances the durable owner epoch +- **AND** the predecessor's late release cannot close the replacement lease + +#### Scenario: Stale selection snapshot cannot repin a retired owner + +- **GIVEN** restart selection loaded account A as active before guarded retirement observes its unavailable persisted status +- **WHEN** guarded retirement tombstones account A's still-current raw legacy mapping +- **THEN** the remainder of that selection excludes account A from the stale snapshot +- **AND** the namespaced process-session mapping is not established on account A + +#### Scenario: Retirement CAS loser excludes the winner's retired owner + +- **GIVEN** two marked restarts read the same raw account A mapping and stale account inputs +- **AND** the first restart marks account A abandoned only for `session_header` interpretation +- **WHEN** the second restart loses its retirement compare-and-set and rereads that marker +- **THEN** the second restart excludes retained account A from its stale inputs +- **AND** it cannot establish replacement affinity on account A + +#### Scenario: Goal marker does not override account-scoped continuity + +- **GIVEN** a marked goal-continuation request carries a nonblank `previous_response_id`, nonblank `conversation`, account-scoped file or image reference, or unresolved tool output +- **WHEN** its hard owner is unavailable +- **THEN** the request fails closed +- **AND** the hard mapping is not abandoned + +#### Scenario: Healthy owner is not abandoned + +- **GIVEN** a marked account-neutral goal-continuation restart has a raw legacy owner that is still active +- **WHEN** the owner is locally capped, excluded, budget-pressured, or transiently unhealthy +- **THEN** the mapping remains owner-bound +- **AND** the restart does not retire it as unavailable + +#### Scenario: Concurrent owner change wins retirement race + +- **GIVEN** restart selection observed a raw legacy mapping to unavailable account A +- **WHEN** another operation rebinds that mapping or restores the owner before the retirement write executes +- **THEN** the compare-and-set retirement does not tombstone the newer state +- **AND** selection preserves fail-closed ownership semantics + +#### Scenario: Scoped API key cannot retire another pool's owner + +- **GIVEN** a raw legacy `codex_session` mapping points to unavailable account A +- **AND** the authenticated API key's effective account-policy scope contains account B but not account A +- **WHEN** the key sends a marked account-neutral goal-continuation restart for that session +- **THEN** the request fails closed before upstream dispatch +- **AND** the raw mapping to account A is neither tombstoned nor rebound + ### Requirement: Dashboard exposes sticky-session administration The system SHALL provide dashboard APIs for listing sticky-session mappings, deleting one mapping, and purging stale mappings. @@ -97,11 +330,11 @@ Prompt-cache and sticky-thread bridge affinity that does not carry a hard contin ### Requirement: Hard continuity remains owner-bound and bounded -Requests that depend on `previous_response_id`, hard turn-state, nonblank `conversation`, account-scoped `input_file.file_id` pins, live or durable bridge ownership, replay/reattach state, or another required owner continuity source MUST NOT silently reroute to an account that cannot preserve continuity. A resolved required owner MUST override bare process-session locality and MUST be selected without consulting or rewriting that soft mapping. A `previous_response_id` is a stored-object continuation reference and remains owner-bound even when the same request also carries a session header, `prompt_cache_key`, or another soft locality key. If independently resolved hard sources identify different accounts, if live referenced-file pins identify different accounts, or if a request has partial live file-pin coverage, the service MUST fail closed before upstream dispatch. A request for which no referenced file has a live pin MUST preserve opaque `file_id` compatibility and proceed without treating the absent process-local metadata as ownership evidence. If the owner account/session is unavailable or saturated, the service MUST fail closed with an explicit retryable continuity/local overload reason instead of flooding the owner queue indefinitely. +Requests that depend on `previous_response_id`, hard turn-state, nonblank `conversation`, account-scoped `input_file.file_id` pins, live or durable bridge ownership, replay/reattach state, or another required owner continuity source MUST NOT silently reroute to an account that cannot preserve continuity. A resolved required owner MUST override bare process-session locality and MUST be selected without consulting or rewriting that soft mapping. A `previous_response_id` is a stored-object continuation reference and remains owner-bound even when the same request also carries a session header, `prompt_cache_key`, or another soft locality key. If independently resolved hard sources identify different accounts, if live durable referenced-file pins identify different accounts, or if a request has partial live durable file-pin coverage, the service MUST fail closed before upstream dispatch. A request for which no referenced file has a live durable pin MUST preserve opaque `file_id` compatibility and proceed without inventing ownership evidence. If the owner account/session is unavailable or saturated, the service MUST fail closed with an explicit retryable continuity/local overload reason instead of flooding the owner queue indefinitely. Every HTTP, compact, direct WebSocket, and HTTP-bridge transport MUST resolve explicit turn state against both live and durable bridge aliases. Live, durable, previous-response, file, and explicit turn-state evidence MUST be compared independently; source ordering MUST NOT choose the first match when distinct sessions or accounts resolve. A reused direct WebSocket MUST repeat nonblank `conversation` ownership validation for each response-create frame because the existing socket account proves only the current route. Single-account routing MUST constrain effective routing without narrowing the ownership-candidate pool used by that validation. -When an HTTP-bridge owner is on another replica, the origin MUST forward its resolved file owner in authenticated full-context metadata, and the receiving owner MUST NOT require the same process-local file pin. A retired direct WebSocket's upstream turn-state token MUST NOT be sent to a different account selected for a later movable bare-session request. +When an HTTP-bridge owner is on another replica, the origin MUST forward its resolved durable file owner in authenticated full-context metadata. The receiving owner MUST perform its own fresh shared-database lookup and MUST require that durable result to match the forwarded owner. A missing or conflicting receiver-side durable owner MUST fail closed before account selection or upstream invocation. A retired direct WebSocket's upstream turn-state token MUST NOT be sent to a different account selected for a later movable bare-session request. A nonblank `conversation` without a dedicated resolved owner MUST proceed only when an explicit hard Codex mapping proves ownership or exactly one account remains in the model/API-key/security-scoped selection pool before transient additional-quota availability, retry exclusions, runtime health, budget, or account-cap filtering. A temporarily quota-filtered, excluded, unhealthy, or capped candidate MUST remain part of this ambiguity check because it may be the actual owner. A bare process-session mapping MUST NOT prove conversation ownership. @@ -122,11 +355,19 @@ A nonblank `conversation` without a dedicated resolved owner MUST proceed only w #### Scenario: File-pinned request owner overrides process-session locality - **GIVEN** a request carries a bare process-session header mapped to account A -- **AND** its `input_file.file_id` is pinned to account B +- **AND** its `input_file.file_id` is durably pinned to account B - **WHEN** the request is routed - **THEN** account B is treated as the required owner - **AND** the process-session mapping is neither consulted as an owner nor rewritten +#### Scenario: File-pinned request owner overrides thread locality + +- **GIVEN** a request carries a `thread-id` whose bounded mapping points to account A +- **AND** its `input_file.file_id` is durably pinned to account B +- **WHEN** the request is routed +- **THEN** account B is treated as the required owner +- **AND** the thread mapping is neither consulted as an owner nor rewritten + #### Scenario: Conflicting hard owners fail closed - **GIVEN** a turn state, previous response, bridge, or input file resolves to account A @@ -138,19 +379,19 @@ A nonblank `conversation` without a dedicated resolved owner MUST proceed only w #### Scenario: Partial or cross-account file pins fail closed - **GIVEN** a request references multiple account-scoped input files -- **AND** at least one file has a live owner pin -- **AND** another file has no live owner pin or the live pins resolve to different accounts +- **AND** at least one file has a live durable owner pin +- **AND** another file has no live durable owner pin or the live pins resolve to different accounts - **WHEN** the request is routed - **THEN** the service fails with `file_owner_unavailable` or `continuity_owner_conflict` - **AND** it does not route the files using a soft affinity account -#### Scenario: Opaque file IDs with no live pins preserve compatibility +#### Scenario: Opaque file IDs with no live durable pins preserve compatibility - **GIVEN** a request references one or more `input_file.file_id` values -- **AND** none of those IDs has a live process-local owner pin +- **AND** none of those IDs has a live durable owner pin - **WHEN** the request is routed - **THEN** the service forwards the opaque file references under ordinary unpinned routing -- **AND** it does not invent a hard owner or fail solely because local pin metadata is absent +- **AND** it does not invent a hard owner or fail solely because durable pin metadata is absent #### Scenario: Ambiguous conversation fails closed @@ -185,7 +426,7 @@ A nonblank `conversation` without a dedicated resolved owner MUST proceed only w #### Scenario: Preferred file owner does not manufacture a conversation owner -- **GIVEN** a request carries nonblank `conversation` continuity and a file pinned to account B +- **GIVEN** a request carries nonblank `conversation` continuity and a file durably pinned to account B - **AND** another account remains in the model/API-key/security ownership pool - **WHEN** no dedicated conversation owner can be resolved - **THEN** file ownership does not narrow the conversation ambiguity check to account B @@ -222,19 +463,19 @@ A nonblank `conversation` without a dedicated resolved owner MUST proceed only w - **THEN** the request remains ambiguous and fails closed - **AND** only the effective routing states are constrained to account A -#### Scenario: Remote bridge owner receives file ownership proof +#### Scenario: Remote bridge owner revalidates forwarded file ownership -- **GIVEN** an input file is pinned only in origin replica A's process-local index +- **GIVEN** origin replica A durably resolves an input file to account A - **AND** the request's HTTP bridge owner runs on replica B -- **WHEN** replica A forwards the request to replica B -- **THEN** the authenticated forwarding context carries the resolved file owner account -- **AND** replica B accepts that proof without requiring a duplicate local pin -- **AND** a missing, tampered, or legacy-unbound proof is rejected +- **WHEN** replica A forwards the request to replica B with authenticated file-owner metadata +- **THEN** replica B MUST freshly resolve the shared durable pin +- **AND** it MUST accept the forwarded owner only when both owner values match +- **AND** a missing, conflicting, tampered, or legacy-unbound proof MUST be rejected before upstream invocation #### Scenario: Retired WebSocket turn state does not cross accounts - **GIVEN** a closed upstream WebSocket on account A supplied an account-scoped turn-state token -- **AND** a later movable bare-session frame selects account B +- **AND** a later movable bare-session frame or marked self-contained goal restart selects account B - **WHEN** the proxy opens the replacement WebSocket - **THEN** it removes account A's stale turn-state token before connect - **AND** account B never receives that token @@ -576,3 +817,89 @@ The background cleanup loop MUST delete ACTIVE and DRAINING `http_bridge_session - **GIVEN** the prompt-cache bridge idle TTL exceeds the prompt-cache affinity max age - **WHEN** the cleanup loop runs against an ACTIVE row whose lease expired but whose `last_seen_at` is within the prompt-cache bridge idle TTL - **THEN** the row and its aliases are preserved so a local reuse keeps its durable ownership and continuity anchors + +### Requirement: Restart removes stale owned bridge state + +On startup, the system MUST remove ordinary persisted HTTP bridge session rows owned by the configured bridge instance from the previous process. A recent server-namespaced account-neutral recovery row MUST instead be changed to ownerless DRAINING with an expired lease while preserving its aliases and original activity timestamp. The cleanup MUST remove ownerless ACTIVE/DRAINING rows with expired leases once their activity predates the abandoned-row retention cutoff. Deleted rows MUST lose their associated durable bridge aliases. The cleanup MUST NOT remove sticky-session mappings or rows owned by other bridge instances. + +#### Scenario: First request after restart starts without stale bridge state + +- **GIVEN** the previous process left ordinary durable HTTP bridge rows owned by the configured instance +- **WHEN** the next process completes startup +- **THEN** those durable bridge rows and their aliases MUST be removed before accepting requests +- **AND** the first request MUST create fresh bridge state instead of reusing the previous process's bridge row +- **AND** sticky-session mappings MUST remain available + +#### Scenario: Recent verified recovery proof survives restart only until retention + +- **GIVEN** the previous process left a recent server-namespaced account-neutral recovery row with task-specific aliases +- **WHEN** the next process completes startup +- **THEN** the row MUST become ownerless DRAINING with an expired lease +- **AND** its task-specific aliases and original activity timestamp MUST remain unchanged +- **AND** a later startup or abandoned-row cleanup MUST remove the row and aliases after the activity timestamp passes the retention cutoff + +#### Scenario: Ownerless stale rows with expired leases are removed + +- **GIVEN** durable HTTP bridge rows exist with no owner instance, expired leases, and activity older than the abandoned-row retention cutoff +- **WHEN** the process completes startup +- **THEN** those rows and their aliases MUST be removed +- **AND** rows owned by other instances MUST NOT be removed + +#### Scenario: Sticky-session mappings are preserved + +- **GIVEN** sticky-session mappings exist for the account +- **WHEN** the process completes startup and purges stale bridge rows +- **THEN** sticky-session mappings MUST remain available for account affinity + +### Requirement: Trusted capability requirements are monotonic across lineage + +Before dispatch, the proxy MUST persist an authenticated `trusted_cyber` +requirement as API-key-scoped, domain-separated opaque hashes for every known +session, accepted or synthesized turn-state, previous-response, and Codex task +lineage alias. Marker writes MUST be monotonic and MUST NOT store raw lineage +or account identifiers. + +A later authenticated request MUST restore REQUIRED before account selection +when any presented alias matches under the same API-key scope. It MUST persist +that requirement onto newly generated aliases. A marker under one API key MUST +NOT establish REQUIRED under another key. Read or write uncertainty MUST fail +before ordinary dispatch. + +When `response.created` first reveals a response ID for a durably REQUIRED +request, the proxy MUST persist the upstream and downstream-visible response +aliases before forwarding that created event. If this propagation fails, the +proxy MUST NOT expose the unpersisted response ID, replay the accepted request, +or penalize the upstream account. + +#### Scenario: No-echo reconnect remains required +- **WHEN** a capability-bearing direct WebSocket turn persists an accepted + session identity and a proxy-synthesized turn state +- **AND** a new connection presents the same session identity without the + capability marker or generated turn state +- **THEN** REQUIRED is restored before its first account selection + +#### Scenario: Echoed synthesized turn state remains required +- **WHEN** the reconnect instead echoes the accepted synthesized turn state +- **THEN** REQUIRED is restored before its first account selection + +#### Scenario: Response-only reconnect remains required +- **WHEN** a capability-bearing turn exposes a response ID only after upstream + acceptance +- **AND** a new connection presents only that `previous_response_id` under the + same API key, without a matching session or turn state +- **THEN** REQUIRED is restored before its first account selection + +#### Scenario: Requirement survives a fresh service instance +- **WHEN** a new repository and proxy service instance reads an alias marked by + an earlier instance +- **THEN** the alias still restores REQUIRED + +#### Scenario: API-key scope is isolated +- **WHEN** API key B presents the same visible lineage identifier previously + marked under API key A +- **THEN** key A's marker does not establish REQUIRED for key B + +#### Scenario: Persistence uncertainty cannot downgrade +- **WHEN** required lineage cannot be read or established durably +- **THEN** the request fails before ordinary account selection or dispatch + diff --git a/openspec/specs/telemetry/context.md b/openspec/specs/telemetry/context.md new file mode 100644 index 0000000000..db9e2015db --- /dev/null +++ b/openspec/specs/telemetry/context.md @@ -0,0 +1,207 @@ +# Telemetry capability — context + +## Purpose + +Give the project visibility into its install base (version distribution, deployment shapes, +client ecosystem, feature usage) without collecting anything that identifies an operator, +an account, or request content. Consent model is informed opt-out: active by default, +one-time dialog with the exact payload, settings toggle, env kill switch. + +Decision record (2026-08-06, maintainer): default-on with first-run confirmation dialog for +both new and existing users; settings toggle; expanded field set over the minimal version. + +## Collection endpoint + +Self-hosted SHM (kOlapsis/shm) server operated by the maintainer at +`https://telemetry.tokmaxxing.com`. SHM provides Ed25519 instance signing, aggregate dashboards, +and public README badges (`/badge/codex-lb/instances`, `/badge/codex-lb/version`). +The SDK path is `/v1/register`, `/v1/activate`, `/v1/snapshot` (note: NOT `/api/v1/`, +which is SHM's admin namespace). codex-lb implements a small Python client (SHM ships +Go/Node SDKs only). + +## Payload schema v1 (the allowlist) + +Everything below derives from existing data (`request_logs`, settings, module registry). +No new per-request instrumentation. `*_bucket` fields use the documented bucket sets. + +Every outbound request body has an explicit Pydantic model and is covered by the wire-schema +allowlist test. The registration body sent to `/v1/register` is: + +```json +{ + "app_name": "codex-lb", + "app_version": "1.20.2", + "deployment_mode": "docker | k8s | pip | bare", + "environment": "", + "instance_id": "", + "os_arch": "linux/x86_64", + "public_key": "" +} +``` + +`app_name` identifies this project, `app_version` supports upgrade/deprecation decisions, +`deployment_mode` and `os_arch` are coarse deployment signals, `environment` is intentionally +empty, and `instance_id` plus `public_key` establish the random signing identity. Activation +sends only `{"action": "activate"}`. + +The signed `/v1/snapshot` body is an envelope. The consent preview renders this same shape via +the same constructor; its timestamp is the current preview-generation time, while an actual +send regenerates the current transmission time. + +```json +{ + "instance_id": "", + "metrics": { "": "..." }, + "timestamp": "2026-08-06T12:00:00Z" +} +``` + +The `metrics` object is the versioned snapshot schema: + +```json +{ + "schema_version": 1, + "instance_id": "", + "version": "1.20.2", + "python": "3.13", + "os": "linux", + "arch": "x86_64", + "uptime_hours": 168, + + "deploy": { + "method": "docker | k8s | pip | bare", + "db_backend": "sqlite | postgres", + "db_size_bucket": "unknown | ", + "replicas": 3, + "reverse_proxy": true + }, + + "accounts": { + "pool_bucket": "", + "plan_mix": {"plus": "", "pro": "", "team": "", "free": ""}, + "workspace_accounts": true, + "routing_policy": "", + "limit_warmup_enabled": true, + "egress_proxy_used": false + }, + + "usage_7d": { + "requests": 203051, + "success_rate": 0.987, + "tokens_input": 18800000000, + "tokens_output": 94000000, + "tokens_cached_ratio": 0.89, + "cost_usd_bucket": "", + "request_kinds": {"responses": 0.0, "chat": 0.0, "images": 0.0, "unknown": 1.0}, + "transport_mix": {"ws": 0.6, "http_bridge": 0.4}, + "service_tier_mix": {"default": 0.90, "flex": 0.05, "priority": 0.05}, + "clients": {"codex-cli": 0.44, "openai-sdk-python": 0.3, "other": 0.02}, + "clients_other_ratio": 0.02, + "models": [ + { + "name": "gpt-5.4-codex", + "share": 0.62, + "reasoning": {"xhigh": 0.31, "high": 0.48, "medium": 0.21}, + "avg_output_tokens_bucket": "" + } + ], + "latency_ms_p50": 1200, + "ttft_ms_p50": 800, + "ttft_ms_p95": 3400, + "rate_limit_429_ratio": 0.004, + "top_upstream_errors": ["server_overloaded", "usage_limit_reached"] + }, + + "features": { + "api_firewall": true, + "quota_planner": true, + "sticky_sessions": true, + "conversation_archive": false, + "automations": false, + "fleet": false, + "model_sources_count": 2, + "api_keys_bucket": "", + "prometheus": false, + "otel": false, + "dashboard_auth": true, + "reset_credits": true, + "image_api_used": true + } +} +``` + +Field notes: + +- `top_upstream_errors`: enum `upstream_error_code` values only, top 5 by count. Free-text + `error_message` is banned by spec. +- `request_kinds`: current `request_logs` rows do not persist ingress route family. The existing + `request_kind` column is a workload class (`normal`, `warmup`, `compaction`, and similar), + while `source` identifies the upstream. Until an authoritative route-family signal exists, + rows are reported as `unknown`; source and model-name heuristics are deliberately forbidden. +- `clients`: canonical family shares from the normative mapping table in `spec.md`. Raw + `useragent_group` values never leave the instance. +- `models[].name`: official model catalog allowlist match; custom/unknown model names fold + into a single `{"name": "other"}` entry. +- Exact `requests` / token counts are transmitted raw deliberately: they power the global + aggregate counter story and cannot identify an instance. Everything correlated with spend + or org size (accounts, keys, cost, DB size) is bucketed. +- `replicas`: size of the configured HTTP bridge instance ring (multi-replica adoption signal). + +## Bucket sets + +- count buckets (accounts, api keys, plan mix): `0`, `1`, `2-5`, `6-20`, `21-100`, `100+` +- `db_size_bucket`: `unknown`, `<100MB`, `100MB-1GB`, `1-5GB`, `5-10GB`, `10-50GB`, `50GB+` +- `cost_usd_bucket` (7d): `<10`, `10-100`, `100-1k`, `1k-10k`, `10k-50k`, `50k+` +- `avg_output_tokens_bucket`: `<250`, `250-1k`, `1k-4k`, `4k-16k`, `16k+` + +## Consent resolution precedence + +`CODEX_LB_TELEMETRY_ENABLED` env (when set) > persisted decision > default +(`undecided` ⇒ active). The dialog is only shown while persisted state is `undecided` and +no env override exists. + +## Consent API and preview cost + +`GET /api/settings/telemetry` always returns `state`, `source`, `active`, and `preview`. The +default GET includes a preview envelope only for undecided/default consent, when the dialog can +appear; decided and environment-overridden responses return `preview: null` without running the +seven-day aggregate queries. Settings requests the same endpoint with +`include_preview=true` to fetch the current envelope on demand. `PUT /api/settings/telemetry` +persists the decision and returns `preview: null`. + +## Cadence and replica ownership + +The startup and 24-hour ticks run through the shared scheduler leader-election gate. Only the +leader constructs aggregates, transmits the snapshot, and logs the undecided-consent startup +notice. Followers perform none of that work, avoiding duplicate snapshots and duplicate notices. + +## Retention + +Each snapshot summarizes the previous seven days of existing local request logs. codex-lb does +not create a second local telemetry history or queue failed transmissions. The project-operated +collector is `https://telemetry.tokmaxxing.com`; its server-side retention duration is not yet +specified, so operators should assume transmitted snapshots remain stored until a published +retention policy or explicit deletion. + +## Failure modes + +- Endpoint down: bounded timeout (5s), at most one retry per interval, debug-level log, + proxy path untouched. Snapshot is rebuilt fresh next interval (no queue/backlog). +- Aggregation query cost: snapshot queries reuse the same 7-day aggregate shapes as the + dashboard reports module; they run on the leader scheduler once per tick and only on an API + request when the undecided dialog or an explicit settings preview needs them. On Postgres + instances with very large `request_logs` this is the same load class as one dashboard load. +- Clock skew / restart loops: the elected leader transmits the startup snapshot; SHM's + `/v1/activate` is idempotent (active → active refreshes last-seen). Rapid restart loops are + bounded by one snapshot per elected-leader process start; no local rate limiter in v1. + +## Example: privacy review quick check + +An instance with accounts `alice@corp.com` (workspace W1) + 12 others, a custom model source +`corp-internal-gpt`, and traffic from an internal tool `senpi/1.0`: + +- payload has `pool_bucket: "6-20"`, `workspace_accounts: true` +- `corp-internal-gpt` traffic appears as `models[].name == "other"` +- `senpi` traffic appears in `clients` under `other` and inflates `clients_other_ratio` +- the strings `alice`, `corp.com`, `W1`, `corp-internal-gpt`, `senpi` appear nowhere in the + serialized payload (schema snapshot test enforces this) diff --git a/openspec/specs/telemetry/spec.md b/openspec/specs/telemetry/spec.md new file mode 100644 index 0000000000..d9e8b674bd --- /dev/null +++ b/openspec/specs/telemetry/spec.md @@ -0,0 +1,223 @@ +# telemetry Specification + +## Purpose +Anonymous install-base telemetry (version distribution, deployment shapes, client ecosystem, feature usage) under an informed opt-out consent model, with a normative privacy allowlist so identifying data can never be transmitted. See `context.md` for the exact outbound field lists. +## Requirements + +### Requirement: Telemetry payload field allowlist + +The service MUST transmit only fields defined for each outbound body (registration, +activation, and snapshot envelope including its nested metrics) in this capability's +`context.md`, and MUST NOT transmit account emails, workspace identifiers, client IP +addresses, API keys, request or response content, raw user-agent strings, per-account +records, or free-text error messages in any telemetry payload. + +The snapshot metrics schema is versioned (`schema_version`). Adding a field to any transmitted +body requires a spec change to this capability; the outbound wire-schema test suite MUST fail +when registration, activation, the snapshot envelope, or nested metrics contain a field not +present in the documented schema. + +#### Scenario: Every outbound body contains only allowlisted fields + +- **WHEN** the sender serializes registration, activation, and snapshot requests +- **THEN** every top-level and nested field is present in the documented schemas, and an + outbound wire-schema regression test rejects any undeclared field in any body + +#### Scenario: Identifying data never serialized + +- **WHEN** the snapshot is built on an instance with linked accounts, API keys, and request + logs containing raw user agents and error messages +- **THEN** the serialized payload contains no email, workspace ID, IP address, API key + material, raw user-agent string, or free-text error message + +### Requirement: Consent state and default activation + +Telemetry consent MUST be a persisted tri-state (`undecided`, `enabled`, `disabled`) defaulting to `undecided`, and while consent is `undecided` the service SHALL treat telemetry as active. + +Upgrading an existing installation MUST introduce the consent state as `undecided` (existing +users get the same informed default-on treatment as new installs). + +#### Scenario: Fresh install defaults to active + +- **WHEN** codex-lb starts for the first time with no persisted consent and no environment + override +- **THEN** consent is `undecided` and telemetry snapshots are transmitted + +#### Scenario: Upgrade treats existing users as undecided + +- **WHEN** an existing installation migrates to a version with this capability +- **THEN** the migrated consent state is `undecided` and the one-time consent dialog is shown + on next dashboard entry + +### Requirement: One-time consent dialog with exact payload preview + +The dashboard MUST present a one-time consent dialog on first entry while consent is +`undecided`, and the dialog MUST display the exact snapshot envelope the instance would +transmit at that moment. Preview and sender MUST use one shared envelope constructor. The +preview timestamp MUST record preview generation time as a representative current timestamp; +the actual send MUST regenerate that value at transmission time. + +A decision (enable or disable) MUST be persisted and the dialog MUST NOT be shown again after +any decision. The dialog MUST offer disabling with no fewer clicks than enabling. + +The consent API MUST build the preview only while the undecided dialog is eligible or when an +operator explicitly requests it for the settings view. The response MUST retain the `preview` +field and set it to `null` when the preview was not requested and is not dialog-relevant. + +#### Scenario: Undecided operator sees payload preview + +- **WHEN** an operator opens the dashboard while consent is `undecided` +- **THEN** a dialog shows the live snapshot JSON with equally prominent enable and disable + actions + +#### Scenario: Decision is final until changed in settings + +- **WHEN** the operator chooses disable in the dialog +- **THEN** consent persists as `disabled`, no snapshot is transmitted afterward, and the + dialog never reappears + +#### Scenario: Decided consent status is a cheap read + +- **WHEN** the dashboard reads consent after a persisted decision without requesting a preview +- **THEN** the response contains `preview: null` and no snapshot aggregation query runs + +#### Scenario: Settings explicitly requests collected data + +- **WHEN** the settings view requests a preview for any consent state +- **THEN** the response contains a current snapshot envelope built with the same schema as the sender + +### Requirement: Settings toggle and environment kill switch + +The dashboard settings MUST expose a telemetry toggle reflecting the resolved consent state, and the environment variable `CODEX_LB_TELEMETRY_ENABLED` MUST override persisted consent when set (`false` disables all transmission, `true` enables and suppresses the consent dialog). + +#### Scenario: Headless deployment disables via environment + +- **WHEN** the service runs with `CODEX_LB_TELEMETRY_ENABLED=false` +- **THEN** no telemetry network traffic occurs regardless of persisted consent, and the + settings toggle shows telemetry as disabled by environment override + +#### Scenario: Toggle flips persisted consent + +- **WHEN** the operator disables telemetry in settings without an environment override +- **THEN** consent persists as `disabled` and transmission stops without restart + +### Requirement: Startup notice while undecided + +While consent is `undecided`, the elected leader MUST emit a single startup log line stating +that anonymous telemetry is active, where the collected-field documentation lives, and how to +disable it. Non-leader replicas MUST NOT duplicate the notice. + +#### Scenario: Headless operator is informed + +- **WHEN** the service starts with consent `undecided` +- **THEN** exactly one log line names the telemetry documentation location and the + `CODEX_LB_TELEMETRY_ENABLED=false` disable path + +### Requirement: Disabled means zero telemetry traffic + +When resolved consent is `disabled`, the service MUST NOT open any network connection to the telemetry endpoint. + +#### Scenario: No connection attempts when disabled + +- **WHEN** telemetry is disabled and the service runs through startup and a 24-hour scheduler + cycle +- **THEN** no connection attempt to the telemetry endpoint is made + +### Requirement: Client family allowlist mapping + +Telemetry client statistics MUST report only canonical client-family identifiers produced by the documented mapping table, MUST map any unmatched user-agent group to `other`, and MUST NOT transmit raw user-agent group values. + +The canonical mapping table (raw `useragent_group` → family): + +| Raw group(s) | Family | +| --- | --- | +| `codex_exec`, `codex-tui` | `codex-cli` | +| `Codex Desktop` | `codex-desktop` | +| `codex_vscode` | `codex-vscode` | +| `AsyncOpenAI` | `openai-sdk-python` | +| `OpenAI` | `openai-sdk-js` | +| `ai`, `ai-sdk` | `vercel-ai-sdk` | +| `opencode` | `opencode` | +| `Mozilla` | `browser` | +| `curl`, `undici`, `node`, `Python-urllib`, `python-requests`, `aiohttp` | `script` | +| anything else | `other` | + +The payload MUST include `clients_other_ratio` so mapping coverage decay is observable +without ever transmitting the unmatched raw values. + +#### Scenario: Private tool names never leave the instance + +- **WHEN** request logs contain a user-agent group not present in the mapping table +- **THEN** its traffic is attributed to `other` and the raw group string is absent from the + payload + +#### Scenario: Codex CLI variants collapse to one family + +- **WHEN** traffic exists from both `codex_exec` and `codex-tui` +- **THEN** the payload reports a single `codex-cli` family combining both + +### Requirement: Model catalog allowlist with per-model reasoning mix + +Telemetry model statistics MUST include only model names present in the official model catalog allowlist, MUST map unmatched model names to `other`, and MUST report reasoning-effort distribution nested per model entry rather than as an instance-global aggregate. + +#### Scenario: Custom model source names are not transmitted + +- **WHEN** an operator has configured a custom model source with a private model name +- **THEN** that traffic appears under `other` and the private name is absent from the payload + +#### Scenario: Reasoning effort is model-scoped + +- **WHEN** the snapshot reports models +- **THEN** each model entry carries its own reasoning-effort share map and no global + reasoning mix field exists + +### Requirement: Fail-honest request-family attribution + +Request-family telemetry MUST be derived only from an authoritative persisted route-family +signal. Rows without such a signal MUST be attributed to `unknown`; the service MUST NOT infer +Chat, Responses, Images, or Audio families from upstream `source` or model name. + +#### Scenario: Ambiguous persisted rows remain unknown + +- **WHEN** persisted request rows identify only workload kind, upstream source, or model name +- **THEN** their request-family share is reported as `unknown` rather than a named route family + +### Requirement: Random instance identity + +The telemetry instance identifier MUST be a UUID generated randomly on first run, MUST NOT be derived from hardware, network, account, or operating-system identity, and MUST be regenerated if deleted. + +#### Scenario: Identifier carries no fingerprint + +- **WHEN** the instance identifier is created +- **THEN** it is a random UUIDv4 persisted locally, and deleting it yields a fresh unrelated + identifier on next start + +### Requirement: Transmission cadence and failure isolation + +The service SHALL transmit one snapshot at startup and one per 24-hour interval thereafter. +In a multi-replica deployment sharing a database, snapshot construction and transmission MUST +run only under the existing leader-election gate so at most one replica performs each tick. +Telemetry transmission failures MUST NOT affect proxy operation, MUST use a bounded timeout, +MUST NOT retry more than once per interval, and MUST log failures at debug level only. + +#### Scenario: Non-leader replica skips telemetry work + +- **WHEN** a telemetry tick runs in a process that does not hold the scheduler leader lease +- **THEN** that process neither builds a snapshot nor attempts a transmission + +#### Scenario: Collection endpoint outage is invisible + +- **WHEN** the telemetry endpoint is unreachable +- **THEN** proxy requests are unaffected, startup is not delayed beyond the bounded timeout, + and no warning-or-higher log noise is produced + +### Requirement: Bucketed sensitive aggregates + +Account pool size, per-plan account counts, API key count, database size, and cost aggregates MUST be transmitted as documented buckets, never as exact values. + +An unmeasurable database size MUST be reported as `unknown`, not as a plausible size bucket. + +#### Scenario: Pool size is a bucket + +- **WHEN** an instance has 13 linked accounts +- **THEN** the payload reports the `6-20` bucket and no exact account count diff --git a/openspec/specs/upstream-proxy-routing/spec.md b/openspec/specs/upstream-proxy-routing/spec.md index ba2abac2f6..75ec4b972e 100644 --- a/openspec/specs/upstream-proxy-routing/spec.md +++ b/openspec/specs/upstream-proxy-routing/spec.md @@ -95,3 +95,67 @@ failure. - **GIVEN** a routed WebSocket context manager enters successfully - **WHEN** the client returns the opened WebSocket and its context to the caller - **THEN** the caller can exit the returned context using the existing ownership contract + +### Requirement: Cached route resolution preserves fail-closed semantics + +Any cache in front of upstream-route resolution MUST store the resolver's outcome verbatim — a resolved route, a permitted direct-egress `None`, or a fail-closed error with its reason. A cache hit MUST reproduce that outcome exactly: it MUST NOT convert a fail-closed outcome or a routed outcome into direct egress, and it MUST NOT substitute a different pool or endpoint than the resolver chose. Cache staleness MUST be bounded by invalidation on admin mutations (same-replica: before the mutating response returns; peers: within one cache-invalidation poll interval) with a TTL backstop for out-of-band edits. + +#### Scenario: Cached fail-closed outcome keeps failing closed + +- **GIVEN** an account-bound pool with no active usable endpoint whose fail-closed resolution outcome is cached +- **WHEN** further upstream operations are attempted for that account +- **THEN** each operation MUST fail before opening an upstream network connection with the same fail-closed reason +- **AND** it MUST NOT use the default pool, environment proxy, or direct egress + +#### Scenario: New binding takes effect without a direct-egress window on the mutating replica + +- **GIVEN** an account whose cached resolution outcome is direct-egress `None` +- **WHEN** an operator saves an active proxy binding for that account +- **THEN** the mutating replica's cached outcome MUST be invalidated before the binding response returns, so subsequent requests on that replica resolve the bound pool + +### Requirement: Confirmed account-proxy connection failures fail over safely + +When an account-routed transport reports that it could not connect to the selected proxy endpoint and proves that the upstream request was not dispatched, the service MUST classify the failure with sanitized structured pre-dispatch provenance. For a route with another usable endpoint in the same proxy pool, the client MUST try that endpoint before moving accounts, including for a non-idempotent request. If the pool cannot connect, movable Responses requests MUST exclude the failed account and retry another eligible account within the existing request budget and attempt limits. + +This behavior MUST cover raw HTTP/SSE, native Responses WebSocket, and the HTTP responses bridge. Before recording transient account backoff, the service MUST release response-create and stream leases held for the failed account. A request-scoped API-key reservation MUST remain singular across an internal pre-dispatch failover, MUST settle or release at the terminal request outcome before the account-health write, and MUST NOT be reacquired solely for the internal failover. If neither settlement nor fallback release can be confirmed, the service MUST leave the health write unapplied. HTTP-bridge startup cleanup MUST release only an unowned current request lifecycle, and each reservation lifecycle MUST drain only its own health writes after confirmed settlement or release. The confirmed failure MUST place the account at the existing bounded transient error-backoff floor, but MUST NOT pause, deactivate, rate-limit, or quota-penalize it. + +The service MUST NOT replay a request when dispatch is unknown or when the request depends on hard previous-response, turn-state, uploaded-file, single-account, or other required account ownership. If no eligible replacement account exists, the service MUST preserve the original sanitized upstream-unavailable failure instead of replacing it with a generated `no_accounts` error. + +#### Scenario: POST uses a healthy endpoint from the same proxy pool + +- **GIVEN** a non-idempotent Responses POST is routed through a proxy pool with two endpoints +- **AND** connecting to the first endpoint fails before request dispatch +- **WHEN** the second endpoint is reachable +- **THEN** the service sends the request through the second endpoint +- **AND** it does not move the request to another account + +#### Scenario: movable request retries another account + +- **GIVEN** two eligible accounts and the first account's complete proxy route refuses connections before dispatch +- **WHEN** a fresh Responses request has no hard account ownership +- **THEN** the service releases the first account's response-create and stream leases +- **AND** it settles or releases any request-scoped API-key reservation before the account-health write +- **AND** it records bounded transient backoff for the first account +- **AND** it excludes the first account and completes through the second account +- **AND** no failure event from the first attempt is forwarded downstream + +#### Scenario: hard account ownership fails closed + +- **GIVEN** a Responses request depends on a previous-response owner or an account-scoped uploaded file +- **AND** the required account's proxy refuses the connection before dispatch +- **WHEN** another account is otherwise eligible +- **THEN** the service does not send the request to the other account +- **AND** it returns the sanitized upstream-unavailable failure for the required account + +#### Scenario: ambiguous transport failure is not replayed + +- **WHEN** a POST transport failure cannot prove that request dispatch was impossible +- **THEN** the service does not use that failure as authorization to retry another proxy endpoint or account + +#### Scenario: empty replacement pool preserves the original failure + +- **GIVEN** a movable request has a confirmed pre-dispatch proxy connection failure +- **AND** no other eligible account can be selected +- **THEN** the client receives the original sanitized upstream-unavailable failure +- **AND** the failure is not replaced with `no_accounts` + diff --git a/openspec/specs/usage-error-metrics/spec.md b/openspec/specs/usage-error-metrics/spec.md new file mode 100644 index 0000000000..484b300d87 --- /dev/null +++ b/openspec/specs/usage-error-metrics/spec.md @@ -0,0 +1,170 @@ +# usage-error-metrics Specification + +## Purpose +Error-rate accounting that counts only genuinely-failed terminals: cancelled client disconnects fold into a separate cancelled_count in live metrics and hourly rollups, with historical rows kept compatible. +## Requirements +### Requirement: Error metrics count only genuinely-failed terminals + +Every materialization of a request-log error count or error rate — the usage +summary metrics, the dashboard overview activity metrics and per-bucket +error-rate trend inputs, the reports daily and summary aggregates, and the +fleet pressure metrics — MUST classify a request-log row as an error only +when `status NOT IN ('success', 'cancelled')`. Rows with `status = +'cancelled'` (normal client-side disconnect terminals, e.g. +`error_code=client_disconnected`) MUST NOT be counted in any error numerator. +Error-rate denominators MUST remain the total request count of the window. +Every request-log producer MUST record a downstream client disconnect as +`status='cancelled'`; in particular the model-source streaming path MUST NOT +record a mid-stream client disconnect as `status='error'`. + +#### Scenario: Cancelled rows do not inflate the dashboard error rate + +- **GIVEN** a window containing 1 successful, 2 cancelled + (`client_disconnected`), and 1 error (`upstream_500`) request-log rows +- **WHEN** the dashboard overview activity metrics are computed +- **THEN** the error count is `1` and the error rate is `0.25` +- **AND** the request total remains `4` + +#### Scenario: Reports and fleet windows exclude cancelled rows from errors + +- **GIVEN** the same window of rows +- **WHEN** the reports summary/daily aggregates and the fleet pressure + metrics are computed +- **THEN** each reports `error_count` / `total_errors` and each fleet + `error_count` equals `1` + +#### Scenario: Model-source stream disconnects land as cancelled + +- **GIVEN** a streamed model-source request whose downstream client + disconnects mid-stream +- **WHEN** the request log is written +- **THEN** the row has `status='cancelled'` and + `error_code='client_disconnected'` +- **AND** the window's error count excludes it, its cancelled count includes + it, and `top_error` does not report `client_disconnected` + +### Requirement: Hourly rollups fold a cancelled_count measure + +The `request_usage_hourly_rollups` table MUST carry a `cancelled_count` +measure (non-null, server default 0), introduced by an additive Alembic +migration whose parent is the current single migration head and whose +downgrade drops only the new column. The hourly fold MUST populate +`cancelled_count` as `sum(status = 'cancelled')` and MUST fold `error_count` +as `sum(status NOT IN ('success', 'cancelled'))`. Account lifecycle mirrors +MUST move `cancelled_count` with the other measures. + +#### Scenario: Fold splits error and cancelled measures + +- **GIVEN** one hour of raw rows with 1 success, 2 cancelled, and 1 error + sharing the same dimensions +- **WHEN** the hourly fold pass folds that hour +- **THEN** the folded bucket has `request_count=4`, `error_count=1`, and + `cancelled_count=2` + +### Requirement: Historical hourly rollup rows keep the old error fold + +Hourly rollup rows folded before the `cancelled_count` measure existed MUST +NOT be backfilled or re-split: their `error_count` keeps the legacy +`sum(status != 'success')` fold and their `cancelled_count` reads 0 via the +column's server default. Error-rate trends over such buckets exhibit a +disclosed step change at deploy. + +#### Scenario: Pre-existing rollup rows are readable unchanged + +- **GIVEN** a rollup row folded before the migration +- **WHEN** the dashboard reads it after upgrading +- **THEN** the read succeeds with `cancelled_count=0` and the row's stored + `error_count` unchanged + +### Requirement: New code repairs the rolling-upgrade fold window + +Because the migration runs before old replicas drain, a legacy replica may +fold post-migration hours with the old error fold and advance the shared +watermark — by up to its full per-pass slice budget, so no fixed trailing +window can bound the damage. The migration MUST persist the legacy-suspect +range start on the fold-state row (`upgrade_repair_from`): existing rows are +stamped with their migration-time `hourly_folded_through`, and the column's +epoch server default covers a state row bootstrapped by an old replica after +the migration (its entire backfill is legacy-suspect); new code's own +bootstrap MUST write the marker as NULL, and NULL MUST only ever be written +by new code, meaning no legacy-suspect range is outstanding. + +While the marker is set, the hourly fold pass MUST refold +`[upgrade_repair_from, hourly_folded_through)` from raw request logs in +bounded slice-sized chunks, persisting progress by advancing the marker with +each chunk's commit and setting it to NULL only once the range is covered — +a crash resumes instead of restarting, and a pass-bounded incomplete repair +continues on later passes. With the marker NULL, the first fold pass of each +new-code process MUST still refold the trailing repair window below the +watermark (a span that comfortably exceeds any rolling-upgrade duration) as +defense against a legacy replica regaining fold leadership after the marker +was cleared. + +Both paths MUST be idempotent (converging DELETE-then-INSERT recomputation), +MUST run under the existing fold leader gate and fold-state row lock, MUST +NOT move the watermark, and MUST NOT touch folded buckets below the +surviving-raw clamp (whole hours fully covered by surviving raw rows; +retention-pruned history is irrecoverable and keeps the disclosed legacy +fold). This is a targeted repair of the rollout window only — not a +historical backfill. + +#### Scenario: A legacy-folded post-migration bucket is repaired + +- **GIVEN** a bucket inside the repair window whose rollup rows carry the + legacy fold (cancelled rows in `error_count`, `cancelled_count=0`, + `client_disconnected` in the error satellite) while its raw rows survive +- **WHEN** the new code runs its first hourly fold pass +- **THEN** the bucket is recomputed with `error_count` excluding cancelled + rows, `cancelled_count` populated, and the `client_disconnected` satellite + rows removed + +#### Scenario: A multi-slice legacy advance is fully repaired via the marker + +- **GIVEN** `upgrade_repair_from` set below legacy-folded buckets spanning + more than one fold slice (a legacy leader advanced several slices in one + pass) +- **WHEN** the new code runs its hourly fold passes +- **THEN** every bucket in `[upgrade_repair_from, watermark)` with surviving + raw rows is recomputed and the marker ends NULL + +#### Scenario: Buckets below the surviving-raw clamp are preserved + +- **GIVEN** a folded bucket inside the repair span whose raw rows were + already pruned by retention +- **WHEN** the repair runs +- **THEN** that bucket's rollup rows are left untouched + +### Requirement: Top error excludes cancelled terminals + +`top_error` computations MUST NOT derive from cancelled rows: raw request-log +scans MUST filter `status NOT IN ('success', 'cancelled')`, the error +satellite fold MUST apply the same status filter going forward, and reads of +historical error-satellite rows (folded under the legacy filter) MUST exclude +the `client_disconnected` error code. + +#### Scenario: client_disconnected no longer dominates top error + +- **GIVEN** a window with 200 cancelled rows (`client_disconnected`) and 3 + error rows (`upstream_500`) +- **WHEN** `top_error` is computed for the dashboard or fleet windows +- **THEN** the result is `upstream_500` + +### Requirement: Cancelled counts surface alongside error counts + +Metric surfaces that expose an error count MUST also expose the window's +cancelled count as an additive field: the dashboard overview metrics +(`cancelledCount`), the usage summary metrics (`cancelled7d`), the reports +daily rows (`cancelled_count`) and summary (`total_cancelled`), and the fleet +pressure metrics (`cancelledCount`). The dashboard overview cancelled total +MUST be sourced from the demand quarter rollup (status grain) for the folded +segment plus the raw tail, so it stays accurate across history already folded +without the hourly `cancelled_count` measure. + +#### Scenario: Dashboard overview reports the status breakdown + +- **GIVEN** a window containing 1 successful, 2 cancelled, and 1 error rows + that are partially folded into the rollups +- **WHEN** the dashboard overview metrics are computed +- **THEN** the metrics expose `requests=4`, `errorCount=1`, and + `cancelledCount=2` + diff --git a/openspec/specs/usage-refresh-policy/spec.md b/openspec/specs/usage-refresh-policy/spec.md index 5e44ca14d6..9228254a6d 100644 --- a/openspec/specs/usage-refresh-policy/spec.md +++ b/openspec/specs/usage-refresh-policy/spec.md @@ -505,10 +505,9 @@ The system SHALL NOT infer weekly secondary semantics solely because a primary-s ### Requirement: Background usage refresh is staggered across accounts -Background usage refresh MUST distribute account refresh attempts across the -configured usage refresh interval instead of refreshing every eligible account -in one burst. Each scheduler slice MUST attempt at most one eligible account. -Over a full cycle, all eligible accounts SHOULD be considered once. +Background usage refresh MUST distribute account refresh attempts across the configured usage refresh interval instead of refreshing every eligible account in one burst. Each scheduler slice MUST attempt at most one eligible account. Over a full cycle, all eligible accounts SHOULD be considered once. + +Each slice MUST select its account before reading usage history and MUST scope its latest-usage lookups, updater input, warm-up candidate evaluation, and recoverable-status evaluation to that selected account. The scheduler MAY retain the full eligible account roster only to choose the deterministic rotation and calculate staggered warm-up phases; that roster MUST NOT cause usage-history reads, upstream refresh attempts, warm-up sends, or status mutations for an unrelated account in the slice. A selected-account refresh failure MUST NOT trigger same-slice fallback to another account. Database sessions used to load scheduler state MUST close before upstream network I/O begins, and concurrent follow-up work MUST NOT share an `AsyncSession`. #### Scenario: Scheduler refreshes one account per slice @@ -516,8 +515,7 @@ Over a full cycle, all eligible accounts SHOULD be considered once. - **WHEN** the scheduler runs consecutive refresh slices - **THEN** the first slice attempts one account - **AND** the second slice attempts the other account -- **AND** cache invalidation for usage-derived routing state runs at the cycle - boundary +- **AND** cache invalidation for usage-derived routing state runs at the cycle boundary #### Scenario: Unrefreshable accounts are skipped by scheduler rotation @@ -527,6 +525,38 @@ Over a full cycle, all eligible accounts SHOULD be considered once. - **WHEN** the scheduler builds the refresh rotation - **THEN** only the active account is considered +#### Scenario: Selected slot scopes usage history and follow-up work + +- **GIVEN** two eligible accounts have stored primary, secondary, and monthly usage +- **AND** the first account is selected for the current scheduler slice +- **WHEN** the scheduler reads before/after usage and evaluates warm-up and recoverable status +- **THEN** every usage-history lookup is filtered to the first account +- **AND** only the first account is passed to usage refresh, warm-up candidate evaluation, and recoverable-status evaluation +- **AND** the second account cannot be mutated or contacted during that slice + +#### Scenario: Warm-up phase cohort does not widen evaluation scope + +- **GIVEN** multiple warm-up-enabled accounts participate in staggered-idle phase calculation +- **AND** one account is selected for the current usage-refresh slice +- **WHEN** refreshed usage is evaluated for warm-up +- **THEN** the phase calculation retains the eligible fleet cohort +- **AND** only the selected account can create a warm-up attempt or send warm-up traffic + +#### Scenario: Selected-account failure does not fail over within the slice + +- **GIVEN** two accounts are eligible for scheduler rotation +- **AND** the first account is selected +- **WHEN** that account's usage refresh fails +- **THEN** the scheduler does not attempt the second account in the same slice +- **AND** the second account remains eligible for its normal later slice + +#### Scenario: Scheduler session closes before selected-account network work + +- **GIVEN** the scheduler loaded the account roster and selected account usage +- **WHEN** the selected account's upstream refresh starts +- **THEN** the scheduler read session is already closed +- **AND** any concurrent warm-up follow-up owns an independent database session + ### Requirement: Usage refresh trusts recognized paid-plan transitions without workspace identity Usage refresh MUST persist a stored account's `plan_type` change when @@ -1255,6 +1285,111 @@ local writes but MUST NOT be the mechanism that guarantees dedup. - **THEN** the unique constraint rejects the duplicate - **AND** the worker treats the rejection as a dedup skip rather than an error +### Requirement: Compact failover settles before account-health writes + +When `compact_responses` holds an API-key usage reservation, it MUST NOT write account health for a compact upstream failure until that reservation has been settled or released. A `failover_next` decision MUST keep the same reservation for the next account and MUST defer the failed account's health write until the next settlement. Timeout and exhaustion terminals MUST keep settle-then-health order. Compact MUST NOT acquire a second reservation mid-request. If usage finalization fails but the fail-safe reservation release succeeds, compact MUST flush deferred health before surfacing `usage_settlement_failed`. If the reservation remains held because that fail-safe release also fails, deferred health MUST stay unapplied. After a compact reservation is finalized, a deferred health-persistence failure MUST NOT replace the successful compact response. If compact exits through cancellation or any exception other than a `ProxyResponseError` that already settled, it MUST settle or release the reservation and flush deferred health before propagating that exception. A later account-selection budget timeout after `failover_next` MUST use that same settle-and-flush path. Deferred health flush MUST complete even if the compact request is cancelled while that flush is awaiting a health write. If one deferred health write fails, compact MUST still attempt the remaining deferred health writes. + +#### Scenario: Compact failover_next defers health until settle + +- **GIVEN** a compact request with a held API-key reservation +- **AND** the first account fails with a `failover_next` class +- **WHEN** a later account completes and settlement runs +- **THEN** `_handle_stream_error` for the failed account runs only after that settlement +- **AND** the request does not acquire another reservation + +#### Scenario: Compact timeout still settles before health + +- **GIVEN** a compact request whose upstream call times out +- **WHEN** the timeout branch records account health +- **THEN** the reservation is settled before `_handle_stream_error` + +#### Scenario: Compact HTTP 500 failover defers health until settle + +- **GIVEN** a compact request with a held API-key reservation +- **AND** the first account exhausts same-account HTTP 500 retries +- **WHEN** a later account completes and settlement runs +- **THEN** `_handle_proxy_error` and extra `record_errors` for the failed account run only after that settlement + +#### Scenario: Compact route failure after failover still applies deferred health + +- **GIVEN** a compact request that deferred health on `failover_next` +- **WHEN** the next account raises `UpstreamProxyRouteError` +- **THEN** the reservation is settled +- **AND** the deferred health write still runs + +#### Scenario: Compact refresh/connect failover defers health until settle + +- **GIVEN** a compact request with a held API-key reservation +- **AND** the first account fails a retryable freshness/connect or post-401 forced-refresh transport error +- **WHEN** a later account completes and settlement runs +- **THEN** `_handle_stream_error` for the failed account runs only after that settlement + +#### Scenario: Compact second 401 failover defers health until settle + +- **GIVEN** a compact request with a held API-key reservation +- **AND** the same account returns 401 again after a forced refresh +- **WHEN** a later account completes and settlement runs +- **THEN** `_handle_proxy_error` for the failed account runs only after that settlement + +#### Scenario: Compact permanent refresh settles before the health mark + +- **GIVEN** a compact request with a held API-key reservation +- **AND** the post-401 forced refresh raises a permanent `RefreshError` +- **WHEN** the compact request records the permanent account failure +- **THEN** the reservation is settled before `mark_permanent_failure` + +#### Scenario: Compact fallback release still flushes deferred health + +- **GIVEN** a compact request that deferred health on `failover_next` +- **AND** a later account completes but usage finalization fails +- **AND** the fail-safe reservation release succeeds +- **WHEN** settlement surfaces `usage_settlement_failed` +- **THEN** the deferred health write still runs +- **AND** it runs before the `usage_settlement_failed` error is raised + +#### Scenario: Compact unsettled reservation keeps deferred health unapplied + +- **GIVEN** a compact request that deferred health on `failover_next` +- **AND** both usage finalization and fail-safe release fail +- **WHEN** settlement surfaces `usage_settlement_failed` +- **THEN** the deferred health write does not run + +#### Scenario: Compact success survives deferred health persistence failure + +- **GIVEN** a compact request that deferred health on `failover_next` +- **AND** a later account completes and usage finalization succeeds +- **WHEN** the deferred health write raises +- **THEN** the successful compact response is still returned + +#### Scenario: Compact unexpected exit still flushes deferred health + +- **GIVEN** a compact request that deferred health on `failover_next` +- **WHEN** the next account attempt raises cancellation or another non-proxy exception +- **THEN** the reservation is settled or released +- **AND** the deferred health write still runs +- **AND** the original exception is propagated + +#### Scenario: Compact deferred health flush completes under cancellation + +- **GIVEN** a compact request that deferred health on `failover_next` +- **AND** a later account completed and settlement started flushing +- **WHEN** the request is cancelled during the deferred health write +- **THEN** the deferred health write still completes + +#### Scenario: Compact continues flushing after one deferred health write fails + +- **GIVEN** a compact request that deferred health for more than one failed account +- **WHEN** the first deferred health write raises +- **THEN** later deferred health writes are still attempted + +#### Scenario: Compact selection timeout after failover still flushes deferred health + +- **GIVEN** a compact request that deferred health on `failover_next` +- **WHEN** selecting the next account exhausts the request budget +- **THEN** the reservation is settled or released +- **AND** the deferred health write still runs +- **AND** the original budget-timeout error is propagated + ### Requirement: Compact budget-exhausted terminals settle the API-key reservation before raising On the HTTP bridge / forwarded compact path the caller passes an `api_key_reservation_override` with `owns_reservation` false, making `compact_responses` the SOLE settler of the API-key usage reservation; therefore EVERY budget-exhausted terminal raise in the compact request path that is reached with a held, unsettled reservation MUST settle the compact API-key usage reservation (release it via `_settle_compact_api_key_usage` with `response` `None`) BEFORE raising the budget-exhausted `ProxyResponseError` (`upstream_request_timeout`), so held API-key quota is not leaked. This MUST apply to the outer-loop preflight budget terminals (before the freshness check, before the freshness reserve, and after the freshness check) and to the post-401 forced-refresh preflight budget terminal, each of which propagates straight to the outer `except ProxyResponseError` handler (which does not settle) and the `finally` (which only writes a request log). The terminal MUST preserve its prior escalation: it still raises the same `502` `upstream_request_timeout` error after settling, and it MUST still release the selected account's `response_create` lease where it already did so. A budget-exhausted terminal that is caught by an enclosing handler that already settles the reservation before raising — the inner upstream-call budget terminals, whose `upstream_request_timeout` error is settled by the retry loop's `upstream_request_timeout` / account-neutral branch — MUST NOT settle a second time, so the reservation is never double-settled. @@ -1328,3 +1463,184 @@ A non-2xx upstream response or the network-failure sentinel MUST NOT count as a - **WHEN** the older successful probe attempts to settle - **THEN** settlement is rejected as stale - **AND** the newer transient error state and reset success streak remain intact + +### Requirement: Implausible persisted rate-limit deadlines do not block recovery + +Background usage refresh MUST treat a persisted `rate_limited` reset deadline +as invalid when it is non-finite, elapsed, or beyond +`RATE_LIMIT_RESET_MAX_HORIZON_SECONDS` (366 days) plus the less-than-one-second +whole-second persistence tolerance. An invalid deadline MUST NOT be treated as +an unexpired explicit cooldown. When the account carries `blocked_at`, recovery +MUST still honor the existing 30-second minimum floor and MUST still require +the existing fresh available quota evidence recorded after the block. Without +`blocked_at`, recent available evidence SHALL suffice. Every applicable quota +window MUST report below `100%` usage before recovery. + +#### Scenario: Scheduler recovers an implausible persisted cooldown + +- **WHEN** an account is persisted as `rate_limited` with a reset deadline more than 366 days in the future +- **AND** its persisted `blocked_at` minimum floor has elapsed +- **AND** a later background usage refresh writes fresh available quota evidence +- **THEN** the scheduler treats the reset deadline as invalid +- **AND** marks the account `active` +- **AND** clears persisted `reset_at` and `blocked_at` + +#### Scenario: Scheduler preserves a plausible unexpired cooldown + +- **GIVEN** an account is persisted as `rate_limited` with a finite reset deadline within 366 days +- **AND** that deadline has not elapsed +- **WHEN** a later background usage refresh writes fresh available quota evidence +- **THEN** the scheduler leaves the account `rate_limited` + +#### Scenario: Scheduler recovers an implausible legacy deadline without a block marker + +- **GIVEN** an account is persisted as `rate_limited` with an implausible reset deadline and no `blocked_at` +- **WHEN** a later background usage refresh writes recent available quota evidence for every applicable window +- **THEN** the scheduler marks the account `active` +- **AND** clears persisted `reset_at` + +### Requirement: Weekly-primary remap tiebreak is data-aware within a fetch + +The weekly-primary to secondary remap tiebreak (`should_use_weekly_primary` / `normalize_weekly_only_rows`) MUST be data-aware within a single refresh fetch and MUST NOT let a sub-second `recorded_at` difference between same-fetch rows decide the winner. + +A row carries real quota metadata when it has a positive `window_minutes` AND a non-null `reset_at`; a row that lacks both is a no-data placeholder. For the data-aware tiebreak, a no-data placeholder MUST be classified as the absence of a measurement and MUST NOT be treated as a measurement of zero usage merely because its stored `used_percent` is zero — a timestamped placeholder must not beat an untimestamped real row, and a same-fetch real row must not be displaced by a placeholder. When two competing rows are from the same fetch (their `recorded_at` values differ by at most `SIBLING_FETCH_MARGIN_SECONDS`, 5.0 seconds, or one/both timestamps are unavailable), a weekly `primary` row that carries real quota metadata MUST be selected over a competing `secondary` row that is a no-data placeholder, and a real `secondary` row MUST be selected over a no-data `primary` placeholder. (Rendering a newer no-data placeholder that wins a cross-fetch comparison as an explicit "unavailable" window is out of scope for this change; the cross-fetch winner is rendered per existing placeholder rules.) + +When both rows carry `recorded_at` and their difference is strictly greater than `SIBLING_FETCH_MARGIN_SECONDS`, the rows are from genuinely different fetches and the newer row MUST win — a later fetch is more authoritative about what upstream currently reports. This preserves the pre-fix cross-fetch behavior so a stale real weekly primary cannot freeze the weekly value over a fresh placeholder from a later fetch. + +This tiebreak MUST be shared by every consumer of `should_use_weekly_primary`, including account-summary remap, dashboard overview and projection aggregation, and per-bucket account usage trend remap, so the weekly quota is reported consistently across all surfaces. + +#### Scenario: Same-fetch real weekly primary beats a no-data secondary placeholder + +- **GIVEN** an account whose latest `primary` usage row reports a weekly window (`window_minutes == 10080`) with a non-null `reset_at` and `used_percent` below 100 +- **AND** the latest `secondary` usage row is a no-data placeholder (`window_minutes` falsy or null, `reset_at` null, `used_percent` 0.0, no credit metadata) +- **AND** the two rows were recorded within `SIBLING_FETCH_MARGIN_SECONDS` (5.0 seconds) of each other in the same refresh cycle +- **WHEN** the system derives the effective secondary (weekly) usage window for account summaries, dashboard overview/projection aggregation, or account usage trends +- **THEN** the weekly `primary` row is selected as the source of weekly usage +- **AND** the reported weekly remaining percent equals `100 - primary.used_percent` +- **AND** the reported value does not jump to 100% remaining + +#### Scenario: Real secondary beats a no-data primary placeholder in the same fetch + +- **GIVEN** an account whose latest `secondary` usage row carries real quota metadata (positive `window_minutes` and a non-null `reset_at`) +- **AND** the latest `primary` usage row is a no-data placeholder +- **AND** the two rows were recorded within `SIBLING_FETCH_MARGIN_SECONDS` of each other +- **WHEN** the system derives the effective secondary usage window +- **THEN** the real `secondary` row is selected as the source of weekly usage +- **AND** the reported weekly remaining percent reflects that row's `used_percent` + +#### Scenario: Genuinely newer row from a later fetch wins regardless of metadata + +- **GIVEN** an account whose latest `primary` usage row reports a weekly window with real quota metadata but was written in an earlier fetch +- **AND** a later fetch wrote a competing `secondary` row whose `recorded_at` is more than `SIBLING_FETCH_MARGIN_SECONDS` (5.0 seconds) after the primary row +- **WHEN** the system derives the effective secondary usage window +- **THEN** the newer row from the later fetch is selected +- **AND** the stale real weekly primary does not freeze the weekly value indefinitely + +#### Scenario: Two real same-fetch weekly rows resolve by reset-at precedence + +- **GIVEN** an account whose latest `primary` and `secondary` usage rows both carry real quota metadata +- **AND** the two rows were recorded within `SIBLING_FETCH_MARGIN_SECONDS` (5.0 seconds) of each other in the same refresh cycle +- **WHEN** the system derives the effective secondary usage window across repeated refresh cycles +- **THEN** the selected row is determined by reset-at precedence and the stable weekly-primary default +- **AND** the selection does not flip between the two rows on a sub-second `recorded_at` difference + +### Requirement: Standard usage refresh snapshots persist atomically + +For one account's successful upstream usage response, the system MUST persist every available normalized standard usage window (`primary`, `secondary`, and any applicable `monthly` window) in one database transaction. All standard rows from that response MUST use the same capture timestamp. If any standard row cannot be persisted or the transaction cannot commit, the system MUST roll back the transaction so none of that response's standard rows becomes visible, and a caller-owned database session MUST remain open and reusable. This atomic unit applies to standard `usage_history` rows; additional per-model usage history and independent live-ingest writes retain their existing persistence contracts. + +#### Scenario: Multi-window response commits as one snapshot + +- **WHEN** a successful account usage response contains multiple normalized standard windows +- **THEN** the system persists all of those standard rows in one transaction with one shared capture timestamp + +#### Scenario: Later row failure leaves no partial snapshot + +- **WHEN** persistence fails after at least one standard row from an account response has been staged +- **THEN** the system rolls back the transaction and no standard row from that response is visible + +#### Scenario: Caller retains its session after rollback + +- **WHEN** a caller-owned session is used for a standard usage snapshot and the snapshot transaction fails +- **THEN** the repository leaves that session open and reusable after rolling back the failed transaction + +### Requirement: Owner-forwarded compact settlement failures fail closed + +An HTTP-bridge owner MUST treat any persistence exception while finalizing or +releasing a forwarded compact API-key usage reservation as a failed settlement, +and MUST NOT swallow it. +The owner MUST log the persistence failure, MUST attempt to release the +reservation through a fresh repository context, and MUST surface a `502` +`usage_settlement_failed` server error regardless of whether that fail-safe +release succeeds. The settlement failure MUST carry trusted internal provenance +that is checked before compact upstream retry, failover, and account-health error +handling, so the compact request is not sent upstream again and the selected +account is not penalized for a local persistence failure. When the reservation +is still `reserved` when the fail-safe release begins and that release succeeds, +the reservation's final status MUST be `released`. This behavior MUST NOT add or +alter stale-reservation cleanup or WebSocket health handling. + +#### Scenario: Forwarded compact finalization fails after upstream success + +- **GIVEN** a signed owner-forwarded compact request whose API-key reservation is `reserved` +- **AND** the upstream compact succeeds but usage finalization raises a persistence exception +- **WHEN** the owner handles the settlement failure +- **THEN** the owner attempts a fail-safe reservation release through a fresh repository context +- **AND** the request returns `502` with error code `usage_settlement_failed` +- **AND** the upstream compact is called exactly once and no account-health error is recorded +- **AND** when the reservation is still `reserved` at fail-safe release and that release succeeds, its status is `released` + +### Requirement: Reset-confirmed warm-up follows the plan-applicable long window + +When reset-confirmed limit warm-up evaluates an account's selected long quota +window, the system MUST use the monthly usage row when that account's plan has +monthly quota capacity and MUST otherwise use the secondary usage row. The +persisted warm-up attempt MUST retain the canonical window name from the +selected usage row. + +#### Scenario: Free monthly reset triggers one monthly warm-up + +- **GIVEN** limit warm-up is enabled globally and for a free-plan account +- **AND** long-window warm-up is selected +- **AND** the account's previous monthly usage sample was exhausted +- **WHEN** background usage refresh records a newer monthly sample with + available quota and a later `reset_at` +- **THEN** the system sends at most one warm-up request for that + account/monthly/reset tuple +- **AND** the durable warm-up attempt records `window="monthly"` + +#### Scenario: Paid plans retain secondary long-window warm-up + +- **GIVEN** an account plan has no monthly quota capacity +- **AND** primary and secondary usage samples are available +- **WHEN** background usage refresh evaluates long-window warm-up +- **THEN** the system uses the secondary usage row +- **AND** it does not substitute an unrelated monthly row + +#### Scenario: First monthly sample is not treated as a reset + +- **GIVEN** a free-plan account has no previous monthly usage sample +- **AND** its latest secondary sample is exhausted +- **WHEN** background usage refresh records the account's first monthly sample +- **THEN** the system does not compare the secondary and monthly `reset_at` + values as one window +- **AND** it does not send a reset-confirmed warm-up for that transition + +#### Scenario: Scheduler scopes monthly snapshots to the selected account + +- **GIVEN** multiple accounts are eligible for background usage refresh +- **AND** one account is selected for the current scheduler slice +- **WHEN** the scheduler loads before and after usage for warm-up evaluation +- **THEN** monthly lookups are filtered to the selected account +- **AND** monthly usage from another account cannot create a warm-up attempt + +### Requirement: Auth Guardian candidate handoff survives session closure + +Auth Guardian MUST preserve stable account identities while its candidate-query session is active and MUST execute selected refresh work after that session closes without reading unloaded or expired state from detached persistence objects. Each selected account MUST still be re-read in the separately owned refresh session before eligibility is confirmed and credentials are refreshed. + +#### Scenario: Stale candidate crosses the query-session boundary + +- **GIVEN** a stale eligible account is selected during an Auth Guardian pass +- **WHEN** the candidate-query session closes before per-account refresh work begins +- **THEN** Auth Guardian refreshes the selected account without a detached-instance failure +- **AND** the refresh worker re-reads the account in its own session before refreshing it + diff --git a/openspec/specs/user-documentation/spec.md b/openspec/specs/user-documentation/spec.md index b83a6f1645..237348d06e 100644 --- a/openspec/specs/user-documentation/spec.md +++ b/openspec/specs/user-documentation/spec.md @@ -59,3 +59,38 @@ OpenSpec remains the normative source of truth. Every docs page that documents s - **THEN** it contains the commented line `# CODEX_LB_LEADER_ELECTION_ENABLED=false` - **AND** no active (uncommented) assignment disables leader election +### Requirement: Generated settings reference stays in sync with the code + +The documentation site SHALL include a settings reference page +(`docs/reference/settings.md`) generated from `Settings.model_fields` by +`scripts/generate_settings_reference.py`. The page SHALL list, for every +setting, the `CODEX_LB_`-prefixed environment variable name, its type, and +its default (environment-derived defaults rendered symbolically), grouped by +functional area; it SHALL document the bare `PORT` special case and SHALL +list the removed (`_REMOVED_SETTINGS`) and deprecated env names sourced from +the code. The generated page SHALL be checked into the repository so the +strict docs build stays hermetic, SHALL carry a header identifying it as +generated, and SHALL link the owning OpenSpec capability. CI unit tests MUST +fail when the checked-in page differs from regenerated output, when the +settings surface exceeds its ratchet (115 fields; lower-only without a +simplicity-budget decision), or when an uncommented `.env.example` assignment +differs from the code default. + +#### Scenario: Settings change without regeneration fails CI + +- **GIVEN** a change to `Settings` fields in `app/core/config/settings.py` +- **WHEN** the unit test suite runs without regenerating `docs/reference/settings.md` +- **THEN** the regenerate-and-diff test fails until the page is regenerated and committed + +#### Scenario: Reference page is reachable and generated + +- **WHEN** a reader opens the published settings reference page +- **THEN** it is in the site navigation and linked from the Configuration page +- **AND** it identifies itself as generated from `scripts/generate_settings_reference.py` +- **AND** it links the owning OpenSpec capability + +#### Scenario: Settings surface growth trips the ratchet + +- **WHEN** the number of `Settings` fields exceeds the ratchet value +- **THEN** the ratchet unit test fails, forcing a simplicity-budget discussion before the surface grows + diff --git a/pyproject.toml b/pyproject.toml index 4e1e36ac0f..f9437a1557 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,10 +82,11 @@ dev = [ "pytest-timeout>=2.4.0", "httpx>=0.28.1", "ruff>=0.14.13", - "ty==0.0.69", + "ty==0.0.73", "openai>=2.16.0", "pytest-xdist>=3.8.0", "pytest-cov>=7.1.0", + "hypothesis>=6.165.3", ] docs = [ "mkdocs-material>=9.6", @@ -127,7 +128,7 @@ codex-lb = "app.cli:main" codex-lb-db = "app.db.migrate:main" [build-system] -requires = ["hatchling==1.31.0"] +requires = ["hatchling==1.32.0"] build-backend = "hatchling.build" [tool.hatch.build] diff --git a/scripts/generate_codex_client_evidence.py b/scripts/generate_codex_client_evidence.py deleted file mode 100644 index eff400f4ae..0000000000 --- a/scripts/generate_codex_client_evidence.py +++ /dev/null @@ -1,409 +0,0 @@ -"""Generate a content-free, fail-closed receipt from one Codex session JSONL. - -This tool owns only the independently derived client evidence domain. It never -accepts or copies server challenge fingerprints. -""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import secrets -import stat -import sys -import time -from collections import Counter -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -SCHEMA = "qk_codex_client_evidence_v2" -LEDGER_SCHEMA = "qk_client_full_checkpoint_tool_ledger_v1" -LEDGER_DOMAIN = b"qk-client-full-checkpoint-tool-ledger-v1\0" -ID_DOMAIN = b"qk-client-full-checkpoint-tool-id-v1\0" -IDENTITY_DOMAIN = b"qk-client-full-checkpoint-tool-identity-v1\0" -ARGUMENTS_DOMAIN = b"qk-client-full-checkpoint-tool-arguments-v1\0" -OUTPUT_DOMAIN = b"qk-client-full-checkpoint-tool-output-v1\0" -PAYLOAD_DOMAIN = b"qk-client-full-checkpoint-tool-payload-v1\0" -AUTHORITY_DOMAIN = b"qk-http-bridge-task-authority-v1\0" -ERROR_CODE_DOMAIN = b"qk-client-terminal-error-code-v1\0" -ERROR_MESSAGE_DOMAIN = b"qk-client-terminal-error-message-v1\0" -_VIOLATIONS = ( - "pending_calls", - "missing_id_events", - "orphan_outputs", - "duplicate_call_ids", - "duplicate_outputs", - "type_mismatches", -) - - -class EvidenceError(Exception): - """A stable, content-free fail-closed error.""" - - def __init__(self, code: str) -> None: - super().__init__(code) - self.code = code - - -@dataclass(frozen=True) -class Snapshot: - data: bytes - device: int - inode: int - size: int - mtime_ns: int - sha256: str - - -def _canonical(value: object) -> bytes: - return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":")).encode() - - -def _domain_hash(domain: bytes, value: object) -> str: - return hashlib.sha256(domain + _canonical(value)).hexdigest() - - -def _read_once(path: Path) -> Snapshot: - flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) - try: - fd = os.open(path, flags) - except OSError as exc: - raise EvidenceError("jsonl_open_failed") from exc - try: - before = os.fstat(fd) - if not stat.S_ISREG(before.st_mode): - raise EvidenceError("jsonl_not_regular_file") - chunks: list[bytes] = [] - while chunk := os.read(fd, 1024 * 1024): - chunks.append(chunk) - after = os.fstat(fd) - finally: - os.close(fd) - if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != ( - after.st_dev, - after.st_ino, - after.st_size, - after.st_mtime_ns, - ): - raise EvidenceError("jsonl_changed_during_read") - data = b"".join(chunks) - if len(data) != after.st_size: - raise EvidenceError("jsonl_short_read") - return Snapshot( - data, - after.st_dev, - after.st_ino, - after.st_size, - after.st_mtime_ns, - hashlib.sha256(data).hexdigest(), - ) - - -def stable_snapshot(path: Path, delay_seconds: float) -> Snapshot: - first = _read_once(path) - if delay_seconds > 0: - time.sleep(delay_seconds) - second = _read_once(path) - if ( - first.device, - first.inode, - first.size, - first.mtime_ns, - first.sha256, - ) != (second.device, second.inode, second.size, second.mtime_ns, second.sha256): - raise EvidenceError("jsonl_not_stable_across_reads") - return second - - -def _event_kind(item_type: object) -> str | None: - if not isinstance(item_type, str): - return None - if item_type.endswith("_call_output"): - return "output" - if item_type.endswith("_call"): - return "call" - return None - - -def _correlation_id(payload: dict[str, Any]) -> str | None: - for key in ("call_id", "id"): - value = payload.get(key) - if isinstance(value, str) and value: - return value - return None - - -def _task_authority_digest(identity: str) -> str: - payload = bytearray(AUTHORITY_DOMAIN) - for tag in ("session-id", "prompt_cache_key", "thread-id"): - tag_bytes = tag.encode() - value_bytes = identity.strip().encode() - payload.extend(len(tag_bytes).to_bytes(2, "big")) - payload.extend(tag_bytes) - payload.extend(len(value_bytes).to_bytes(4, "big")) - payload.extend(value_bytes) - return hashlib.sha256(payload).hexdigest() - - -def _terminal_error_summary(payload: dict[str, Any]) -> dict[str, object]: - error = payload.get("error") - if error is None: - return {"present": False, "class": "none", "code_digest": None, "message_digest": None} - if isinstance(error, dict): - code = error.get("code") - message = error.get("message") - return { - "present": True, - "class": "object", - "code_digest": _domain_hash(ERROR_CODE_DOMAIN, code) if code is not None else None, - "message_digest": _domain_hash(ERROR_MESSAGE_DOMAIN, message) if message is not None else None, - } - return { - "present": True, - "class": type(error).__name__, - "code_digest": None, - "message_digest": _domain_hash(ERROR_MESSAGE_DOMAIN, error), - } - - -def generate_evidence(snapshot: Snapshot, expected_task_id: str) -> dict[str, object]: - if not snapshot.data or not snapshot.data.endswith(b"\n"): - raise EvidenceError("jsonl_missing_complete_newline") - records: list[tuple[int, dict[str, Any]]] = [] - for line_number, line in enumerate(snapshot.data.splitlines(), 1): - if not line: - continue - try: - record = json.loads(line) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise EvidenceError("jsonl_parse_failed") from exc - if not isinstance(record, dict): - raise EvidenceError("jsonl_record_not_object") - records.append((line_number, record)) - - session_meta = [ - record["payload"] - for _, record in records - if record.get("type") == "session_meta" and isinstance(record.get("payload"), dict) - ] - if len(session_meta) != 1 or session_meta[0].get("id") != expected_task_id: - raise EvidenceError("session_meta_task_mismatch") - if isinstance(session_meta[0].get("source"), dict): - raise EvidenceError("non_root_session_source_unsupported") - - # The exact supported client emits root session_id/thread_id equal to the - # session_meta id. Require every persisted occurrence to agree. - identity_values: dict[str, list[str]] = {"session_id": [], "thread_id": []} - - def collect(value: object) -> None: - if isinstance(value, dict): - for key, child in value.items(): - if key in identity_values and isinstance(child, str): - identity_values[key].append(child) - collect(child) - elif isinstance(value, list): - for child in value: - collect(child) - - for _, record in records: - collect(record) - if any(not values or any(value != expected_task_id for value in values) for values in identity_values.values()): - raise EvidenceError("root_transport_identity_not_derivable") - - entries: list[dict[str, object]] = [] - pending: dict[str, str] = {} - seen_calls: set[str] = set() - seen_outputs: set[str] = set() - violations: Counter[str] = Counter() - last_started_line: int | None = None - last_complete_line: int | None = None - last_complete_payload: dict[str, Any] | None = None - latest_turn_counts: Counter[str] = Counter() - - for line_number, record in records: - payload_value = record.get("payload") - payload: dict[str, Any] = payload_value if isinstance(payload_value, dict) else {} - item_type = payload.get("type") - if item_type == "task_started": - last_started_line = line_number - latest_turn_counts.clear() - elif item_type == "task_complete": - last_complete_line = line_number - last_complete_payload = payload - elif last_started_line is not None and (last_complete_line is None or line_number > last_complete_line): - if record.get("type") == "response_item": - if item_type in {"message", "agent_message"} and payload.get("role") != "user": - latest_turn_counts["assistant_outputs"] += 1 - kind = _event_kind(item_type) - if kind == "call": - latest_turn_counts["tool_calls"] += 1 - elif kind == "output": - latest_turn_counts["tool_outputs"] += 1 - - if record.get("type") != "response_item": - continue - kind = _event_kind(item_type) - if kind is None: - continue - raw_id = _correlation_id(payload) - id_digest = _domain_hash(ID_DOMAIN, raw_id) if raw_id is not None else None - identity = { - "type": item_type, - "name": payload.get("name") if isinstance(payload.get("name"), str) else None, - "namespace": payload.get("namespace") if isinstance(payload.get("namespace"), str) else None, - } - if kind == "call": - arguments = ( - {"source": "arguments", "value": payload["arguments"]} - if "arguments" in payload - else {"source": "input", "value": payload["input"]} - if "input" in payload - else {"source": "missing"} - ) - output: object = {"source": "not_applicable"} - else: - arguments = {"source": "not_applicable"} - output = {"source": "output", "value": payload["output"]} if "output" in payload else {"source": "missing"} - entries.append( - { - "ordinal": len(entries) + 1, - "jsonl_line": line_number, - "kind": kind, - "item_type": item_type, - "id_digest": id_digest, - "tool_identity_digest": _domain_hash(IDENTITY_DOMAIN, identity), - "arguments_digest": _domain_hash(ARGUMENTS_DOMAIN, arguments), - "output_digest": _domain_hash(OUTPUT_DOMAIN, output), - "payload_without_id_digest": _domain_hash( - PAYLOAD_DOMAIN, {key: value for key, value in payload.items() if key not in {"call_id", "id"}} - ), - } - ) - if raw_id is None: - violations["missing_id_events"] += 1 - elif kind == "call": - if raw_id in seen_calls: - violations["duplicate_call_ids"] += 1 - else: - seen_calls.add(raw_id) - pending[raw_id] = str(item_type) - elif raw_id in seen_outputs: - violations["duplicate_outputs"] += 1 - else: - seen_outputs.add(raw_id) - call_type = pending.pop(raw_id, None) - if call_type is None: - violations["orphan_outputs"] += 1 - elif f"{call_type}_output" != item_type: - violations["type_mismatches"] += 1 - - violations["pending_calls"] = len(pending) - unresolved_count = sum(violations[key] for key in _VIOLATIONS) - if unresolved_count: - raise EvidenceError("tool_ledger_unresolved") - if last_started_line is None or last_complete_line is None or last_complete_line <= last_started_line: - raise EvidenceError("latest_turn_not_terminal") - post_terminal = [ - (record.get("type"), (record.get("payload") or {}).get("type")) - for line_number, record in records - if line_number > last_complete_line - ] - if any(item != ("event_msg", "item_completed") for item in post_terminal): - raise EvidenceError("unsupported_post_terminal_event") - - ledger_payload = {"schema": LEDGER_SCHEMA, "entries": entries} - task_authority_digest = _task_authority_digest(expected_task_id) - strong_session_hash = hashlib.sha256( - _canonical({"kind": "task_authority", "value": task_authority_digest}) - ).hexdigest() - terminal_error = _terminal_error_summary(last_complete_payload or {}) - evidence: dict[str, object] = { - "schema": SCHEMA, - "content_free": True, - "server_challenge_fields_included": False, - "remote_session_jsonl_sha256": snapshot.sha256, - "remote_session_jsonl_size_bytes": snapshot.size, - "remote_session_jsonl_last_offset": snapshot.size, - "remote_session_jsonl_line_count": len(records), - "task_identity": expected_task_id, - "session_identity": expected_task_id, - "task_authority_digest": task_authority_digest, - "strong_session_hash": strong_session_hash, - "full_checkpoint_tool_ledger_digest": hashlib.sha256(LEDGER_DOMAIN + _canonical(ledger_payload)).hexdigest(), - "full_checkpoint_tool_ledger_event_count": len(entries), - "unresolved_count": unresolved_count, - "ledger_validation": {key: violations[key] for key in _VIOLATIONS}, - "terminal": { - "last_task_started_line": last_started_line, - "last_task_complete_line": last_complete_line, - "post_terminal_item_completed_count": len(post_terminal), - "error_terminal": terminal_error["present"], - "error": terminal_error, - "latest_turn_assistant_output_count": latest_turn_counts["assistant_outputs"], - "latest_turn_tool_call_count": latest_turn_counts["tool_calls"], - "latest_turn_tool_output_count": latest_turn_counts["tool_outputs"], - }, - "transport_identity": { - "session_header": "session-id", - "thread_header": "thread-id", - "prompt_cache_key_source": "session_id_default", - }, - } - return evidence - - -def _create_new(path: Path, payload: bytes) -> None: - path.parent.resolve(strict=True) - if path.exists(): - raise EvidenceError("output_exists") - temp = path.with_name(f".{path.name}.{secrets.token_hex(8)}.tmp") - fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0), 0o600) - try: - written = 0 - while written < len(payload): - count = os.write(fd, payload[written:]) - if count <= 0: - raise EvidenceError("output_short_write") - written += count - os.fsync(fd) - finally: - os.close(fd) - try: - os.link(temp, path) - except FileExistsError as exc: - raise EvidenceError("output_exists") from exc - finally: - temp.unlink(missing_ok=True) - dir_fd = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) - try: - os.fsync(dir_fd) - finally: - os.close(dir_fd) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--jsonl", required=True, type=Path) - parser.add_argument("--task-id", required=True) - parser.add_argument("--output", required=True, type=Path) - parser.add_argument("--stability-delay-ms", type=int, default=250) - args = parser.parse_args(argv) - try: - if args.stability_delay_ms < 0: - raise EvidenceError("invalid_stability_delay") - evidence = generate_evidence(stable_snapshot(args.jsonl, args.stability_delay_ms / 1000), args.task_id) - encoded = _canonical(evidence) + b"\n" - _create_new(args.output, encoded) - print(f"evidence_path={args.output}") - print(f"evidence_sha256={hashlib.sha256(encoded).hexdigest()}") - return 0 - except EvidenceError as exc: - print(f"ERROR {exc.code}", file=sys.stderr) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/generate_settings_reference.py b/scripts/generate_settings_reference.py index d3c23c5222..9ad7fb6519 100644 --- a/scripts/generate_settings_reference.py +++ b/scripts/generate_settings_reference.py @@ -260,8 +260,14 @@ def render_settings_reference() -> str: "", "*Specs: [user-documentation]" "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/user-documentation) · " + "[responses-api-compat]" + "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/responses-api-compat) · " + "[rate-limit-reset-credits]" + "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/rate-limit-reset-credits) · " "[deployment-installation]" - "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation)*", + "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation) · " + "[proxy-runtime-observability]" + "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/proxy-runtime-observability)*", "", ] ) diff --git a/tests/conftest.py b/tests/conftest.py index a886d66e92..7d96d60c1e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import os import tempfile from pathlib import Path @@ -142,6 +143,17 @@ def _disable_request_log_count_cache(monkeypatch): monkeypatch.setattr(logs_repository_module, "_COUNT_CACHE_TTL_SECONDS", 0.0) +@pytest.fixture(autouse=True) +def _disable_account_usage_summary_cache(monkeypatch): + """Zero the account request-usage summary cache TTL so listing summaries + stay exact within a test. The TTL is a fixed constant in production; + cache-behavior tests patch it back to a positive value.""" + import app.modules.accounts.repository as accounts_repository_module + + accounts_repository_module._clear_request_usage_summary_cache() + monkeypatch.setattr(accounts_repository_module, "_SUMMARY_CACHE_TTL_SECONDS", 0.0) + + @pytest.fixture(autouse=True) def _disable_rate_limit_reset_credits_scheduler_startup(monkeypatch): import app.main as main_module @@ -163,6 +175,13 @@ def _disable_data_retention_scheduler_startup(monkeypatch): monkeypatch.setattr(main_module, "build_data_retention_scheduler", lambda: _NoopScheduler()) +@pytest.fixture(autouse=True) +def _disable_telemetry_scheduler_startup(monkeypatch): + import app.main as main_module + + monkeypatch.setattr(main_module, "build_telemetry_scheduler", lambda: _NoopScheduler()) + + @pytest.fixture(autouse=True) def _disable_leader_election_startup(monkeypatch): """Replace the ambient app-lifespan leader election with a no-op. @@ -385,3 +404,162 @@ def _reset_shutdown_task_admission(): shutdown_state.reset() yield shutdown_state.reset() + + +_SESSION_LOOP: asyncio.AbstractEventLoop | None = None + +# Both task names the live-usage ingestor owns (consumer and throttled +# trailing cache invalidation); the fence below reclaims them by name when the +# singleton no longer tracks them. +_LIVE_INGEST_TASK_NAMES = ("live-usage-ingestor", "live-usage-trailing-invalidation") + + +def _pending_live_ingest_tasks(loop: asyncio.AbstractEventLoop) -> list[asyncio.Task]: + return [task for task in asyncio.all_tasks(loop) if not task.done() and task.get_name() in _LIVE_INGEST_TASK_NAMES] + + +@pytest_asyncio.fixture(scope="session", autouse=True) +async def _capture_session_loop(): + """Expose the shared session loop to sync fixture teardowns. + + The live-usage ingestor fence below must run coroutine cleanup from a + synchronous teardown (see its docstring for why it cannot be an async + fixture), and pytest-asyncio has no public API to reach the session loop + from sync code. + """ + global _SESSION_LOOP + _SESSION_LOOP = asyncio.get_running_loop() + yield + _SESSION_LOOP = None + + +async def _reap_leaked_live_usage_ingestor() -> None: + """Stop and reset the live-usage ingestor singleton. + + Mirrors ``stop_live_usage_ingestor()`` but never re-raises: every awaited + task ends up done, and ``_consume_dead_live_ingest_task_failures`` then + retrieves and reports its exception exactly once. Also sweeps by name for + ingestor-owned tasks (consumer and trailing invalidation) the stop path no + longer tracks — a stop that was itself cancelled between clearing the + global and awaiting the tasks. + + Only tasks bound to the loop this coroutine runs on are cancelled and + awaited. A leaked singleton can hold tasks that belong to a different + loop entirely — integration tests run ``TestClient`` portals whose loop + is a private per-portal loop that is already closed by teardown time. + Cancelling such a task raises ``RuntimeError('Event loop is closed')`` + from ``call_soon`` and awaiting it raises the cross-loop RuntimeError; + neither can ever reap it. Those tasks are inert (a closed loop never + steps again), so they are enrolled for exception accounting and left + alone. + """ + from app.core.usage.live_hub import register_live_usage_publisher + from app.modules.usage import live_ingest + + ingestors: list[live_ingest.LiveUsageIngestor] = [] + if live_ingest._ingestor is not None: + ingestors.append(live_ingest._ingestor) + live_ingest._ingestor = None + # Displaced (nested-over) registrations hold live tasks too, and a stale + # stack entry must never be restored into a later test. + ingestors.extend(live_ingest._displaced_ingestors) + live_ingest._displaced_ingestors.clear() + register_live_usage_publisher(None) + leaked: list[asyncio.Task[None]] = [] + for ingestor in ingestors: + for task in (ingestor._consumer, ingestor._trailing_invalidation): + if task is not None and task not in leaked: + leaked.append(task) + ingestor._consumer = None + ingestor._trailing_invalidation = None + loop = asyncio.get_running_loop() + for task in _pending_live_ingest_tasks(loop): + if task not in leaked: + leaked.append(task) + reapable: list[asyncio.Task[None]] = [] + for task in leaked: + live_ingest._owned_tasks.add(task) + if task.get_loop() is loop: + reapable.append(task) + for task in reapable: + task.cancel() + for task in reapable: + try: + await task + except (Exception, asyncio.CancelledError): + # Settled and reported by _drain_live_ingest_task_failures. + continue + + +def _drain_live_ingest_task_failures() -> list[str]: + """Collect failures from dead ingestor-owned tasks, loop-free. + + ``asyncio.all_tasks`` only returns unfinished tasks, so a leaked task that + already died with an exception is invisible to the pending sweep; its + unretrieved exception would otherwise fire the loop exception handler when + the task object is garbage-collected inside a LATER test (test_proxy_utils' + startup-probe assertions capture exactly that). live_ingest's done + callback normally settles each task the moment it completes (retrieving + the exception into the strong ``_owned_task_failures`` handoff); the sweep + over the weak registry here additionally settles tasks whose callback is + still queued because the task finished in the loop's final iteration. + Settlement is gated by live_ingest's settled-task registry, so each task + is reported exactly once even when both paths observe it. + """ + from app.modules.usage import live_ingest + + for task in list(live_ingest._owned_tasks): + if task.done(): + live_ingest._record_owned_task_result(task) + failures = [f"{name!r} died with {exc_repr}" for name, exc_repr in live_ingest._owned_task_failures] + live_ingest._owned_task_failures.clear() + return failures + + +@pytest.fixture(autouse=True) +def _stop_leaked_live_usage_ingestor(): + """Fence the module-global live-usage ingestor per test (issue #1755). + + The suite runs on a session-scoped asyncio loop, so a task leaked by one + test survives into every later test. Any test that enters the real app + lifespan starts the live-usage ingestor singleton + (``app.modules.usage.live_ingest._ingestor``) whose ``live-usage-ingestor`` + consumer task lands on that shared loop; if the lifespan is cancelled + before its shutdown path reaches ``stop_live_usage_ingestor()`` (e.g. a + ``wait_for``-bounded assertion times out mid-drain), the consumer outlives + the test. The zombie then poisons unrelated tests: it eats into the otel + lifespan test's drain budget and surfaces as an unobserved-task exception + inside test_proxy_utils' startup-probe loop-exception assertions — the + exact failing pairing from #1755. Stop and reset the singleton after every + test so no ingestor task ever crosses a test boundary. + + Deliberately a sync fixture that only enters the event loop when a leak is + actually present: an async fixture's teardown would spin the shared loop + after EVERY test, and the loop's clock calls ``time.monotonic()`` — which + several tests monkeypatch globally with finite or call-count-sensitive + fakes that are still active while function-scoped teardowns run (e.g. + test_conversation_archive's exhausting iterator). Leak detection itself is + loop-passive: reading the module globals, enumerating + ``asyncio.all_tasks(loop)`` on the idle session loop, and retrieving + exceptions from already-dead owned tasks never runs the loop. + """ + yield + from app.core.usage import live_hub + from app.modules.usage import live_ingest + + loop = _SESSION_LOOP + loop_usable = loop is not None and not loop.is_closed() and not loop.is_running() + needs_reap = ( + live_ingest._ingestor is not None + or bool(live_ingest._displaced_ingestors) + or live_hub._publisher is not None + or (loop_usable and loop is not None and _pending_live_ingest_tasks(loop)) + ) + if needs_reap and loop_usable and loop is not None: + loop.run_until_complete(_reap_leaked_live_usage_ingestor()) + failures = _drain_live_ingest_task_failures() + if failures: + pytest.fail( + "test leaked a live-usage ingestor whose task(s) already failed: " + "; ".join(failures), + pytrace=False, + ) diff --git a/tests/e2e/test_codex_daybreak_profile.py b/tests/e2e/test_codex_daybreak_profile.py new file mode 100644 index 0000000000..d7c53143f8 --- /dev/null +++ b/tests/e2e/test_codex_daybreak_profile.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +import pytest + +pytestmark = pytest.mark.e2e + +_RUN_E2E = os.environ.get("CODEX_LB_RUN_CODEX_PROFILE_E2E") == "1" +_ROOT = Path(__file__).resolve().parents[2] +_BASE_CONFIG = _ROOT / "docs/examples/codex/config.toml" +_PROFILE = _ROOT / "docs/examples/codex/daybreak-blue.config.toml" +_API_KEY = "inert-daybreak-profile-key" +_CapturedRequest = tuple[str, str, dict[str, str]] + + +class _Server(ThreadingHTTPServer): + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), _Handler) + self.requests: list[_CapturedRequest] = [] + + +class _Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def _capture(self) -> None: + assert isinstance(self.server, _Server) + self.server.requests.append( + (self.command, urlsplit(self.path).path, {name.lower(): value for name, value in self.headers.items()}) + ) + + def _respond(self, status: int, body: bytes = b"") -> None: + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + self.send_header("Connection", "close") + if body: + self.send_header("Content-Type", "application/json") + self.end_headers() + if body: + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + self._capture() + self._respond(503) + + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + if length := int(self.headers.get("content-length", "0")): + self.rfile.read(length) + self._capture() + self._respond(400, b'{"error":{"code":"inert_probe_complete","message":"inert probe complete"}}') + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 - stdlib override + del format, args + + +def _write_config(codex_home: Path, port: int) -> None: + config = _BASE_CONFIG.read_text(encoding="utf-8") + for provider, path in (("codex-lb", "ordinary"), ("codex-lb-daybreak-blue", "daybreak")): + section = f"[model_providers.{provider}]" + section_start = config.index(section) + base_start = config.index('base_url = "', section_start) + base_end = config.index("\n", base_start) + config = config[:base_start] + f'base_url = "http://127.0.0.1:{port}/{path}"' + config[base_end:] + codex_home.mkdir(parents=True) + (codex_home / "config.toml").write_text(config, encoding="utf-8") + shutil.copyfile(_PROFILE, codex_home / "daybreak-blue.config.toml") + + +def _run_codex( + codex: str, sandbox_exec: str, root: Path, port: int, *, include_key: bool +) -> subprocess.CompletedProcess[str]: + codex_home = root / "codex-home" + _write_config(codex_home, port) + env = { + "CODEX_HOME": str(codex_home), + "HOME": str(root), + "LANG": "C.UTF-8", + "NO_PROXY": "127.0.0.1,localhost", + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "SHELL": "/bin/sh", + "TERM": "dumb", + "TMPDIR": str(root), + "USER": "codex-profile-e2e", + } + if include_key: + env["CODEX_LB_API_KEY"] = _API_KEY + policy = f'(version 1)(allow default)(deny network-outbound)(allow network-outbound (remote ip "localhost:{port}"))' + command = [ + sandbox_exec, + "-p", + policy, + codex, + "exec", + "--strict-config", + "--profile", + "daybreak-blue", + "--skip-git-repo-check", + "--ephemeral", + "--ignore-rules", + "-C", + str(root), + "Reply with OK only.", + ] + return subprocess.run(command, cwd=root, env=env, capture_output=True, text=True, timeout=45, check=False) + + +@pytest.mark.skipif(not _RUN_E2E, reason="set CODEX_LB_RUN_CODEX_PROFILE_E2E=1 for the installed-Codex proof") +def test_installed_codex_daybreak_profile_emits_authenticated_capability_before_fallback(tmp_path: Path) -> None: + if sys.platform != "darwin": + pytest.skip("the network-deny harness requires macOS sandbox-exec") + codex, sandbox_exec = shutil.which("codex"), shutil.which("sandbox-exec") + if codex is None or sandbox_exec is None: + pytest.skip("installed codex and sandbox-exec are required") + + server = _Server() + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + port = server.server_address[1] + result = _run_codex(codex, sandbox_exec, tmp_path / "configured", port, include_key=True) + assert server.requests, result.stdout + result.stderr + method, path, headers = server.requests[0] + assert (method, path, headers["upgrade"].lower()) == ("GET", "/daybreak/responses", "websocket") + assert headers["authorization"] == f"Bearer {_API_KEY}" + assert headers["x-codex-lb-required-capability"] == "trusted_cyber" + + fallbacks = [(path, headers) for method, path, headers in server.requests if method == "POST"] + assert fallbacks, result.stdout + result.stderr + assert all(path == "/daybreak/responses" for path, _headers in fallbacks) + assert all(headers["authorization"] == f"Bearer {_API_KEY}" for _path, headers in fallbacks) + assert all(headers["x-codex-lb-required-capability"] == "trusted_cyber" for _path, headers in fallbacks) + + server.requests.clear() + missing = _run_codex(codex, sandbox_exec, tmp_path / "missing-key", port, include_key=False) + output = missing.stdout + missing.stderr + assert missing.returncode != 0 + assert "Missing environment variable" in output and "CODEX_LB_API_KEY" in output + assert server.requests == [] + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/tests/e2e/test_openai_sdk_compat.py b/tests/e2e/test_openai_sdk_compat.py index daee81f098..23d1584511 100644 --- a/tests/e2e/test_openai_sdk_compat.py +++ b/tests/e2e/test_openai_sdk_compat.py @@ -311,14 +311,17 @@ async def sdk_client( if hasattr(result, "__await__"): await result - transport = e2e_client._transport # noqa: SLF001 import httpx + import httpx2 + + transport = e2e_client._transport # noqa: SLF001 + assert isinstance(transport, httpx.ASGITransport) client = openai.AsyncOpenAI( api_key=created["key"], base_url="http://testserver/v1", - http_client=httpx.AsyncClient( - transport=transport, + http_client=httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=transport.app), base_url="http://testserver", ), ) diff --git a/tests/e2e/test_v1_responses_openai_sdk.py b/tests/e2e/test_v1_responses_openai_sdk.py index 0a582a65b1..2f54c8d47d 100644 --- a/tests/e2e/test_v1_responses_openai_sdk.py +++ b/tests/e2e/test_v1_responses_openai_sdk.py @@ -327,12 +327,15 @@ async def sdk_client( result = registry.update(snapshot) if hasattr(result, "__await__"): await result - # Reuse the same ASGITransport that e2e_client built. + # Reuse the same ASGI app that e2e_client built, wrapped for httpx2. + httpx = __import__("httpx") + httpx2 = __import__("httpx2") transport = e2e_client._transport # noqa: SLF001 + assert isinstance(transport, httpx.ASGITransport) client = openai.AsyncOpenAI( api_key=created["key"], base_url="http://testserver/v1", - http_client=__import__("httpx").AsyncClient(transport=transport, base_url="http://testserver"), + http_client=httpx2.AsyncClient(transport=httpx2.ASGITransport(app=transport.app), base_url="http://testserver"), ) yield client await client.close() diff --git a/tests/fixtures/codex_client_evidence/sanitized_terminal.jsonl b/tests/fixtures/codex_client_evidence/sanitized_terminal.jsonl deleted file mode 100644 index 741ea65cd2..0000000000 --- a/tests/fixtures/codex_client_evidence/sanitized_terminal.jsonl +++ /dev/null @@ -1,9 +0,0 @@ -{"type":"session_meta","payload":{"id":"01a00000-0000-7000-8000-000000000001","source":"appServer","session_id":"01a00000-0000-7000-8000-000000000001"}} -{"type":"turn_context","payload":{"type":"turn_context","thread_id":"01a00000-0000-7000-8000-000000000001"}} -{"type":"event_msg","payload":{"type":"task_started"}} -{"type":"response_item","payload":{"type":"message","role":"user","content":"SANITIZED_USER_TEXT"}} -{"type":"response_item","payload":{"type":"message","role":"assistant","content":"SANITIZED_ASSISTANT_TEXT"}} -{"type":"response_item","payload":{"type":"custom_tool_call","call_id":"RAW_CALL_ID_MUST_NOT_ESCAPE","name":"sanitized_tool","input":"RAW_ARGUMENT_MUST_NOT_ESCAPE"}} -{"type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"RAW_CALL_ID_MUST_NOT_ESCAPE","output":"RAW_OUTPUT_MUST_NOT_ESCAPE"}} -{"type":"event_msg","payload":{"type":"task_complete","error":null}} -{"type":"event_msg","payload":{"type":"item_completed"}} diff --git a/tests/fixtures/http_responses/abandoned_pending_real_transport_shape_v1.json b/tests/fixtures/http_responses/abandoned_pending_real_transport_shape_v1.json deleted file mode 100644 index a872cd30fe..0000000000 --- a/tests/fixtures/http_responses/abandoned_pending_real_transport_shape_v1.json +++ /dev/null @@ -1,242 +0,0 @@ -{ - "pending_tool_calls": { - "count": 1, - "types": [ - "custom_tool_call" - ] - }, - "provenance": { - "contains_call_ids": false, - "contains_credentials": false, - "contains_message_text": false, - "contains_raw_ids": false, - "sanitized": true, - "source_kind": "production_request_structure" - }, - "request": { - "item_count": 143, - "item_shape_catalog": [ - { - "call_id_present": false, - "content_count": null, - "content_types": null, - "extra_key_count": 0, - "id_kind": "none", - "known_keys": ["role", "tools", "type"], - "meta_keys": null, - "phase": null, - "role": "developer", - "status": null, - "type": "additional_tools" - }, - { - "call_id_present": false, - "content_count": 1, - "content_types": ["input_text"], - "extra_key_count": 0, - "id_kind": "none", - "known_keys": ["content", "role", "type"], - "meta_keys": null, - "phase": null, - "role": "developer", - "status": null, - "type": "message" - }, - { - "call_id_present": false, - "content_count": 1, - "content_types": ["input_text"], - "extra_key_count": 0, - "id_kind": "msg", - "known_keys": ["content", "id", "role", "type"], - "meta_keys": null, - "phase": null, - "role": "user", - "status": null, - "type": "message" - }, - { - "call_id_present": false, - "content_count": 4, - "content_types": ["input_text", "input_text", "input_text", "input_text"], - "extra_key_count": 0, - "id_kind": "msg", - "known_keys": ["content", "id", "role", "type"], - "meta_keys": null, - "phase": null, - "role": "developer", - "status": null, - "type": "message" - }, - { - "call_id_present": false, - "content_count": 1, - "content_types": ["input_text"], - "extra_key_count": 0, - "id_kind": "msg", - "known_keys": ["content", "id", "role", "type"], - "meta_keys": null, - "phase": null, - "role": "developer", - "status": null, - "type": "message" - }, - { - "call_id_present": false, - "content_count": 2, - "content_types": ["input_text", "input_text"], - "extra_key_count": 0, - "id_kind": "msg", - "known_keys": ["content", "id", "role", "type"], - "meta_keys": null, - "phase": null, - "role": "user", - "status": null, - "type": "message" - }, - { - "call_id_present": false, - "content_count": null, - "content_types": null, - "extra_key_count": 0, - "id_kind": "reasoning", - "known_keys": ["content", "encrypted_content", "id", "summary", "type"], - "meta_keys": null, - "phase": null, - "role": null, - "status": null, - "type": "reasoning" - }, - { - "call_id_present": true, - "content_count": null, - "content_types": null, - "extra_key_count": 0, - "id_kind": "other", - "known_keys": ["arguments", "call_id", "id", "name", "namespace", "type"], - "meta_keys": null, - "phase": null, - "role": null, - "status": null, - "type": "function_call" - }, - { - "call_id_present": true, - "content_count": null, - "content_types": null, - "extra_key_count": 0, - "id_kind": "other", - "known_keys": ["call_id", "id", "output", "type"], - "meta_keys": null, - "phase": null, - "role": null, - "status": null, - "type": "function_call_output" - }, - { - "call_id_present": true, - "content_count": null, - "content_types": null, - "extra_key_count": 0, - "id_kind": "other", - "known_keys": ["call_id", "id", "input", "name", "status", "type"], - "meta_keys": null, - "phase": null, - "role": null, - "status": "completed", - "type": "custom_tool_call" - }, - { - "call_id_present": true, - "content_count": null, - "content_types": null, - "extra_key_count": 0, - "id_kind": "other", - "known_keys": ["call_id", "id", "output", "type"], - "meta_keys": null, - "phase": null, - "role": null, - "status": null, - "type": "custom_tool_call_output" - }, - { - "call_id_present": false, - "content_count": 1, - "content_types": ["output_text"], - "extra_key_count": 0, - "id_kind": "msg", - "known_keys": ["content", "id", "phase", "role", "type"], - "meta_keys": null, - "phase": "final_answer", - "role": "assistant", - "status": null, - "type": "message" - }, - { - "call_id_present": false, - "content_count": 1, - "content_types": ["output_text"], - "extra_key_count": 0, - "id_kind": "msg", - "known_keys": ["content", "id", "phase", "role", "type"], - "meta_keys": null, - "phase": "commentary", - "role": "assistant", - "status": null, - "type": "message" - }, - { - "call_id_present": false, - "content_count": 1, - "content_types": ["input_text"], - "extra_key_count": 0, - "id_kind": "agent", - "known_keys": ["author", "content", "id", "recipient", "type"], - "meta_keys": null, - "phase": null, - "role": null, - "status": null, - "type": "agent_message" - }, - { - "call_id_present": false, - "content_count": 3, - "content_types": ["input_text", "input_text", "input_text"], - "extra_key_count": 0, - "id_kind": "msg", - "known_keys": ["content", "id", "role", "type"], - "meta_keys": null, - "phase": null, - "role": "developer", - "status": null, - "type": "message" - } - ], - "item_shape_sequence": [ - 0, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 3, 4, 4, 5, 2, 2, 6, 7, 8, 6, 9, 10, 6, 9, 10, 6, 9, 10, 6, 9, 10, - 6, 6, 6, 9, 10, 6, 9, 10, 6, 6, 9, 10, 6, 9, 10, 6, 9, 10, 6, 9, 10, - 6, 9, 10, 6, 9, 10, 6, 9, 10, 6, 9, 10, 6, 9, 10, 6, 9, 10, 6, 11, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 6, 12, 9, 10, 6, 9, 10, 6, 7, 8, 6, 9, - 10, 6, 9, 10, 6, 9, 10, 6, 13, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 14, 2, 2 - ], - "stored_count": 111, - "top_keys": [ - "client_metadata", - "include", - "input", - "model", - "parallel_tool_calls", - "prompt_cache_key", - "reasoning", - "store", - "stream", - "text", - "tool_choice" - ] - }, - "schema": "qk_http_responses_sanitized_shape_fixture_v1" -} diff --git a/tests/fixtures/http_responses/pending_settlement_real_transport_shapes_v1.json b/tests/fixtures/http_responses/pending_settlement_real_transport_shapes_v1.json deleted file mode 100644 index 27cf32dd5e..0000000000 --- a/tests/fixtures/http_responses/pending_settlement_real_transport_shapes_v1.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "cases": [ - { - "case_id": "stored_85_custom_call_followup", - "pending_type": "custom_tool_call", - "stored_count": 85, - "suffix_shape": ["reasoning", "custom_tool_call", "custom_tool_call_output", "user_message"] - }, - { - "case_id": "stored_288_function_call_agent_followup", - "pending_type": "function_call", - "stored_count": 288, - "suffix_shape": ["reasoning", "function_call", "function_call_output", "agent_message", "user_message"] - }, - { - "case_id": "stored_194_custom_call_terminal", - "pending_type": "custom_tool_call", - "stored_count": 194, - "suffix_shape": ["reasoning", "custom_tool_call", "custom_tool_call_output"] - } - ], - "provenance": { - "contains_call_ids": false, - "contains_credentials": false, - "contains_message_text": false, - "contains_raw_ids": false, - "sanitized": true, - "source_kind": "production_request_structure" - }, - "schema": "qk_http_responses_pending_settlement_shape_fixture_v1" -} diff --git a/tests/integration/compact_test_helpers.py b/tests/integration/compact_test_helpers.py new file mode 100644 index 0000000000..5fecf22561 --- /dev/null +++ b/tests/integration/compact_test_helpers.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import base64 +import json + + +def _encode_jwt(payload: dict) -> str: + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + body = base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") + return f"header.{body}.sig" + + +def _make_auth_json(account_id: str, email: str, *, plan_type: str = "plus") -> dict: + payload = { + "email": email, + "chatgpt_account_id": account_id, + "https://api.openai.com/auth": {"chatgpt_plan_type": plan_type}, + } + return { + "tokens": { + "idToken": _encode_jwt(payload), + "accessToken": "access-token", + "refreshToken": "refresh-token", + "accountId": account_id, + }, + } diff --git a/tests/integration/test_account_deletion_background.py b/tests/integration/test_account_deletion_background.py new file mode 100644 index 0000000000..aa08fb2f76 --- /dev/null +++ b/tests/integration/test_account_deletion_background.py @@ -0,0 +1,1033 @@ +"""Background (chunked) account deletion: drain, fold interleave, restart, +idempotency, and supersede semantics for both delete_history variants.""" + +from __future__ import annotations + +import base64 +import json +from collections.abc import Callable +from datetime import timedelta +from typing import cast + +import pytest +from sqlalchemy import Table, func, select, text, update +from sqlalchemy.sql import Select + +from app.core.crypto import TokenEncryptor +from app.core.utils.time import utcnow +from app.db.models import ( + Account, + AccountStatus, + AccountUsageRollup, + RequestDemandQuarterRollup, + RequestLog, + RequestUsageHourlyRollup, + StickySession, + StickySessionKind, + UsageHistory, +) +from app.db.session import SessionLocal, get_background_session, sqlite_writer_section +from app.modules.accounts.deletion import _request_logs_chunk, run_account_deletion_pass +from app.modules.accounts.repository import ACCOUNT_PENDING_DELETION_REASON, AccountsRepository +from app.modules.accounts.usage_rollup import run_fold_pass +from app.modules.accounts.usage_time_rollup import run_hourly_fold_pass, to_dimension +from app.modules.request_logs.repository import RequestLogsRepository +from app.modules.usage.repository import UsageRepository + +pytestmark = pytest.mark.integration + +_ORPHAN_DIMENSION = to_dimension(None) + + +@pytest.fixture(autouse=True) +def _no_background_wake(monkeypatch): + """Keep the drain under explicit test control. + + The suite's stand-in leader election runs scheduler bodies inline, so the + delete API's worker wake would drain accounts concurrently with (and race) + the passes these tests drive step by step. The scheduler's own tick (one + pass at startup plus every interval) is neutralized as well: a tick firing + between a DELETE and the assertions would drain the account these tests + expect to still be marked. + """ + monkeypatch.setattr("app.modules.accounts.service.request_account_deletion_run", lambda: None) + + async def _no_tick(self) -> None: + return None + + monkeypatch.setattr("app.modules.accounts.deletion.AccountDeletionScheduler._run_once", _no_tick) + + +def _make_account(account_id: str, email: str) -> Account: + encryptor = TokenEncryptor() + return Account( + id=account_id, + email=email, + plan_type="plus", + access_token_encrypted=encryptor.encrypt("access"), + refresh_token_encrypted=encryptor.encrypt("refresh"), + id_token_encrypted=encryptor.encrypt("id"), + last_refresh=utcnow(), + status=AccountStatus.ACTIVE, + deactivation_reason=None, + ) + + +async def _add_log(logs_repo: RequestLogsRepository, *, account_id: str, request_id: str, requested_at) -> None: + await logs_repo.add_log( + account_id=account_id, + request_id=request_id, + model="gpt-5.1-codex", + input_tokens=100, + output_tokens=50, + latency_ms=100, + status="success", + error_code=None, + requested_at=requested_at, + cost_usd=0.01, + ) + + +async def _seed_account(account_id: str, *, log_count: int, usage_count: int = 0, requested_at=None) -> None: + requested_at = requested_at or (utcnow() - timedelta(days=2)) + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + usage_repo = UsageRepository(session) + await accounts_repo.upsert(_make_account(account_id, f"{account_id}@example.com")) + for index in range(log_count): + await _add_log( + logs_repo, + account_id=account_id, + request_id=f"req_{account_id}_{index}", + requested_at=requested_at, + ) + for index in range(usage_count): + await usage_repo.add_entry(account_id, float(index), window="primary") + + +async def _account_row(account_id: str) -> Account | None: + async with SessionLocal() as session: + return await session.get(Account, account_id) + + +async def _attached_log_count(account_id: str) -> int: + async with SessionLocal() as session: + return ( + await session.execute(select(func.count(RequestLog.id)).where(RequestLog.account_id == account_id)) + ).scalar_one() + + +async def _log_rows(prefix: str) -> list[RequestLog]: + async with SessionLocal() as session: + return list( + (await session.execute(select(RequestLog).where(RequestLog.request_id.like(f"req_{prefix}%")))) + .scalars() + .all() + ) + + +async def _hourly_rows_for_dimension(dimension: str) -> list[RequestUsageHourlyRollup]: + async with SessionLocal() as session: + return list( + ( + await session.execute( + select(RequestUsageHourlyRollup).where(RequestUsageHourlyRollup.account_id == dimension) + ) + ) + .scalars() + .all() + ) + + +async def _demand_rows_for_dimension(dimension: str) -> list[RequestDemandQuarterRollup]: + async with SessionLocal() as session: + return list( + ( + await session.execute( + select(RequestDemandQuarterRollup).where(RequestDemandQuarterRollup.account_id == dimension) + ) + ) + .scalars() + .all() + ) + + +async def _lifetime_rollup(account_id: str) -> AccountUsageRollup | None: + async with SessionLocal() as session: + return await session.get(AccountUsageRollup, account_id) + + +async def _run_one_detach_chunk(account_id: str, *, batch_size: int, delete_history: bool = False) -> int: + async with get_background_session() as session: + async with sqlite_writer_section(): + affected = await _request_logs_chunk( + session, account_id, delete_history=delete_history, batch_size=batch_size + ) + await session.commit() + return affected + + +@pytest.mark.asyncio +async def test_delete_api_marks_and_hides_immediately(async_client, db_setup): + await _seed_account("acc_bg_mark", log_count=2, usage_count=2) + + delete = await async_client.delete("/api/accounts/acc_bg_mark") + assert delete.status_code == 200 + assert delete.json()["status"] == "deleted" + + # Hidden from the listing immediately, before any background work ran. + accounts = await async_client.get("/api/accounts") + assert accounts.status_code == 200 + assert all(entry["accountId"] != "acc_bg_mark" for entry in accounts.json()["accounts"]) + + # The row itself survives, terminal and marked, until the worker drains it. + row = await _account_row("acc_bg_mark") + assert row is not None + assert row.status is AccountStatus.DEACTIVATED + assert row.deactivation_reason == ACCOUNT_PENDING_DELETION_REASON + assert row.delete_requested_at is not None + assert await _attached_log_count("acc_bg_mark") == 2 + + # Repeat request is idempotent and does not escalate the frozen variant. + repeat = await async_client.delete("/api/accounts/acc_bg_mark?delete_history=true") + assert repeat.status_code == 200 + row = await _account_row("acc_bg_mark") + assert row is not None + assert row.delete_history_requested is False + + # A marked account is gone from the operator's perspective: reactivation + # reports not-found instead of racing the deletion worker. + reactivate = await async_client.post("/api/accounts/acc_bg_mark/reactivate") + assert reactivate.status_code == 404 + + # Credential exports must not keep serving decrypted tokens during the + # drain window: the synchronous delete 404'd here immediately. + for export_path in ("export", "export/auth", "export/opencode-auth"): + export = await async_client.post(f"/api/accounts/acc_bg_mark/{export_path}") + assert export.status_code == 404, export_path + + # Every other ID-based account route treats the marked row as gone too — + # the synchronous delete returned 404 on all of them once the row was + # removed. + base = "/api/accounts/acc_bg_mark" + assert (await async_client.get(f"{base}/trends")).status_code == 404 + assert (await async_client.get(f"{base}/usage-reset-credits")).status_code == 404 + assert (await async_client.post(f"{base}/usage-reset-credits/consume")).status_code == 404 + assert (await async_client.post(f"{base}/probe")).status_code == 404 + assert (await async_client.post(f"{base}/pause")).status_code == 404 + assert (await async_client.patch(base, json={"securityWorkAuthorized": True})).status_code == 404 + assert (await async_client.put(f"{base}/alias", json={"alias": "ghost"})).status_code == 404 + assert (await async_client.put(f"{base}/limit-warmup", json={"enabled": True})).status_code == 404 + assert (await async_client.put(f"{base}/routing-policy", json={"routingPolicy": "preserve"})).status_code == 404 + + +def _fake_id_token(payload: dict) -> str: + encoded = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=") + return f"header.{encoded}.signature" + + +@pytest.mark.asyncio +async def test_begin_delete_preserves_seat_identity_before_token_wipe(db_setup): + """Legacy rows carry their seat identity only inside the id-token claims; + targeted reauthentication (a supersede path) verifies the seat against + chatgpt_user_id or those claims, so the wipe must backfill the non-secret + identity first.""" + encryptor = TokenEncryptor() + async with SessionLocal() as session: + account = _make_account("acc_bg_seat", "acc_bg_seat@example.com") + assert account.chatgpt_user_id is None + account.id_token_encrypted = encryptor.encrypt(_fake_id_token({"sub": "user-legacy-seat"})) + await AccountsRepository(session).upsert(account) + + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_seat") + + row = await _account_row("acc_bg_seat") + assert row is not None + # The ciphertext is wiped (no usable credentials remain on the row)... + assert encryptor.decrypt(row.id_token_encrypted) == "" + # ...but the seat identity survives in the non-secret column. + assert row.chatgpt_user_id == "user-legacy-seat" + + +@pytest.mark.asyncio +async def test_marked_account_wipes_tokens_and_rejects_stale_status_writes(db_setup): + await _seed_account("acc_bg_fence", log_count=1) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_fence") + + # The surviving row must not carry usable credentials: readers that do + # not know the marker (pre-upgrade replicas during a rolling deploy) can + # only produce empty credentials from it. + row = await _account_row("acc_bg_fence") + assert row is not None + encryptor = TokenEncryptor() + assert encryptor.decrypt(row.access_token_encrypted) == "" + assert encryptor.decrypt(row.refresh_token_encrypted) == "" + assert encryptor.decrypt(row.id_token_encrypted) == "" + + # Stale in-flight settlements (e.g. a late 429 for a request selected + # before the DELETE) must not replace the terminal state and make the + # account selectable again mid-drain. + async with SessionLocal() as session: + repo = AccountsRepository(session) + assert await repo.update_status("acc_bg_fence", AccountStatus.RATE_LIMITED, "rate_limited") is False + assert ( + await repo.update_status_if_current( + "acc_bg_fence", + AccountStatus.RATE_LIMITED, + "rate_limited", + expected_status=AccountStatus.DEACTIVATED, + expected_deactivation_reason=ACCOUNT_PENDING_DELETION_REASON, + ) + is False + ) + row = await _account_row("acc_bg_fence") + assert row is not None + assert row.status is AccountStatus.DEACTIVATED + assert row.deactivation_reason == ACCOUNT_PENDING_DELETION_REASON + assert row.delete_requested_at is not None + + +@pytest.mark.asyncio +async def test_chunked_soft_delete_drains_across_chunk_boundaries(db_setup): + await _seed_account("acc_bg_soft", log_count=7, usage_count=5) + async with SessionLocal() as session: + session.add( + StickySession( + key="sticky_bg_soft", + kind=StickySessionKind.CODEX_SESSION, + account_id="acc_bg_soft", + ) + ) + await session.commit() + + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_soft") + + outcomes = await run_account_deletion_pass(batch_size=3) + assert outcomes == {"acc_bg_soft": "finalized"} + + assert await _account_row("acc_bg_soft") is None + logs = await _log_rows("acc_bg_soft") + assert len(logs) == 7 + assert all(row.account_id is None and row.deleted_at is not None for row in logs) + async with SessionLocal() as session: + usage_left = ( + await session.execute(select(func.count(UsageHistory.id)).where(UsageHistory.account_id == "acc_bg_soft")) + ).scalar_one() + sticky_left = ( + await session.execute( + select(func.count(StickySession.key)).where(StickySession.account_id == "acc_bg_soft") + ) + ).scalar_one() + assert usage_left == 0 + assert sticky_left == 0 + assert await _lifetime_rollup("acc_bg_soft") is None + + # Idempotent: a second pass finds nothing to do. + assert await run_account_deletion_pass(batch_size=3) == {} + + +@pytest.mark.asyncio +async def test_chunked_hard_delete_removes_history(db_setup): + await _seed_account("acc_bg_hard", log_count=5, usage_count=2) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_hard", delete_history=True) + + outcomes = await run_account_deletion_pass(batch_size=2) + assert outcomes == {"acc_bg_hard": "finalized"} + + assert await _account_row("acc_bg_hard") is None + assert await _log_rows("acc_bg_hard") == [] + + +@pytest.mark.asyncio +async def test_fold_interleaved_between_chunks_does_not_resurrect_soft(db_setup): + """A fold slice committing between detach chunks re-attributes still- + attached rows to the account; finalization's fold-locked mirrors must + move ALL of it to the orphaned-deleted dimension.""" + now = utcnow() + account_dimension = to_dimension("acc_bg_fold") + # Group A (2 rows) old enough for the first fold; group B (2 rows) folded + # only by the interleaved fold below. + await _seed_account("acc_bg_fold", log_count=2, requested_at=now - timedelta(days=5)) + async with SessionLocal() as session: + logs_repo = RequestLogsRepository(session) + for index in range(2): + await _add_log( + logs_repo, + account_id="acc_bg_fold", + request_id=f"req_acc_bg_fold_b{index}", + requested_at=now - timedelta(days=2), + ) + + # First fold covers only group A (target = now-3d - FOLD_LAG). + await run_fold_pass(now=now - timedelta(days=3)) + await run_hourly_fold_pass(now=now - timedelta(days=3)) + assert await _hourly_rows_for_dimension(account_dimension) != [] + + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_fold") + + # One chunk detaches the two oldest (group A) rows; group B stays attached. + assert await _run_one_detach_chunk("acc_bg_fold", batch_size=2) == 2 + assert await _attached_log_count("acc_bg_fold") == 2 + + # Interleaved folds aggregate group B while it is still attributed. + await run_fold_pass(now=now) + await run_hourly_fold_pass(now=now) + assert await _lifetime_rollup("acc_bg_fold") is not None + interleaved_hourly = await _hourly_rows_for_dimension(account_dimension) + assert sum(row.request_count for row in interleaved_hourly if not row.is_deleted) >= 2 + + # Resume and finish the deletion. + outcomes = await run_account_deletion_pass(batch_size=2) + assert outcomes == {"acc_bg_fold": "finalized"} + + # No folded row anywhere still carries the account dimension... + assert await _hourly_rows_for_dimension(account_dimension) == [] + assert await _demand_rows_for_dimension(account_dimension) == [] + assert await _lifetime_rollup("acc_bg_fold") is None + # ...and the orphaned-deleted dimension preserves the full folded history. + orphan_hourly = await _hourly_rows_for_dimension(_ORPHAN_DIMENSION) + assert sum(row.request_count for row in orphan_hourly if row.is_deleted) == 4 + logs = await _log_rows("acc_bg_fold") + assert len(logs) == 4 + assert all(row.account_id is None and row.deleted_at is not None for row in logs) + + # Folds after finalization see only detached raw rows: nothing new may + # appear under the account dimension. + await run_hourly_fold_pass(now=now + timedelta(days=1)) + await run_fold_pass(now=now + timedelta(days=1)) + assert await _hourly_rows_for_dimension(account_dimension) == [] + assert await _lifetime_rollup("acc_bg_fold") is None + + +@pytest.mark.asyncio +async def test_fold_interleaved_between_chunks_does_not_resurrect_hard(db_setup): + now = utcnow() + account_dimension = to_dimension("acc_bg_fhard") + await _seed_account("acc_bg_fhard", log_count=2, requested_at=now - timedelta(days=5)) + async with SessionLocal() as session: + logs_repo = RequestLogsRepository(session) + for index in range(2): + await _add_log( + logs_repo, + account_id="acc_bg_fhard", + request_id=f"req_acc_bg_fhard_b{index}", + requested_at=now - timedelta(days=2), + ) + await run_hourly_fold_pass(now=now - timedelta(days=3)) + + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_fhard", delete_history=True) + + assert await _run_one_detach_chunk("acc_bg_fhard", batch_size=2, delete_history=True) == 2 + await run_hourly_fold_pass(now=now) + + outcomes = await run_account_deletion_pass(batch_size=2) + assert outcomes == {"acc_bg_fhard": "finalized"} + + assert await _hourly_rows_for_dimension(account_dimension) == [] + assert await _demand_rows_for_dimension(account_dimension) == [] + assert await _log_rows("acc_bg_fhard") == [] + + +@pytest.mark.asyncio +async def test_pass_round_robins_chunks_across_pending_accounts(db_setup, monkeypatch): + """One account's long drain must not starve another: each round advances + every pending account by at most one full chunk.""" + from app.modules.accounts import deletion + + await _seed_account("acc_bg_rr_a", log_count=3) + await _seed_account("acc_bg_rr_b", log_count=3) + async with SessionLocal() as session: + repo = AccountsRepository(session) + assert await repo.begin_delete("acc_bg_rr_a") + assert await repo.begin_delete("acc_bg_rr_b") + + chunk_calls: list[str] = [] + original_chunk = deletion._request_logs_chunk + + async def spy_chunk(session, account_id, *, delete_history, batch_size): + chunk_calls.append(account_id) + return await original_chunk(session, account_id, delete_history=delete_history, batch_size=batch_size) + + monkeypatch.setattr(deletion, "_request_logs_chunk", spy_chunk) + + outcomes = await run_account_deletion_pass(batch_size=1) + assert outcomes == {"acc_bg_rr_a": "finalized", "acc_bg_rr_b": "finalized"} + # Full chunks alternate between the two accounts instead of draining one + # account to completion first. + assert chunk_calls[:6] == [ + "acc_bg_rr_a", + "acc_bg_rr_b", + "acc_bg_rr_a", + "acc_bg_rr_b", + "acc_bg_rr_a", + "acc_bg_rr_b", + ] + assert await _account_row("acc_bg_rr_a") is None + assert await _account_row("acc_bg_rr_b") is None + + +@pytest.mark.asyncio +async def test_pass_picks_up_account_marked_mid_pass(db_setup, monkeypatch): + """A DELETE that lands while a pass is draining another account is picked + up by the between-rounds re-scan, not deferred to the next tick.""" + from app.modules.accounts import deletion + + await _seed_account("acc_bg_mid_a", log_count=2) + await _seed_account("acc_bg_mid_b", log_count=1) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_mid_a") + + original_advance = deletion._advance_account + marked_second = False + + async def advance_and_mark(account_id, *, batch_size, drained=None): + nonlocal marked_second + if not marked_second: + marked_second = True + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_mid_b") + return await original_advance(account_id, batch_size=batch_size, drained=drained) + + monkeypatch.setattr(deletion, "_advance_account", advance_and_mark) + + outcomes = await run_account_deletion_pass(batch_size=1) + assert outcomes == {"acc_bg_mid_a": "finalized", "acc_bg_mid_b": "finalized"} + assert await _account_row("acc_bg_mid_a") is None + assert await _account_row("acc_bg_mid_b") is None + + +@pytest.mark.asyncio +async def test_restart_resumes_partial_drain(db_setup): + await _seed_account("acc_bg_resume", log_count=5, usage_count=3) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_resume") + + # Simulate a crash after one detach chunk: progress lives in the rows. + assert await _run_one_detach_chunk("acc_bg_resume", batch_size=2) == 2 + assert await _attached_log_count("acc_bg_resume") == 3 + + # A fresh pass (restarted leader) resumes from the database state. + outcomes = await run_account_deletion_pass(batch_size=2) + assert outcomes == {"acc_bg_resume": "finalized"} + assert await _account_row("acc_bg_resume") is None + logs = await _log_rows("acc_bg_resume") + assert len(logs) == 5 + assert all(row.account_id is None for row in logs) + + +@pytest.mark.asyncio +async def test_straggler_row_settled_mid_drain_is_finalized(db_setup): + """A stream that settles its request-log row after the drain chunks ran + is caught by finalization's residual sweep.""" + await _seed_account("acc_bg_late", log_count=3) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_late") + assert await _run_one_detach_chunk("acc_bg_late", batch_size=10) == 3 + + async with SessionLocal() as session: + await _add_log( + RequestLogsRepository(session), + account_id="acc_bg_late", + request_id="req_acc_bg_late_straggler", + requested_at=utcnow() - timedelta(hours=1), + ) + + outcomes = await run_account_deletion_pass(batch_size=10) + assert outcomes == {"acc_bg_late": "finalized"} + logs = await _log_rows("acc_bg_late") + assert len(logs) == 4 + assert all(row.account_id is None and row.deleted_at is not None for row in logs) + + +@pytest.mark.asyncio +async def test_credential_replacement_supersedes_pending_deletion(db_setup): + await _seed_account("acc_bg_super", log_count=4) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_super") + assert await _run_one_detach_chunk("acc_bg_super", batch_size=2) == 2 + + # Re-import lands on the same row via the slot-identity path and clears + # the marker (credential replacement supersedes the deletion). + async with SessionLocal() as session: + replacement = _make_account("acc_bg_super", "acc_bg_super@example.com") + saved = await AccountsRepository(session).upsert(replacement, merge_by_email=True) + assert saved.id == "acc_bg_super" + + row = await _account_row("acc_bg_super") + assert row is not None + assert row.delete_requested_at is None + assert row.status is AccountStatus.ACTIVE + + outcomes = await run_account_deletion_pass(batch_size=2) + assert outcomes == {} + assert await _account_row("acc_bg_super") is not None + # Rows detached before the supersede stay detached; the rest survive. + assert await _attached_log_count("acc_bg_super") == 2 + + +async def _legacy_replace_credentials(account_id: str, encryptor: TokenEncryptor) -> None: + """Mimic a credential replacement by a pre-upgrade replica: fresh + ciphertext and status, but the marker columns its ORM does not know stay + untouched.""" + async with SessionLocal() as session: + await session.execute( + update(Account) + .where(Account.id == account_id) + .values( + access_token_encrypted=encryptor.encrypt("fresh-access"), + refresh_token_encrypted=encryptor.encrypt("fresh-refresh"), + id_token_encrypted=encryptor.encrypt("fresh-id"), + status=AccountStatus.ACTIVE, + deactivation_reason=None, + ) + ) + await session.commit() + + +@pytest.mark.asyncio +async def test_legacy_replica_replacement_supersedes_mid_drain(db_setup): + """A replacement handled by a pre-upgrade replica cannot clear the marker; + fresh (non-wiped) ciphertext on a marked row must itself supersede.""" + encryptor = TokenEncryptor() + await _seed_account("acc_bg_legacy", log_count=3) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_legacy") + assert await _run_one_detach_chunk("acc_bg_legacy", batch_size=2) == 2 + + await _legacy_replace_credentials("acc_bg_legacy", encryptor) + + outcomes = await run_account_deletion_pass(batch_size=2) + assert outcomes == {"acc_bg_legacy": "superseded"} + row = await _account_row("acc_bg_legacy") + assert row is not None + # The worker cleared the marker itself and preserved the fresh material. + assert row.delete_requested_at is None + assert encryptor.decrypt(row.refresh_token_encrypted) == "fresh-refresh" + # Rows detached before the replacement stay detached; the rest survive. + assert await _attached_log_count("acc_bg_legacy") == 1 + # The account is no longer rescanned on later passes. + assert await run_account_deletion_pass(batch_size=2) == {} + + +@pytest.mark.asyncio +async def test_legacy_replacement_with_empty_refresh_token_supersedes(db_setup): + """A legal replacement may carry an empty refresh token while providing + fresh access/id material; a refresh-only wipe check would mistake it for + the original wipe and finalize the freshly replaced account.""" + encryptor = TokenEncryptor() + await _seed_account("acc_bg_legacy_er", log_count=1) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_legacy_er") + assert await _run_one_detach_chunk("acc_bg_legacy_er", batch_size=10) == 1 + + async with SessionLocal() as session: + await session.execute( + update(Account) + .where(Account.id == "acc_bg_legacy_er") + .values( + access_token_encrypted=encryptor.encrypt("fresh-access"), + refresh_token_encrypted=encryptor.encrypt(""), + id_token_encrypted=encryptor.encrypt("fresh-id"), + status=AccountStatus.ACTIVE, + deactivation_reason=None, + ) + ) + await session.commit() + + outcomes = await run_account_deletion_pass(batch_size=10) + assert outcomes == {"acc_bg_legacy_er": "superseded"} + row = await _account_row("acc_bg_legacy_er") + assert row is not None + assert row.delete_requested_at is None + assert encryptor.decrypt(row.access_token_encrypted) == "fresh-access" + + +@pytest.mark.asyncio +async def test_legacy_replica_replacement_before_finalize_is_abandoned(db_setup): + encryptor = TokenEncryptor() + await _seed_account("acc_bg_legacy_fin", log_count=1) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_legacy_fin") + assert await _run_one_detach_chunk("acc_bg_legacy_fin", batch_size=10) == 1 + + await _legacy_replace_credentials("acc_bg_legacy_fin", encryptor) + + async with SessionLocal() as session: + assert await AccountsRepository(session).delete("acc_bg_legacy_fin", only_pending=True) is False + row = await _account_row("acc_bg_legacy_fin") + assert row is not None + assert row.delete_requested_at is None + assert encryptor.decrypt(row.refresh_token_encrypted) == "fresh-refresh" + + +@pytest.mark.asyncio +async def test_repeat_delete_short_circuits_without_waiting_on_chunk_lock(db_setup): + """A repeat DELETE must keep the millisecond contract even while a drain + chunk transaction holds the account row lock.""" + import asyncio + + async with SessionLocal() as probe: + if probe.get_bind().dialect.name != "postgresql": + pytest.skip("row-lock wait behavior is PostgreSQL-specific") + + await _seed_account("acc_bg_repeat", log_count=1) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_repeat") + + async with SessionLocal() as locker: + # Hold the same lock a drain chunk holds for its whole transaction. + await locker.execute(select(Account.id).where(Account.id == "acc_bg_repeat").with_for_update(key_share=True)) + async with SessionLocal() as session: + repeat = await asyncio.wait_for(AccountsRepository(session).begin_delete("acc_bg_repeat"), timeout=2.0) + assert repeat is True + await locker.rollback() + + +@pytest.mark.asyncio +async def test_chunk_self_heals_drift_from_unfenced_replicas(db_setup): + """During a rolling deploy, pre-upgrade replicas' unfenced writers can + replace the terminal status or recreate API-key assignments on a marked + row; the next chunk transaction must re-fence both.""" + from app.db.models import ApiKey, ApiKeyAccountAssignment + from app.modules.accounts import deletion + + await _seed_account("acc_bg_heal", log_count=2) + async with SessionLocal() as session: + session.add( + ApiKey( + id="key_bg_heal", + name="heal-key", + key_hash="hash_bg_heal", + key_prefix="sk-heal", + account_assignment_scope_enabled=True, + ) + ) + await session.commit() + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_heal") + + # Old-replica drift: unfenced status write + unconditional assignment + # insert (tokens stay wiped, so this is NOT a credential replacement). + async with SessionLocal() as session: + await session.execute( + update(Account) + .where(Account.id == "acc_bg_heal") + .values(status=AccountStatus.RATE_LIMITED, deactivation_reason="rate_limited") + ) + session.add(ApiKeyAccountAssignment(api_key_id="key_bg_heal", account_id="acc_bg_heal")) + await session.commit() + + affected = await deletion._run_chunk(deletion._usage_history_chunk, "acc_bg_heal", batch_size=10) + assert affected is not None + + row = await _account_row("acc_bg_heal") + assert row is not None + assert row.status is AccountStatus.DEACTIVATED + assert row.deactivation_reason == ACCOUNT_PENDING_DELETION_REASON + assert row.delete_requested_at is not None + async with SessionLocal() as session: + assigned = ( + await session.execute( + select(func.count()) + .select_from(ApiKeyAccountAssignment) + .where(ApiKeyAccountAssignment.account_id == "acc_bg_heal") + ) + ).scalar_one() + assert assigned == 0 + + +@pytest.mark.asyncio +async def test_finalization_serializes_against_inflight_log_insert(db_setup): + """PostgreSQL: an in-flight stream's request-log insert holds the FK KEY + SHARE on the account row; finalization's FOR UPDATE row upgrade must wait + for it, so the late row is swept instead of surviving as a live orphan + via ON DELETE SET NULL.""" + import asyncio + + async with SessionLocal() as probe: + if probe.get_bind().dialect.name != "postgresql": + pytest.skip("FK KEY SHARE / FOR UPDATE interleaving is PostgreSQL-specific") + + await _seed_account("acc_bg_inflight", log_count=1) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_inflight") + assert await _run_one_detach_chunk("acc_bg_inflight", batch_size=10) == 1 + + async with SessionLocal() as inflight: + # In-flight stream: the insert takes (and holds) FK KEY SHARE on the + # account row until commit. + inflight.add( + RequestLog( + account_id="acc_bg_inflight", + request_id="req_acc_bg_inflight_late", + requested_at=utcnow(), + model="gpt-5.1-codex", + status="success", + input_tokens=1, + output_tokens=1, + cost_usd=0.0, + ) + ) + await inflight.flush() + + pass_task = asyncio.create_task(run_account_deletion_pass(batch_size=10)) + # Finalization must block on the row upgrade while the insert is open. + done, _ = await asyncio.wait({pass_task}, timeout=1.0) + commit_first = not done + await inflight.commit() + outcomes = await pass_task + + assert commit_first, "finalization finished while an uncommitted FK insert held KEY SHARE" + assert outcomes == {"acc_bg_inflight": "finalized"} + assert await _account_row("acc_bg_inflight") is None + logs = await _log_rows("acc_bg_inflight") + assert len(logs) == 2 + # The late row was swept by the residual sweep, not orphaned live. + assert all(row.account_id is None and row.deleted_at is not None for row in logs) + + +@pytest.mark.asyncio +async def test_assignment_insert_rechecks_marker_atomically(db_setup): + """replace_account_assignments must skip marked accounts even when an + earlier validation (different transaction) still believed they existed.""" + from app.db.models import ApiKey, ApiKeyAccountAssignment + from app.modules.api_keys.repository import ApiKeysRepository + + await _seed_account("acc_bg_atomic", log_count=0) + async with SessionLocal() as session: + session.add( + ApiKey( + id="key_bg_atomic", + name="atomic-recheck-key", + key_hash="hash_bg_atomic", + key_prefix="sk-atomic", + account_assignment_scope_enabled=True, + ) + ) + await session.commit() + + # DELETE lands after validation would have passed. + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_atomic") + + async with SessionLocal() as session: + await ApiKeysRepository(session).replace_account_assignments("key_bg_atomic", ["acc_bg_atomic"]) + + async with SessionLocal() as session: + assigned = ( + await session.execute( + select(func.count()) + .select_from(ApiKeyAccountAssignment) + .where(ApiKeyAccountAssignment.api_key_id == "key_bg_atomic") + ) + ).scalar_one() + assert assigned == 0 + + +@pytest.mark.asyncio +async def test_marked_account_cannot_be_assigned_to_api_key(async_client, db_setup): + await _seed_account("acc_bg_assign", log_count=1) + + delete = await async_client.delete("/api/accounts/acc_bg_assign") + assert delete.status_code == 200 + + # A key update racing (or following) the DELETE must not recreate an + # assignment that would re-surface the deleted account in key listings. + create = await async_client.post("/api/api-keys/", json={"name": "post-delete-key"}) + assert create.status_code == 200 + key_id = create.json()["id"] + update_resp = await async_client.patch( + f"/api/api-keys/{key_id}", + json={"assignedAccountIds": ["acc_bg_assign"]}, + ) + assert update_resp.status_code == 400 + assert update_resp.json()["error"]["code"] == "invalid_api_key_payload" + + +@pytest.mark.asyncio +async def test_supersede_after_partial_drain_preserves_folded_attribution(db_setup): + """Rows drained before a supersede stay drained, and folded rollups keep + attributing that traffic to the revived account — with no double count + from later folds (drained below-watermark rows are never re-folded).""" + now = utcnow() + account_dimension = to_dimension("acc_bg_sfold") + await _seed_account("acc_bg_sfold", log_count=2, requested_at=now - timedelta(days=5)) + + # Fold the two rows under the account dimension first. + await run_fold_pass(now=now - timedelta(days=3)) + await run_hourly_fold_pass(now=now - timedelta(days=3)) + folded_before = sum(row.request_count for row in await _hourly_rows_for_dimension(account_dimension)) + assert folded_before == 2 + + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_sfold") + # One chunk detaches both already-folded rows. + assert await _run_one_detach_chunk("acc_bg_sfold", batch_size=10) == 2 + + # Re-import supersedes before finalization ever runs. + async with SessionLocal() as session: + saved = await AccountsRepository(session).upsert( + _make_account("acc_bg_sfold", "acc_bg_sfold@example.com"), merge_by_email=True + ) + assert saved.id == "acc_bg_sfold" + assert await run_account_deletion_pass(batch_size=10) == {} + + # Folded attribution is the permanent end state: unchanged by later + # folds (no loss, no double count), while raw rows stay detached. + await run_fold_pass(now=now) + await run_hourly_fold_pass(now=now) + folded_after = sum(row.request_count for row in await _hourly_rows_for_dimension(account_dimension)) + assert folded_after == folded_before + assert await _lifetime_rollup("acc_bg_sfold") is not None + logs = await _log_rows("acc_bg_sfold") + assert len(logs) == 2 + assert all(row.account_id is None and row.deleted_at is not None for row in logs) + + +@pytest.mark.asyncio +async def test_supersede_between_drain_and_finalize_is_abandoned(db_setup): + await _seed_account("acc_bg_race", log_count=2) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_race") + assert await _run_one_detach_chunk("acc_bg_race", batch_size=10) == 2 + + # Marker cleared right before finalization (replacement won the race). + async with SessionLocal() as session: + await session.execute( + update(Account) + .where(Account.id == "acc_bg_race") + .values(delete_requested_at=None, delete_history_requested=False) + ) + await session.commit() + + async with SessionLocal() as session: + assert await AccountsRepository(session).delete("acc_bg_race", only_pending=True) is False + assert await _account_row("acc_bg_race") is not None + + +def _batch_pinning_cases() -> tuple[tuple[Callable[[str, int], Select[tuple[int]]], Table, str], ...]: + from app.db.models import AdditionalUsageHistory + from app.modules.accounts import deletion + + return ( + (deletion._usage_history_batch, cast("Table", UsageHistory.__table__), "idx_usage_account_time"), + ( + deletion._additional_usage_history_batch, + cast("Table", AdditionalUsageHistory.__table__), + "ix_additional_usage_distinct_labels", + ), + (deletion._request_logs_batch, cast("Table", RequestLog.__table__), "idx_logs_account_kind_deleted_latest"), + ) + + +def test_chunk_batch_statements_pin_account_leading_index_order(): + """The chunk batch shape is what keeps the planner off sequential scans. + + An ``account_id = :id LIMIT n`` subquery plans as a LIMIT-terminated Seq + Scan on the production planner for exactly the large accounts the drain + targets (equality folds account_id out of the sort pathkeys). The batch + builders must keep (a) the range predicate pair (never plain equality) + and (b) an ORDER BY that lists the target account-leading index's exact + column order, so that index is the only sort-free plan. + """ + from sqlalchemy.dialects import postgresql as postgresql_dialect + + for batch_fn, table, index_name in _batch_pinning_cases(): + index = next(idx for idx in table.indexes if idx.name == index_name) + sql = str(batch_fn("acc_bg_pin", 50).compile(dialect=postgresql_dialect.dialect())) + expected_order = ", ".join(f"{table.name}.{column.name}" for column in index.columns) + assert f"ORDER BY {expected_order}" in sql, sql + assert f"{table.name}.account_id >= " in sql, sql + assert f"{table.name}.account_id <= " in sql, sql + assert f"{table.name}.account_id = " not in sql, sql + + +@pytest.mark.asyncio +async def test_chunk_batch_query_plan_uses_account_leading_indexes_postgresql(db_setup): + """The batch ORDER BY must be served by the pinned index, not a sort. + + Sequential/bitmap scans and (incremental) sorts are disabled so the + planner has to surface an ordered index path for the batch shape; the + only index that can provide the ORDER BY after the leading account_id + range is the pinned account-leading index. A drained (or missing) + account's probe must terminate on the same index instead of falling + back to a heap scan. + """ + await _seed_account("acc_bg_plan", log_count=8, usage_count=8) + async with SessionLocal() as session: + if session.get_bind().dialect.name != "postgresql": + pytest.skip("PostgreSQL-only query plan test") + + from app.db.models import AdditionalUsageHistory + + session.add_all( + AdditionalUsageHistory( + account_id="acc_bg_plan", + quota_key="codex_spark", + limit_name="GPT-5.3-Codex-Spark", + metered_feature="codex_bengalfox", + window="primary", + used_percent=float(index), + ) + for index in range(8) + ) + await session.commit() + + await session.execute(text("SET enable_seqscan = off")) + await session.execute(text("SET enable_bitmapscan = off")) + await session.execute(text("SET enable_sort = off")) + await session.execute(text("SET enable_incremental_sort = off")) + for batch_fn, _table, index_name in _batch_pinning_cases(): + for account_id in ("acc_bg_plan", "acc_bg_plan_drained_probe"): + compiled = batch_fn(account_id, 5).compile( + dialect=session.get_bind().dialect, compile_kwargs={"literal_binds": True} + ) + plan = (await session.execute(text(f"EXPLAIN (FORMAT JSON) {compiled}"))).scalar_one() + plan_json = json.dumps(plan) + assert index_name in plan_json, (account_id, plan_json) + assert "Seq Scan" not in plan_json, (account_id, plan_json) + assert "Sort Key" not in plan_json, (account_id, plan_json) + + +@pytest.mark.asyncio +async def test_pass_probes_drained_tables_once_per_pass(db_setup, monkeypatch): + """A table observed empty for an account is not re-probed on later rounds + of the same pass: each probe is a full account-row-locking transaction, + and per-account statistics can go stale exactly during the churn window.""" + from app.modules.accounts import deletion + + await _seed_account("acc_bg_memo", log_count=3) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_memo") + + calls = {"usage_history": 0, "additional_usage_history": 0, "request_logs": 0} + for attr, key in ( + ("_usage_history_chunk", "usage_history"), + ("_additional_usage_history_chunk", "additional_usage_history"), + ("_request_logs_chunk", "request_logs"), + ): + original = getattr(deletion, attr) + + def _make_spy(original=original, key=key): + async def spy(session, account_id, *, delete_history, batch_size): + calls[key] += 1 + return await original(session, account_id, delete_history=delete_history, batch_size=batch_size) + + return spy + + monkeypatch.setattr(deletion, attr, _make_spy()) + + outcomes = await run_account_deletion_pass(batch_size=1) + assert outcomes == {"acc_bg_memo": "finalized"} + # usage tables: exactly one (empty) probe in round 1, then skipped while + # rounds 2-4 drain the logs; request_logs: three one-row chunks plus the + # final empty probe that lets the pass finalize. + assert calls == {"usage_history": 1, "additional_usage_history": 1, "request_logs": 4} + assert await _account_row("acc_bg_memo") is None diff --git a/tests/integration/test_account_usage_rollup.py b/tests/integration/test_account_usage_rollup.py index 5273cd24d6..06864e1dcf 100644 --- a/tests/integration/test_account_usage_rollup.py +++ b/tests/integration/test_account_usage_rollup.py @@ -396,7 +396,7 @@ async def test_account_delete_removes_rollup_row(db_setup): logs_repo, account_id="acc_del2", request_id="req_2", - requested_at=now - timedelta(hours=12), + requested_at=now - FOLD_LAG / 2, ) await run_fold_pass(now=now + timedelta(days=1)) assert len(await _rollup_rows()) == 1 @@ -444,3 +444,167 @@ async def test_backfill_start_skips_excluded_prefix(db_setup): summaries = await _summaries() assert summaries["acc_prefix"].request_count == 1 + + +@pytest.mark.asyncio +async def test_fold_absorbs_widened_watermark_gap(db_setup): + """Upgrading from the previous 24h fold lag leaves the watermark far + behind the new ``now - FOLD_LAG`` target; the next pass must absorb the + gap as ordinary backfill slices with reported totals unchanged.""" + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + await accounts_repo.upsert(_make_account("acc_gap", "gap@example.com")) + await _add_log( + logs_repo, account_id="acc_gap", request_id="req_gap_old", requested_at=now - timedelta(hours=30) + ) + await _add_log( + logs_repo, account_id="acc_gap", request_id="req_gap_mid", requested_at=now - timedelta(hours=12) + ) + await _add_log( + logs_repo, account_id="acc_gap", request_id="req_gap_young", requested_at=now - timedelta(minutes=10) + ) + + # Simulate the pre-upgrade deployment: a fold pass whose target lands a + # day back, leaving rows between the old and new targets unfolded. + await run_fold_pass(now=now - timedelta(hours=24) + FOLD_LAG) + assert await _watermark() == now - timedelta(hours=24) + before = await _summaries() + assert before["acc_gap"].request_count == 3 + + await run_fold_pass(now=now) + assert await _watermark() == now - FOLD_LAG + rows = await _rollup_rows() + assert len(rows) == 1 + assert rows[0].request_count == 2 # old + mid folded, young stays live + + assert await _summaries() == before + + +@pytest.mark.asyncio +async def test_summary_cache_serves_within_ttl_per_signature(db_setup, monkeypatch): + import app.modules.accounts.repository as accounts_repository_module + + monkeypatch.setattr(accounts_repository_module, "_SUMMARY_CACHE_TTL_SECONDS", 30.0) + accounts_repository_module._clear_request_usage_summary_cache() + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + await accounts_repo.upsert(_make_account("acc_ttl", "ttl@example.com")) + await _add_log(logs_repo, account_id="acc_ttl", request_id="req_ttl_1", requested_at=now - timedelta(minutes=5)) + + first = await _summaries() + assert first["acc_ttl"].request_count == 1 + + async with SessionLocal() as session: + await _add_log( + RequestLogsRepository(session), + account_id="acc_ttl", + request_id="req_ttl_2", + requested_at=now - timedelta(minutes=1), + ) + + # Same signature within the TTL: served from cache, staleness tolerated. + assert (await _summaries())["acc_ttl"].request_count == 1 + # A different account-id signature is a different cache entry. + scoped = await _summaries(["acc_ttl"]) + assert scoped["acc_ttl"].request_count == 2 + + accounts_repository_module._clear_request_usage_summary_cache() + assert (await _summaries())["acc_ttl"].request_count == 2 + + +@pytest.mark.asyncio +async def test_summary_cache_cleared_on_account_delete(db_setup, monkeypatch): + import app.modules.accounts.repository as accounts_repository_module + + monkeypatch.setattr(accounts_repository_module, "_SUMMARY_CACHE_TTL_SECONDS", 30.0) + accounts_repository_module._clear_request_usage_summary_cache() + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + await accounts_repo.upsert(_make_account("acc_keep", "keep@example.com")) + await accounts_repo.upsert(_make_account("acc_gone", "gone@example.com")) + await _add_log(logs_repo, account_id="acc_keep", request_id="req_keep", requested_at=now - timedelta(minutes=5)) + await _add_log(logs_repo, account_id="acc_gone", request_id="req_gone", requested_at=now - timedelta(minutes=5)) + + first = await _summaries() + assert "acc_gone" in first + + async with SessionLocal() as session: + assert await AccountsRepository(session).delete("acc_gone") + + after = await _summaries() + assert "acc_gone" not in after + assert after["acc_keep"].request_count == 1 + + +@pytest.mark.asyncio +async def test_summary_cache_cleared_on_identity_consolidation(db_setup, monkeypatch): + import app.modules.accounts.repository as accounts_repository_module + + monkeypatch.setattr(accounts_repository_module, "_SUMMARY_CACHE_TTL_SECONDS", 30.0) + accounts_repository_module._clear_request_usage_summary_cache() + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + canonical = _make_account("acc_cc", "cc@example.com", chatgpt_account_id="chatgpt_cc") + duplicate = _make_account("acc_cc__copy", "cc@example.com", chatgpt_account_id="chatgpt_cc") + await accounts_repo.upsert(canonical, merge_by_email=False) + await accounts_repo.upsert(duplicate, merge_by_email=False) + await _add_log(logs_repo, account_id="acc_cc", request_id="req_cc_1", requested_at=now - timedelta(minutes=5)) + await _add_log( + logs_repo, account_id="acc_cc__copy", request_id="req_cc_2", requested_at=now - timedelta(minutes=5) + ) + + first = await _summaries() + assert first["acc_cc"].request_count == 1 + assert first["acc_cc__copy"].request_count == 1 + + async with SessionLocal() as session: + reauth = _make_account("acc_cc", "cc@example.com", chatgpt_account_id="chatgpt_cc") + saved = await AccountsRepository(session).upsert(reauth, merge_by_email=False, merge_by_chatgpt_identity=True) + assert saved.id == "acc_cc" + + after = await _summaries() + assert "acc_cc__copy" not in after + assert after["acc_cc"].request_count == 2 + + +@pytest.mark.asyncio +async def test_summary_cache_fill_discarded_when_invalidated_mid_flight(db_setup, monkeypatch): + """A fill already computing when deletion/consolidation clears the cache + must not re-populate it with its pre-clear result (generation fence).""" + import app.modules.accounts.repository as accounts_repository_module + + monkeypatch.setattr(accounts_repository_module, "_SUMMARY_CACHE_TTL_SECONDS", 30.0) + accounts_repository_module._clear_request_usage_summary_cache() + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + await accounts_repo.upsert(_make_account("acc_racefill", "racefill@example.com")) + await _add_log( + logs_repo, account_id="acc_racefill", request_id="req_rf", requested_at=now - timedelta(minutes=5) + ) + + real_read_state = accounts_repository_module.AccountUsageRollupRepository.read_state + + async def _read_state_with_racing_clear(self, account_ids=None): + result = await real_read_state(self, account_ids) + # Simulate a consolidation/deletion committing and clearing while + # this fill is still between its two statements. + accounts_repository_module._clear_request_usage_summary_cache() + return result + + monkeypatch.setattr( + accounts_repository_module.AccountUsageRollupRepository, "read_state", _read_state_with_racing_clear + ) + first = await _summaries() + assert first["acc_racefill"].request_count == 1 + # The interleaved clear must have won: nothing was cached. + assert accounts_repository_module._request_usage_summary_cache == {} diff --git a/tests/integration/test_accounts_api_extended.py b/tests/integration/test_accounts_api_extended.py index 144bcc9a0d..3e39c84d43 100644 --- a/tests/integration/test_accounts_api_extended.py +++ b/tests/integration/test_accounts_api_extended.py @@ -13,6 +13,7 @@ from app.core.utils.time import naive_utc_to_epoch, utcnow from app.db.models import Account, AccountStatus, RequestLog from app.db.session import SessionLocal +from app.modules.accounts.deletion import run_account_deletion_pass from app.modules.accounts.repository import AccountsRepository from app.modules.proxy.account_cache import clear_account_routing_unavailable, is_account_routing_unavailable from app.modules.request_logs.repository import RequestLogsRepository @@ -592,7 +593,17 @@ async def test_delete_account_removes_from_list(async_client): @pytest.mark.asyncio -async def test_delete_account_soft_deletes_request_logs(async_client, db_setup): +async def test_delete_account_soft_deletes_request_logs(async_client, db_setup, monkeypatch): + # The suite's inline leader election would let the API's worker wake race + # the explicit pass below; keep the drain under test control. The + # scheduler's own startup/interval tick is neutralized for the same + # reason. + monkeypatch.setattr("app.modules.accounts.service.request_account_deletion_run", lambda: None) + + async def _no_tick(self) -> None: + return None + + monkeypatch.setattr("app.modules.accounts.deletion.AccountDeletionScheduler._run_once", _no_tick) async with SessionLocal() as session: accounts_repo = AccountsRepository(session) logs_repo = RequestLogsRepository(session) @@ -612,6 +623,10 @@ async def test_delete_account_soft_deletes_request_logs(async_client, db_setup): delete = await async_client.delete("/api/accounts/acc_delete_logs") assert delete.status_code == 200 + # The API only marks the account; the background worker drains the rows. + outcomes = await run_account_deletion_pass() + assert outcomes["acc_delete_logs"] == "finalized" + async with SessionLocal() as session: row = ( await session.execute(select(RequestLog).where(RequestLog.request_id == "req_delete_logs_1")) @@ -629,7 +644,17 @@ async def test_delete_account_soft_deletes_request_logs(async_client, db_setup): @pytest.mark.asyncio -async def test_delete_account_with_delete_history_hard_deletes_request_logs(async_client, db_setup): +async def test_delete_account_with_delete_history_hard_deletes_request_logs(async_client, db_setup, monkeypatch): + # The suite's inline leader election would let the API's worker wake race + # the explicit pass below; keep the drain under test control. The + # scheduler's own startup/interval tick is neutralized for the same + # reason. + monkeypatch.setattr("app.modules.accounts.service.request_account_deletion_run", lambda: None) + + async def _no_tick(self) -> None: + return None + + monkeypatch.setattr("app.modules.accounts.deletion.AccountDeletionScheduler._run_once", _no_tick) async with SessionLocal() as session: accounts_repo = AccountsRepository(session) logs_repo = RequestLogsRepository(session) @@ -650,6 +675,10 @@ async def test_delete_account_with_delete_history_hard_deletes_request_logs(asyn assert delete.status_code == 200 assert delete.json()["status"] == "deleted" + # The API only marks the account; the background worker drains the rows. + outcomes = await run_account_deletion_pass() + assert outcomes["acc_hard_delete"] == "finalized" + async with SessionLocal() as session: result = await session.execute(select(RequestLog).where(RequestLog.request_id == "req_hard_delete_1")) assert result.scalar_one_or_none() is None diff --git a/tests/integration/test_api_keys_api.py b/tests/integration/test_api_keys_api.py index 093eff9b7b..4508b5dbe0 100644 --- a/tests/integration/test_api_keys_api.py +++ b/tests/integration/test_api_keys_api.py @@ -4,12 +4,14 @@ import base64 import contextlib import json +from dataclasses import replace from datetime import timedelta from types import SimpleNamespace from typing import cast import pytest from fastapi.responses import JSONResponse +from sqlalchemy import exc as sqlalchemy_exc from sqlalchemy import select, update import app.core.clients.proxy as core_proxy_module @@ -26,7 +28,7 @@ from app.db.session import SessionLocal from app.modules.api_keys.last_used_coalescer import get_api_key_last_used_coalescer from app.modules.api_keys.repository import ApiKeysRepository -from app.modules.api_keys.service import ApiKeyCreateData, ApiKeysService, LimitRuleInput +from app.modules.api_keys.service import ApiKeyCreateData, ApiKeyInvalidError, ApiKeysService, LimitRuleInput from app.modules.model_sources.forwarding import ( SourceChatCompletion, SourceResponsesStream, @@ -777,10 +779,14 @@ async def fake_stream(payload, _headers, _access_token, _account_id, base_url=No @pytest.mark.asyncio -async def test_api_key_enforces_service_tier_for_responses(async_client, monkeypatch): - await _populate_test_registry() - model_ids = sorted(_TEST_MODELS) - forced_model = model_ids[0] +@pytest.mark.parametrize( + ("enforced_service_tier", "expected_service_tier"), + [("fast", "priority"), ("ULTRAFAST", "ultrafast")], +) +async def test_api_key_enforces_service_tier_for_responses( + async_client, monkeypatch, enforced_service_tier, expected_service_tier +): + forced_model = "gpt-5.6-sol" enable = await async_client.put( "/api/settings", @@ -793,27 +799,47 @@ async def test_api_key_enforces_service_tier_for_responses(async_client, monkeyp ) assert enable.status_code == 200 + account_id = await _import_account( + async_client, + f"acc_enforced_{expected_service_tier}_service_tier", + f"enforced-{expected_service_tier}-service-tier@example.com", + ) + advertising_model = replace( + _make_upstream_model(forced_model), + raw={"service_tiers": [{"slug": expected_service_tier}]}, + ) + await get_model_registry().update( + {"pro": [advertising_model]}, + per_account_results={account_id: ("pro", [advertising_model])}, + active_account_plans={account_id: "pro"}, + ) + created = await async_client.post( "/api/api-keys/", json={ "name": "enforced-service-tier", "allowedModels": [forced_model], "enforcedModel": forced_model, - "enforcedServiceTier": "fast", + "enforcedServiceTier": enforced_service_tier, }, ) assert created.status_code == 200 key = created.json()["key"] - assert created.json()["enforcedServiceTier"] == "priority" - - await _import_account(async_client, "acc_enforced_service_tier", "enforced-service-tier@example.com") + assert created.json()["enforcedServiceTier"] == expected_service_tier seen: dict[str, str | None] = {} async def fake_stream(payload, _headers, _access_token, _account_id, base_url=None, raise_for_status=False): seen["service_tier"] = payload.service_tier usage = {"input_tokens": 3, "output_tokens": 2} - event = {"type": "response.completed", "response": {"id": "resp_enforced_service_tier", "usage": usage}} + event = { + "type": "response.completed", + "response": { + "id": "resp_enforced_service_tier", + "service_tier": expected_service_tier, + "usage": usage, + }, + } yield f"data: {json.dumps(event)}\n\n" monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) @@ -833,7 +859,15 @@ async def fake_stream(payload, _headers, _access_token, _account_id, base_url=No assert response.status_code == 200 _ = [line async for line in response.aiter_lines() if line] - assert seen["service_tier"] == "priority" + assert seen["service_tier"] == expected_service_tier + + async with SessionLocal() as session: + result = await session.execute(select(RequestLog).order_by(RequestLog.requested_at.desc())) + latest_log = result.scalars().first() + assert latest_log is not None + assert latest_log.requested_service_tier == expected_service_tier + assert latest_log.actual_service_tier == expected_service_tier + assert latest_log.service_tier == expected_service_tier @pytest.mark.asyncio @@ -2207,6 +2241,134 @@ async def test_api_key_update_accepts_extended_enforced_reasoning(async_client): assert updated.json()["enforcedReasoningEffort"] == "max" +@pytest.mark.asyncio +async def test_api_key_reasoning_effort_allowlist_is_normalized_and_enforced(async_client): + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "bounded-reasoning-key", + "allowedReasoningEfforts": ["XHIGH", "low", "high", "low"], + }, + ) + assert created.status_code == 200 + key_id = created.json()["id"] + key = created.json()["key"] + assert created.json()["allowedReasoningEfforts"] == ["low", "high", "xhigh"] + + conflicting_update = await async_client.patch( + f"/api/api-keys/{key_id}", + json={"enforcedReasoningEffort": "low"}, + ) + assert conflicting_update.status_code == 400 + assert conflicting_update.json()["error"]["code"] == "invalid_api_key_payload" + + empty_create = await async_client.post( + "/api/api-keys/", + json={"name": "empty-reasoning-key", "allowedReasoningEfforts": []}, + ) + assert empty_create.status_code == 400 + + enabled = await async_client.put( + "/api/settings", + json={ + "stickyThreadsEnabled": False, + "preferEarlierResetAccounts": False, + "totpRequiredOnLogin": False, + "apiKeyAuthEnabled": True, + }, + ) + assert enabled.status_code == 200 + + blocked = await async_client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {key}"}, + json={ + "model": "model-alpha", + "instructions": "hello", + "input": [], + "reasoning": {"effort": "max"}, + }, + ) + assert blocked.status_code == 403 + assert blocked.json()["error"] == { + "message": "This API key does not have access to reasoning effort 'max'", + "type": "permission_error", + "code": "reasoning_effort_not_allowed", + "param": "reasoning.effort", + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["/backend-api/codex/responses/compact", "/v1/responses/compact"]) +async def test_api_key_reasoning_allowlist_rejects_compact_before_upstream(async_client, monkeypatch, endpoint): + enabled = await async_client.put( + "/api/settings", + json={ + "stickyThreadsEnabled": False, + "preferEarlierResetAccounts": False, + "totpRequiredOnLogin": False, + "apiKeyAuthEnabled": True, + }, + ) + assert enabled.status_code == 200 + created = await async_client.post( + "/api/api-keys/", + json={"name": "compact-allowlist-key", "allowedReasoningEfforts": ["low"]}, + ) + assert created.status_code == 200 + key = created.json()["key"] + + async def fail_upstream(*_args, **_kwargs): + raise AssertionError("compact upstream was reached after policy rejection") + + monkeypatch.setattr(proxy_module, "core_compact_responses", fail_upstream) + response = await async_client.post( + endpoint, + headers={"Authorization": f"Bearer {key}"}, + json={"model": "model-alpha", "instructions": "hi", "input": [], "reasoning": {"effort": "max"}}, + ) + + assert response.status_code == 403 + assert response.json()["error"]["code"] == "reasoning_effort_not_allowed" + + +@pytest.mark.asyncio +async def test_api_key_reasoning_allowlist_rejects_chat_completions_before_upstream(async_client, monkeypatch): + enabled = await async_client.put( + "/api/settings", + json={ + "stickyThreadsEnabled": False, + "preferEarlierResetAccounts": False, + "totpRequiredOnLogin": False, + "apiKeyAuthEnabled": True, + }, + ) + assert enabled.status_code == 200 + created = await async_client.post( + "/api/api-keys/", + json={"name": "chat-allowlist-key", "allowedReasoningEfforts": ["low"]}, + ) + assert created.status_code == 200 + key = created.json()["key"] + + async def fail_upstream(*_args, **_kwargs): + raise AssertionError("chat completions upstream was reached after policy rejection") + + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_upstream) + response = await async_client.post( + "/v1/chat/completions", + headers={"Authorization": f"Bearer {key}"}, + json={ + "model": "model-alpha", + "messages": [{"role": "user", "content": "hi"}], + "reasoning_effort": "max", + }, + ) + + assert response.status_code == 403 + assert response.json()["error"]["code"] == "reasoning_effort_not_allowed" + + @pytest.mark.asyncio async def test_stream_usage_logs_actual_service_tier(async_client, monkeypatch): enable = await async_client.put( @@ -3558,6 +3720,10 @@ async def fake_sqlite_writer_section(): stale_second = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") abandoned_before_first_heartbeat = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") fresh = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert stale is not None + assert stale_second is not None + assert abandoned_before_first_heartbeat is not None + assert fresh is not None await session.execute( update(ApiKeyUsageReservation) .where(ApiKeyUsageReservation.id.in_([stale.reservation_id, stale_second.reservation_id])) @@ -3607,6 +3773,127 @@ async def fake_sqlite_writer_section(): assert limits[0].current_value == fresh_reservation.items[0].reserved_delta +@pytest.mark.asyncio +async def test_limit_free_admission_creates_no_reservation_rows(async_client): + """Limit-free keys skip the reservation ledger: admission returns no + reservation, writes no rows, and stale-reservation reclamation has + nothing to release; limited keys keep creating reservations.""" + del async_client + now = utcnow() + + async with SessionLocal() as session: + repo = ApiKeysRepository(session) + service = ApiKeysService(repo) + unlimited = await service.create_key( + ApiKeyCreateData(name="limit-free-admission", allowed_models=None, expires_at=None) + ) + limited = await service.create_key( + ApiKeyCreateData( + name="limited-admission-regression", + allowed_models=None, + expires_at=None, + limits=[ + LimitRuleInput(limit_type="total_tokens", limit_window="weekly", max_value=50_000), + ], + ) + ) + unlimited_reservation = await service.enforce_limits_for_request(unlimited.id, request_model="gpt-5.1") + limited_reservation = await service.enforce_limits_for_request(limited.id, request_model="gpt-5.1") + + assert unlimited_reservation is None + assert limited_reservation is not None + assert limited_reservation.has_applicable_limits is True + + async with SessionLocal() as session: + rows = await session.execute( + select(ApiKeyUsageReservation).where(ApiKeyUsageReservation.api_key_id == unlimited.id) + ) + assert rows.scalars().all() == [] + repo = ApiKeysRepository(session) + limited_row = await repo.get_usage_reservation(limited_reservation.reservation_id) + assert limited_row is not None + assert limited_row.status == "reserved" + # A future cutoff would reclaim any reserved row; the limit-free key + # contributed none, so only the limited key's reservation is released. + released_count = await repo.release_stale_usage_reservations( + cutoff=now + timedelta(hours=1), + max_age_cutoff=now + timedelta(hours=1), + ) + assert released_count == 1 + + +@pytest.mark.asyncio +async def test_enforce_limits_lazy_reset_and_expiry_with_narrowed_admission_load(async_client): + """Regression for the narrowed admission load (``get_for_limit_enforcement``). + + The enforcement path loads only ``is_active``/``expires_at`` plus the + ``limits`` collection. The lazy expired-limit reset (which commits + mid-enforcement and refetches through the same narrowed load) and the + key-expiry rejection must behave exactly as with the full-graph load. + """ + del async_client + now = utcnow() + + async with SessionLocal() as session: + repo = ApiKeysRepository(session) + service = ApiKeysService(repo) + created = await service.create_key( + ApiKeyCreateData( + name="narrowed-admission-load", + allowed_models=None, + expires_at=None, + limits=[ + LimitRuleInput(limit_type="total_tokens", limit_window="daily", max_value=50_000), + ], + ) + ) + limits = await repo.get_limits_by_key(created.id) + assert len(limits) == 1 + # Exhausted AND expired: without the lazy reset the enforcement + # would reject; the reset must zero the counter and advance reset_at. + limits[0].current_value = 50_000 + limits[0].reset_at = now - timedelta(hours=2) + await session.commit() + + async with SessionLocal() as session: + repo = ApiKeysRepository(session) + service = ApiKeysService(repo) + reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None + assert reservation.has_applicable_limits is True + + async with SessionLocal() as session: + repo = ApiKeysRepository(session) + limits = await repo.get_limits_by_key(created.id) + assert len(limits) == 1 + assert limits[0].reset_at > now + reserved = await repo.get_usage_reservation(reservation.reservation_id) + assert reserved is not None + assert reserved.status == "reserved" + assert limits[0].current_value == reserved.items[0].reserved_delta + + async with SessionLocal() as session: + repo = ApiKeysRepository(session) + # Fail-loud contract of the narrowed load: unlisted columns and the + # assignment relationships raise instead of lazy loading. + row = await repo.get_for_limit_enforcement(created.id) + assert row is not None + assert row.is_active is True + assert row.expires_at is None + assert len(row.limits) == 1 + with pytest.raises(sqlalchemy_exc.InvalidRequestError): + _ = row.name + with pytest.raises(sqlalchemy_exc.InvalidRequestError): + _ = row.account_assignments + + async with SessionLocal() as session: + repo = ApiKeysRepository(session) + service = ApiKeysService(repo) + await repo.update(created.id, expires_at=now - timedelta(minutes=1)) + with pytest.raises(ApiKeyInvalidError): + await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + + @pytest.mark.asyncio async def test_release_stale_usage_reservations_max_age_ceiling_beats_orphaned_heartbeat(async_client, monkeypatch): """Issue #1594: a leaked heartbeat keeps refreshing ``updated_at`` forever. @@ -3636,6 +3923,8 @@ async def fake_sqlite_writer_section(): ) heartbeat_kept = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") fresh = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert heartbeat_kept is not None + assert fresh is not None # An orphaned heartbeat keeps the reservation's updated_at current # even though it was created past the hard age ceiling. await session.execute( diff --git a/tests/integration/test_auth_middleware.py b/tests/integration/test_auth_middleware.py index 641d79c7e0..de3448bbcb 100644 --- a/tests/integration/test_auth_middleware.py +++ b/tests/integration/test_auth_middleware.py @@ -20,7 +20,6 @@ from app.modules.api_keys.repository import ApiKeysRepository from app.modules.api_keys.service import ApiKeyCreateData, ApiKeysService from app.modules.dashboard_auth.service import DASHBOARD_SESSION_COOKIE, get_dashboard_session_store -from app.modules.proxy.rowless_recovery_repository import RowlessRecoveryRepository pytestmark = pytest.mark.integration @@ -972,110 +971,6 @@ async def test_disabled_dashboard_auth_mode_bypasses_guard_and_disables_password assert disable_totp.json()["error"]["code"] == "password_management_disabled" -@pytest.mark.asyncio -async def test_rowless_recovery_admin_surface_requires_trusted_proxy_operator( - async_client, - app_instance, - monkeypatch, -): - _set_dashboard_auth_env(monkeypatch, mode=DashboardAuthMode.DISABLED) - disabled = await async_client.get("/api/http-bridge/rowless-recovery") - assert disabled.status_code == 403 - - _set_dashboard_auth_env( - monkeypatch, - mode=DashboardAuthMode.TRUSTED_HEADER, - trust_proxy_headers=True, - ) - without_actor = await async_client.get("/api/http-bridge/rowless-recovery") - assert without_actor.status_code == 401 - - bearer_only = await async_client.get( - "/api/http-bridge/rowless-recovery", - headers={"Authorization": "Bearer ordinary-proxy-key"}, - ) - assert bearer_only.status_code == 401 - - allowed = await async_client.get( - "/api/http-bridge/rowless-recovery", - headers={"Remote-User": "operator@example.com"}, - ) - assert allowed.status_code == 200 - assert allowed.json() == [] - status_response = await async_client.get( - "/api/http-bridge/rowless-recovery/status", - headers={"Remote-User": "operator@example.com"}, - ) - assert status_response.status_code == 200 - assert status_response.json() == { - "stateCounts": {"captured": 0, "approved": 0, "unknown": 0, "consumed": 0}, - "markerBoundStateCounts": {"captured": 0, "approved": 0, "unknown": 0, "consumed": 0}, - "replayFenceCount": 0, - "markerBoundAuthorityCount": 0, - "activeAutomaticAuthorityCount": 0, - "preRowlessImageCompatible": True, - "preMarkerRecoveryImageCompatible": True, - "preAutomaticRecoveryImageCompatible": True, - "minimumRollbackCapability": None, - } - - async def marker_bound_counts(self, *, marker_bound_only=False): - del self, marker_bound_only - return { - "captured": 1, - "approved": 0, - "unknown": 0, - "consumed": 0, - } - - with monkeypatch.context() as status_patch: - status_patch.setattr(RowlessRecoveryRepository, "authority_state_counts", marker_bound_counts) - marker_status = await async_client.get( - "/api/http-bridge/rowless-recovery/status", - headers={"Remote-User": "operator@example.com"}, - ) - assert marker_status.status_code == 200 - assert marker_status.json()["markerBoundAuthorityCount"] == 1 - assert marker_status.json()["preRowlessImageCompatible"] is False - assert marker_status.json()["preMarkerRecoveryImageCompatible"] is False - assert marker_status.json()["preAutomaticRecoveryImageCompatible"] is True - assert marker_status.json()["minimumRollbackCapability"] == "rowless_marker_recovery_v2" - - async def one_active_automatic(self): - del self - return 1 - - with monkeypatch.context() as status_patch: - status_patch.setattr(RowlessRecoveryRepository, "active_automatic_authority_count", one_active_automatic) - automatic_status = await async_client.get( - "/api/http-bridge/rowless-recovery/status", - headers={"Remote-User": "operator@example.com"}, - ) - assert automatic_status.status_code == 200 - assert automatic_status.json()["activeAutomaticAuthorityCount"] == 1 - assert automatic_status.json()["preAutomaticRecoveryImageCompatible"] is False - assert automatic_status.json()["minimumRollbackCapability"] == "rowless_automatic_recovery_v3" - - _set_dashboard_auth_env( - monkeypatch, - mode=DashboardAuthMode.TRUSTED_HEADER, - trust_proxy_headers=True, - trusted_proxy_cidrs="10.0.0.0/8", - ) - remote_transport = ASGITransport(app=app_instance, client=("203.0.113.24", 50001)) - async with AsyncClient(transport=remote_transport, base_url="http://lb.example") as remote_client: - spoofed = await remote_client.get( - "/api/http-bridge/rowless-recovery", - headers={"Remote-User": "attacker@example.com"}, - ) - spoofed_status = await remote_client.get( - "/api/http-bridge/rowless-recovery/status", - headers={"Remote-User": "attacker@example.com"}, - ) - assert spoofed.status_code == 401 - assert spoofed_status.status_code == 401 - - @pytest.mark.asyncio async def test_trusted_header_proxy_auth_with_fallback_password_reports_no_active_session(async_client, monkeypatch): """Proxy-authenticated user with configured fallback password must see passwordSessionActive=False.""" diff --git a/tests/integration/test_cache_invalidation_bus.py b/tests/integration/test_cache_invalidation_bus.py index fd38a4998f..199289b31c 100644 --- a/tests/integration/test_cache_invalidation_bus.py +++ b/tests/integration/test_cache_invalidation_bus.py @@ -386,6 +386,66 @@ def test_namespace_log_labels_cover_all_namespaces() -> None: } +@pytest.mark.asyncio +async def test_pending_bump_survives_a_cancelled_flush(db_setup, monkeypatch) -> None: + """The marker is cleared before the write is awaited, so a cancelled write + must restore it — otherwise the namespace is neither written nor pending + and no later cycle can retry it. (At process stop no cycle remains either + way; shutdown delivery is explicitly out of scope, and the restore there + only keeps the pending set honest.)""" + namespace = "test_flush_cancelled" + started = asyncio.Event() + + async def never_finishes(ns: str) -> bool: + started.set() + await asyncio.Event().wait() + return True + + poller = CacheInvalidationPoller(SessionLocal) + monkeypatch.setattr(poller, "bump", never_finishes) + poller.request_bump(namespace) + + flush_task = asyncio.create_task(poller._flush_pending_bumps()) + await asyncio.wait_for(started.wait(), timeout=2.0) + assert namespace not in poller._pending_bumps, "marker is cleared before the write, by design" + + flush_task.cancel() + with pytest.raises(asyncio.CancelledError): + await flush_task + + assert namespace in poller._pending_bumps + assert await _namespace_version(namespace) is None + + +@pytest.mark.asyncio +async def test_pending_bump_survives_a_raising_flush_and_does_not_starve_others(db_setup, monkeypatch) -> None: + """A raise is abnormal (bump() reports failure by returning False), so it + must not abort the flush: the loop is sorted, and a persistently raising + namespace sorting first would otherwise starve every namespace after it + on every cycle. The raiser stays pending; the rest still land.""" + raising_namespace = "test_flush_raised_a" + healthy_namespace = "test_flush_raised_b" + real_bump = CacheInvalidationPoller.bump + + poller = CacheInvalidationPoller(SessionLocal) + + async def raising_for_one(ns: str) -> bool: + if ns == raising_namespace: + raise RuntimeError("driver exploded") + return await real_bump(poller, ns) + + monkeypatch.setattr(poller, "bump", raising_for_one) + poller.request_bump(raising_namespace) + poller.request_bump(healthy_namespace) + + await poller._flush_pending_bumps() + + assert raising_namespace in poller._pending_bumps + assert await _namespace_version(raising_namespace) is None + assert healthy_namespace not in poller._pending_bumps + assert await _namespace_version(healthy_namespace) == 1 + + @pytest.mark.asyncio async def test_pending_coalesced_bump_flushes_after_recovery(db_setup) -> None: namespace = "test_pending_flush" @@ -793,3 +853,41 @@ async def test_inflight_poll_does_not_clobber_concurrent_local_bump(db_setup) -> await source._poll_once() assert source_calls == [] assert source._known_versions.get(NAMESPACE_RESET_CREDITS) == 2 + + +@pytest.mark.asyncio +async def test_aborted_bump_is_retried_by_the_running_poller(db_setup, monkeypatch) -> None: + """End-to-end: the point of restoring the marker is that the background + poller actually retries. Without the restore the first raise loses the + namespace and no later cycle ever writes its version.""" + namespace = "test_abort_retried_by_poller" + attempts = 0 + real_bump = CacheInvalidationPoller.bump + + poller = CacheInvalidationPoller(SessionLocal, poll_interval_seconds=0.01) + + retry_settled = asyncio.Event() + + async def failing_then_real(ns: str) -> bool: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise RuntimeError("driver exploded") + result = await real_bump(poller, ns) + # Signal only after bump() fully returns: the committed row can become + # visible while the shielded session cleanup is still running, and + # stopping the poller at that instant would cancel the retry mid-flight. + retry_settled.set() + return result + + monkeypatch.setattr(poller, "bump", failing_then_real) + poller.request_bump(namespace) + await poller.start() + try: + await asyncio.wait_for(retry_settled.wait(), timeout=5.0) + finally: + await poller.stop() + + assert attempts >= 2, "the poller must retry the aborted namespace" + assert await _namespace_version(namespace) == 1 + assert namespace not in poller._pending_bumps diff --git a/tests/integration/test_daybreak_capability_routes.py b/tests/integration/test_daybreak_capability_routes.py new file mode 100644 index 0000000000..3b976dd2da --- /dev/null +++ b/tests/integration/test_daybreak_capability_routes.py @@ -0,0 +1,778 @@ +from __future__ import annotations + +import base64 +from collections.abc import Mapping +from typing import Any + +import pytest +from fastapi.routing import APIRoute +from fastapi.testclient import TestClient +from httpx import AsyncClient, Response +from starlette.routing import WebSocketRoute +from starlette.testclient import WebSocketDenialResponse + +import app.modules.proxy.api as proxy_api_module +from app.core.auth import dependencies as auth_dependencies +from app.core.clients.proxy import CODEX_LB_REQUIRED_CAPABILITY_HEADER, CodexControlResponse +from app.db.session import SessionLocal +from app.modules.api_keys.repository import ApiKeysRepository +from app.modules.api_keys.service import ApiKeyCreateData, ApiKeysService +from app.modules.proxy.service import ProxyService + +pytestmark = pytest.mark.integration + +_CAPABILITY_HEADERS = {CODEX_LB_REQUIRED_CAPABILITY_HEADER: "trusted_cyber"} +_TRANSPORT_DENIAL = { + "error": { + "code": "required_capability_transport_unsupported", + "message": "Required capability routing is only supported over the Responses WebSocket transport.", + "type": "invalid_request_error", + } +} +_IMAGE_BYTES = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16 +_IMAGE_DATA_URL = f"data:image/png;base64,{base64.b64encode(_IMAGE_BYTES).decode('ascii')}" + +_RouteKey = tuple[str, str, str] +_FAIL_CLOSED_HTTP_ROUTES: frozenset[_RouteKey] = frozenset( + { + ("HTTP", "POST", "/backend-api/codex/realtime/calls"), + ("HTTP", "POST", "/backend-api/codex/thread/goal/get"), + ("HTTP", "GET", "/backend-api/codex/thread/goal/get"), + ("HTTP", "POST", "/backend-api/codex/thread/goal/set"), + ("HTTP", "POST", "/backend-api/codex/thread/goal/clear"), + ("HTTP", "POST", "/backend-api/codex/analytics-events/events"), + ("HTTP", "POST", "/backend-api/codex/memories/trace_summarize"), + ("HTTP", "POST", "/backend-api/codex/safety/arc"), + ("HTTP", "POST", "/backend-api/codex/alpha/search"), + ("HTTP", "GET", "/backend-api/codex/agent-identities/jwks"), + ("HTTP", "POST", "/backend-api/codex/responses"), + ("HTTP", "POST", "/backend-api/codex/responses/"), + ("HTTP", "GET", "/backend-api/codex/opportunistic/admission"), + ("HTTP", "POST", "/backend-api/codex/images/generations"), + ("HTTP", "POST", "/backend-api/codex/images/edits"), + ("HTTP", "POST", "/backend-api/codex/responses/compact"), + ("HTTP", "POST", "/internal/bridge/responses"), + ("HTTP", "POST", "/v1/responses"), + ("HTTP", "POST", "/v1/responses/"), + ("HTTP", "POST", "/v1/warmup"), + ("HTTP", "POST", "/v1/warmup/{mode}"), + ("HTTP", "POST", "/v1/audio/transcriptions"), + ("HTTP", "POST", "/v1/images/generations"), + ("HTTP", "POST", "/v1/images/edits"), + ("HTTP", "POST", "/v1/chat/completions"), + ("HTTP", "POST", "/v1/embeddings"), + ("HTTP", "POST", "/v1/responses/compact"), + ("HTTP", "POST", "/backend-api/transcribe"), + ("HTTP", "POST", "/backend-api/files"), + ("HTTP", "POST", "/backend-api/files/{file_id}/uploaded"), + ("HTTP", "POST", "/v1/reset-credit"), + ("HTTP", "POST", "/api/codex/rate-limit-reset-credits/consume/"), + ("HTTP", "POST", "/api/codex/rate-limit-reset-credits/consume"), + } +) +_LOCAL_AUTHENTICATED_ROUTES: frozenset[_RouteKey] = frozenset( + { + ("HTTP", "GET", "/backend-api/codex/models"), + ("HTTP", "GET", "/v1/models"), + ("HTTP", "GET", "/v1/usage"), + ("HTTP", "POST", "/v1/images/variations"), + ("HTTP", "GET", "/v1/reset-credit"), + ("HTTP", "GET", "/api/codex/usage/"), + ("HTTP", "GET", "/api/codex/usage"), + } +) +_RESPONSES_WEBSOCKET_ROUTES: frozenset[_RouteKey] = frozenset( + { + ("WS", "WEBSOCKET", "/backend-api/codex/responses"), + ("WS", "WEBSOCKET", "/v1/responses"), + } +) +_FAIL_CLOSED_WEBSOCKET_ROUTES: frozenset[_RouteKey] = frozenset( + { + ("WS", "WEBSOCKET", "/backend-api/codex/{call_id:realtime_live_call_id}"), + ("WS", "WEBSOCKET", "/v1/live/{call_id:realtime_live_call_id}"), + ("WS", "WEBSOCKET", "/v1/realtime"), + } +) +_SEPARATE_NAMESPACE_ROUTES: frozenset[_RouteKey] = frozenset( + { + ("HTTP", "GET", "/backend-api/wham/agent-identities/jwks"), + } +) + + +def test_registered_proxy_route_inventory_has_one_explicit_capability_policy(app_instance) -> None: + registered: set[_RouteKey] = set() + for route in app_instance.routes: + endpoint = getattr(route, "endpoint", None) + if getattr(endpoint, "__module__", None) != proxy_api_module.__name__: + continue + if isinstance(route, APIRoute): + registered.update(("HTTP", method, route.path) for method in route.methods or ()) + elif isinstance(route, WebSocketRoute): + registered.add(("WS", "WEBSOCKET", route.path)) + + policy_groups = ( + _FAIL_CLOSED_HTTP_ROUTES, + _LOCAL_AUTHENTICATED_ROUTES, + _RESPONSES_WEBSOCKET_ROUTES, + _FAIL_CLOSED_WEBSOCKET_ROUTES, + _SEPARATE_NAMESPACE_ROUTES, + ) + classified = frozenset().union(*policy_groups) + + assert registered == classified + assert sum(len(group) for group in policy_groups) == len(classified) + + +async def _create_api_key(name: str) -> str: + async with SessionLocal() as session: + created = await ApiKeysService(ApiKeysRepository(session)).create_key( + ApiKeyCreateData(name=name, allowed_models=None) + ) + return created.key + + +async def _request( + async_client: AsyncClient, + method: str, + path: str, + *, + headers: Mapping[str, str], + request_kwargs: Mapping[str, Any], +) -> Response: + return await async_client.request(method, path, headers=headers, **request_kwargs) + + +_PROVIDER_ROUTING_CASES = [ + pytest.param("GET", "/backend-api/codex/thread/goal/get", {}, id="thread-goal-get"), + pytest.param("POST", "/backend-api/codex/thread/goal/get", {"json": {}}, id="thread-goal-get-post"), + pytest.param("POST", "/backend-api/codex/thread/goal/set", {"json": {}}, id="thread-goal-set"), + pytest.param("POST", "/backend-api/codex/thread/goal/clear", {"json": {}}, id="thread-goal-clear"), + pytest.param( + "POST", + "/backend-api/codex/analytics-events/events", + {"json": {}}, + id="analytics-events", + ), + pytest.param( + "POST", + "/backend-api/codex/memories/trace_summarize", + {"json": {}}, + id="memory-trace", + ), + pytest.param("POST", "/backend-api/codex/realtime/calls", {"content": b"v=offer\r\n"}, id="realtime-call"), + pytest.param("POST", "/backend-api/codex/safety/arc", {"json": {}}, id="safety-arc"), + pytest.param("POST", "/backend-api/codex/alpha/search", {"json": {}}, id="alpha-search"), + pytest.param("GET", "/backend-api/codex/agent-identities/jwks", {}, id="agent-identities"), + pytest.param("GET", "/backend-api/codex/opportunistic/admission", {}, id="opportunistic-admission"), + pytest.param("POST", "/v1/warmup", {"json": {"mode": "normal"}}, id="warmup-body"), + pytest.param("POST", "/v1/warmup/normal", {}, id="warmup-path"), + pytest.param( + "POST", + "/v1/chat/completions", + {"json": {"model": "gpt-5.6-sol", "messages": [{"role": "user", "content": "inert"}]}}, + id="chat-completions", + ), + pytest.param( + "POST", + "/v1/embeddings", + {"json": {"model": "text-embedding-3-small", "input": "inert"}}, + id="embeddings", + ), +] + +_PROVIDER_BINARY_ROUTE_CASES = [ + pytest.param( + "POST", + "/backend-api/files", + {"json": {"file_name": "inert.txt", "file_size": 1, "use_case": "codex"}}, + id="files-create", + ), + pytest.param( + "POST", + "/backend-api/files/file_inert/uploaded", + {"json": {}}, + id="files-finalize", + ), + pytest.param( + "POST", + "/backend-api/transcribe", + {"files": {"file": ("inert.wav", b"inert", "audio/wav")}}, + id="native-transcription", + ), + pytest.param( + "POST", + "/v1/audio/transcriptions", + { + "data": {"model": "gpt-4o-transcribe"}, + "files": {"file": ("inert.wav", b"inert", "audio/wav")}, + }, + id="v1-transcription", + ), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("method", "path", "request_kwargs"), _PROVIDER_ROUTING_CASES) +async def test_daybreak_capability_fails_closed_on_unsupported_routing_http_surfaces( + async_client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, + method: str, + path: str, + request_kwargs: Mapping[str, Any], +) -> None: + async def fail_before_routing(*_args: Any, **_kwargs: Any) -> None: + pytest.fail("capability-bearing provider request must fail before routing") + + monkeypatch.setattr(ProxyService, "thread_goal_request", fail_before_routing) + monkeypatch.setattr(ProxyService, "codex_control_request", fail_before_routing) + monkeypatch.setattr(proxy_api_module, "_opportunistic_admission_denial", fail_before_routing) + monkeypatch.setattr(proxy_api_module, "_select_chat_model_source", fail_before_routing) + monkeypatch.setattr(proxy_api_module, "_select_embeddings_model_source", fail_before_routing) + key = await _create_api_key(f"Daybreak route guard {path}") + + response = await _request( + async_client, + method, + path, + headers={"Authorization": f"Bearer {key}", **_CAPABILITY_HEADERS}, + request_kwargs=request_kwargs, + ) + + assert response.status_code == 400 + assert response.json() == _TRANSPORT_DENIAL + + +@pytest.mark.asyncio +@pytest.mark.parametrize("auth_state", ["missing", "invalid"]) +async def test_daybreak_capability_unsupported_http_authenticates_before_denial( + async_client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, + auth_state: str, +) -> None: + async def fail_before_routing(*_args: Any, **_kwargs: Any) -> None: + pytest.fail("unauthenticated capability intent must fail before routing") + + monkeypatch.setattr(ProxyService, "thread_goal_request", fail_before_routing) + headers = dict(_CAPABILITY_HEADERS) + if auth_state == "invalid": + headers["Authorization"] = "Bearer invalid-daybreak-key" + + response = await async_client.get( + "/backend-api/codex/thread/goal/get", + headers=headers, + ) + + assert response.status_code == 401 + assert response.json()["error"]["code"] == "invalid_api_key" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", + [ + "/v1/responses", + "/backend-api/codex/responses", + "/v1/chat/completions", + "/v1/embeddings", + "/v1/images/generations", + "/v1/warmup", + "/v1/warmup/default", + ], +) +@pytest.mark.parametrize("auth_state", ["valid", "missing", "invalid"]) +async def test_daybreak_json_routes_reject_capability_before_body_validation( + async_client: AsyncClient, + path: str, + auth_state: str, +) -> None: + headers = { + **_CAPABILITY_HEADERS, + "Content-Type": "application/json", + } + if auth_state == "valid": + key = await _create_api_key(f"Daybreak pre-body guard {path}") + headers["Authorization"] = f"Bearer {key}" + elif auth_state == "invalid": + headers["Authorization"] = "Bearer invalid-daybreak-key" + + response = await async_client.post(path, headers=headers, content=b"{") + + if auth_state == "valid": + assert response.status_code == 400 + assert response.json() == _TRANSPORT_DENIAL + return + assert response.status_code == 401 + assert response.json()["error"]["code"] == "invalid_api_key" + + +@pytest.mark.asyncio +async def test_daybreak_capability_does_not_bypass_api_firewall(async_client: AsyncClient) -> None: + add_response = await async_client.post("/api/firewall/ips", json={"ipAddress": "10.20.30.40"}) + assert add_response.status_code == 200 + key = await _create_api_key("Daybreak firewall") + + response = await async_client.post( + "/v1/responses", + headers={ + "Authorization": f"Bearer {key}", + **_CAPABILITY_HEADERS, + "Content-Type": "application/json", + }, + content=b"{", + ) + + assert response.status_code == 403 + assert response.json()["error"]["code"] == "ip_forbidden" + + +_RESET_CREDIT_CONSUME_CASES = [ + pytest.param( + "/v1/reset-credit", + {"account_id": "acct_inert", "redeem_id": "credit_inert"}, + id="self-service-reset-credit", + ), + pytest.param( + "/api/codex/rate-limit-reset-credits/consume", + {"redeem_request_id": "redeem_inert"}, + id="codex-usage-reset-credit", + ), + pytest.param( + "/api/codex/rate-limit-reset-credits/consume/", + {"redeem_request_id": "redeem_inert"}, + id="codex-usage-reset-credit-slash", + ), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("path", "payload"), _RESET_CREDIT_CONSUME_CASES) +@pytest.mark.parametrize("auth_state", ["valid", "missing", "invalid"]) +async def test_daybreak_capability_reset_credit_consumes_authenticate_then_fail_closed( + async_client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, + path: str, + payload: Mapping[str, str], + auth_state: str, +) -> None: + async def fail_before_identity_or_account_io(*_args: Any, **_kwargs: Any) -> None: + pytest.fail("capability-bearing reset-credit request must fail before identity, account, or upstream I/O") + + monkeypatch.setattr(auth_dependencies, "fetch_usage", fail_before_identity_or_account_io) + monkeypatch.setattr(proxy_api_module.AccountsRepository, "get_by_id", fail_before_identity_or_account_io) + monkeypatch.setattr(proxy_api_module, "_fetch_authoritative_reset_credit", fail_before_identity_or_account_io) + monkeypatch.setattr(proxy_api_module, "_ensure_v1_reset_credit_account_fresh", fail_before_identity_or_account_io) + monkeypatch.setattr( + proxy_api_module, + "_consume_rate_limit_reset_credit_for_request", + fail_before_identity_or_account_io, + ) + monkeypatch.setattr(proxy_api_module, "consume_reset_credit", fail_before_identity_or_account_io) + headers = dict(_CAPABILITY_HEADERS) + if auth_state == "valid": + key = await _create_api_key(f"Daybreak reset-credit guard {path}") + headers["Authorization"] = f"Bearer {key}" + elif auth_state == "invalid": + headers["Authorization"] = "Bearer invalid-daybreak-key" + + response = await async_client.post(path, headers=headers, json=payload) + + if auth_state == "valid": + assert response.status_code == 400 + assert response.json() == _TRANSPORT_DENIAL + else: + assert response.status_code == 401 + assert response.json()["error"]["code"] == "invalid_api_key" + + +@pytest.mark.asyncio +async def test_headerless_reset_credit_routes_keep_existing_auth_and_account_behavior( + async_client: AsyncClient, +) -> None: + key = await _create_api_key("Ordinary reset-credit behavior") + headers = {"Authorization": f"Bearer {key}"} + + self_service = await async_client.post( + "/v1/reset-credit", + headers=headers, + json={"account_id": "acct_missing", "redeem_id": "credit_missing"}, + ) + codex_usage = await async_client.post( + "/api/codex/rate-limit-reset-credits/consume", + headers=headers, + json={"redeem_request_id": "redeem_inert"}, + ) + + assert self_service.status_code == 403 + assert self_service.json()["error"]["code"] != "required_capability_transport_unsupported" + assert codex_usage.status_code == 401 + assert codex_usage.json()["error"]["code"] == "invalid_api_key" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("auth_state", ["valid", "missing", "invalid"]) +async def test_daybreak_capability_cannot_be_appended_to_signed_internal_bridge( + async_client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, + auth_state: str, +) -> None: + from app.core.config.settings import get_settings + from app.core.openai.requests import ResponsesRequest + from app.modules.proxy.http_bridge_forwarding import HTTPBridgeForwardContext, build_owner_forward_headers + + async def fail_before_bridge_routing(*_args: Any, **_kwargs: Any) -> None: + pytest.fail("capability-bearing internal bridge request must fail before account routing") + + monkeypatch.setattr(ProxyService, "validate_http_bridge_legacy_forward_anchor", fail_before_bridge_routing) + monkeypatch.setattr(proxy_api_module, "_stream_responses", fail_before_bridge_routing) + payload = ResponsesRequest.model_validate( + {"model": "gpt-5.6-sol", "instructions": "", "input": "inert", "stream": True} + ) + context = HTTPBridgeForwardContext( + origin_instance="origin-inert", + target_instance=get_settings().http_responses_session_bridge_instance_id, + codex_session_affinity=True, + downstream_turn_state="turn_inert", + ) + authorization: str | None = None + if auth_state == "valid": + key = await _create_api_key("Daybreak internal bridge guard") + authorization = f"Bearer {key}" + elif auth_state == "invalid": + authorization = "Bearer invalid-daybreak-key" + inbound_headers = {} if authorization is None else {"authorization": authorization} + headers = build_owner_forward_headers(headers=inbound_headers, payload=payload, context=context) + headers[CODEX_LB_REQUIRED_CAPABILITY_HEADER] = "trusted_cyber" + + response = await async_client.post( + "/internal/bridge/responses", + headers=headers, + json=payload.model_dump_for_forwarding(), + ) + + if auth_state == "valid": + assert response.status_code == 400 + assert response.json() == _TRANSPORT_DENIAL + else: + assert response.status_code == 401 + assert response.json()["error"]["code"] == "invalid_api_key" + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("method", "path", "request_kwargs"), _PROVIDER_BINARY_ROUTE_CASES) +@pytest.mark.parametrize("auth_state", ["valid", "missing", "invalid"]) +async def test_daybreak_capability_fails_closed_before_provider_body_or_account_routing( + async_client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, + method: str, + path: str, + request_kwargs: Mapping[str, Any], + auth_state: str, +) -> None: + async def fail_before_routing(*_args: Any, **_kwargs: Any) -> None: + pytest.fail("capability-bearing provider request must fail before body parsing or routing") + + monkeypatch.setattr(proxy_api_module, "_parse_transcription_multipart", fail_before_routing) + monkeypatch.setattr(ProxyService, "create_file", fail_before_routing) + monkeypatch.setattr(ProxyService, "finalize_file", fail_before_routing) + monkeypatch.setattr(ProxyService, "transcribe", fail_before_routing) + headers = dict(_CAPABILITY_HEADERS) + if auth_state == "valid": + key = await _create_api_key(f"Daybreak binary route guard {path}") + headers["Authorization"] = f"Bearer {key}" + elif auth_state == "invalid": + headers["Authorization"] = "Bearer invalid-daybreak-key" + + response = await _request( + async_client, + method, + path, + headers=headers, + request_kwargs=request_kwargs, + ) + + if auth_state == "valid": + assert response.status_code == 400 + assert response.json() == _TRANSPORT_DENIAL + else: + assert response.status_code == 401 + assert response.json()["error"]["code"] == "invalid_api_key" + + +@pytest.mark.asyncio +async def test_headerless_provider_http_keeps_existing_routing_behavior( + async_client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = 0 + + async def ordinary_thread_goal(*_args: Any, **_kwargs: Any) -> dict[str, str]: + nonlocal calls + calls += 1 + return {"route": "ordinary"} + + monkeypatch.setattr(ProxyService, "thread_goal_request", ordinary_thread_goal) + + response = await async_client.get("/backend-api/codex/thread/goal/get") + + assert response.status_code == 200 + assert response.json() == {"route": "ordinary"} + assert calls == 1 + + +_IMAGE_CASES = [ + pytest.param( + "POST", + "/backend-api/codex/images/generations", + {"json": {"model": "gpt-image-2", "prompt": "inert"}}, + id="native-generation", + ), + pytest.param( + "POST", + "/v1/images/generations", + {"json": {"model": "gpt-image-2", "prompt": "inert"}}, + id="v1-generation", + ), + pytest.param( + "POST", + "/backend-api/codex/v1/images/generations", + {"json": {"model": "gpt-image-2", "prompt": "inert"}}, + id="rewritten-native-generation", + ), + pytest.param( + "POST", + "/backend-api/codex/images/edits", + { + "json": { + "model": "gpt-image-2", + "prompt": "inert", + "images": [{"image_url": _IMAGE_DATA_URL}], + } + }, + id="native-edit", + ), + pytest.param( + "POST", + "/v1/images/edits", + { + "data": {"model": "gpt-image-2", "prompt": "inert"}, + "files": {"image": ("inert.png", _IMAGE_BYTES, "image/png")}, + }, + id="v1-edit", + ), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("method", "path", "request_kwargs"), _IMAGE_CASES) +@pytest.mark.parametrize("auth_state", ["valid", "missing", "invalid"]) +async def test_daybreak_capability_image_routes_authenticate_then_fail_closed( + async_client: AsyncClient, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + method: str, + path: str, + request_kwargs: Mapping[str, Any], + auth_state: str, +) -> None: + async def fail_before_routing(*_args: Any, **_kwargs: Any) -> None: + pytest.fail("capability-bearing image request must fail before image routing") + + monkeypatch.setattr(proxy_api_module, "_proxy_images_generation_request", fail_before_routing) + monkeypatch.setattr(proxy_api_module, "_proxy_images_edit_request", fail_before_routing) + headers = dict(_CAPABILITY_HEADERS) + if auth_state == "valid": + key = await _create_api_key(f"Daybreak image guard {path}") + headers["Authorization"] = f"Bearer {key}" + elif auth_state == "invalid": + headers["Authorization"] = "Bearer invalid-daybreak-key" + + with caplog.at_level("WARNING", logger="app.modules.proxy.api"): + response = await _request( + async_client, + method, + path, + headers=headers, + request_kwargs=request_kwargs, + ) + + if auth_state == "valid": + assert response.status_code == 400 + assert response.json() == _TRANSPORT_DENIAL + route = "edits" if path.endswith("/edits") else "generations" + assert caplog.text.count(f"images_route_complete route={route} ") == 1 + assert "status=400 outcome=invalid_request" in caplog.text + else: + assert response.status_code == 401 + assert response.json()["error"]["code"] == "invalid_api_key" + + +@pytest.mark.asyncio +async def test_headerless_image_route_keeps_existing_validation_behavior(async_client: AsyncClient) -> None: + response = await async_client.post( + "/backend-api/codex/images/generations", + json={"model": "not-an-image-model", "prompt": "inert"}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] != "required_capability_transport_unsupported" + + +_LOCAL_ROUTE_CASES = [ + pytest.param("GET", "/backend-api/codex/models", 200, id="native-models"), + pytest.param("GET", "/v1/models", 200, id="v1-models"), + pytest.param("GET", "/v1/usage", 200, id="self-service-usage"), + pytest.param("POST", "/v1/images/variations", 404, id="unsupported-image-variations"), + pytest.param("GET", "/v1/reset-credit", 200, id="self-service-reset-credit-list"), + pytest.param("GET", "/api/codex/usage", 200, id="codex-usage"), + pytest.param("GET", "/api/codex/usage/", 200, id="codex-usage-slash"), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("method", "path", "expected_status"), _LOCAL_ROUTE_CASES) +async def test_daybreak_capability_allows_authenticated_local_routes_without_account_routing( + async_client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, + method: str, + path: str, + expected_status: int, +) -> None: + async def fail_before_account_or_upstream_io(*_args: Any, **_kwargs: Any) -> None: + pytest.fail("local capability-bearing route must not select an account or call upstream") + + monkeypatch.setattr(auth_dependencies, "fetch_usage", fail_before_account_or_upstream_io) + monkeypatch.setattr(ProxyService, "_select_account_with_budget", fail_before_account_or_upstream_io) + monkeypatch.setattr(ProxyService, "get_rate_limit_payload", fail_before_account_or_upstream_io) + key = await _create_api_key(f"Daybreak local route {path}") + + response = await async_client.request( + method, + path, + headers={"Authorization": f"Bearer {key}", **_CAPABILITY_HEADERS}, + ) + + assert response.status_code == expected_status + if expected_status == 404: + assert response.json()["error"]["code"] != "required_capability_transport_unsupported" + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("method", "path", "_expected_status"), _LOCAL_ROUTE_CASES) +@pytest.mark.parametrize("auth_state", ["missing", "invalid"]) +async def test_daybreak_local_routes_authenticate_capability_carrier( + async_client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, + method: str, + path: str, + _expected_status: int, + auth_state: str, +) -> None: + async def fail_before_upstream_identity_io(*_args: Any, **_kwargs: Any) -> None: + pytest.fail("unauthenticated capability carrier must fail before upstream identity I/O") + + monkeypatch.setattr(auth_dependencies, "fetch_usage", fail_before_upstream_identity_io) + headers = dict(_CAPABILITY_HEADERS) + if auth_state == "invalid": + headers["Authorization"] = "Bearer invalid-daybreak-key" + + response = await async_client.request(method, path, headers=headers) + + assert response.status_code == 401 + assert response.json()["error"]["code"] == "invalid_api_key" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("auth_state", ["missing", "invalid", "valid"]) +async def test_capability_header_outside_codex_provider_namespace_keeps_existing_behavior( + async_client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, + auth_state: str, +) -> None: + calls = 0 + + async def ordinary_wham_control(*_args: Any, **_kwargs: Any) -> CodexControlResponse: + nonlocal calls + calls += 1 + return CodexControlResponse(status_code=200, body=b"{}", headers={"content-type": "application/json"}) + + monkeypatch.setattr(ProxyService, "codex_control_request", ordinary_wham_control) + headers = dict(_CAPABILITY_HEADERS) + if auth_state == "valid": + key = await _create_api_key("WHAM namespace control") + headers["Authorization"] = f"Bearer {key}" + elif auth_state == "invalid": + headers["Authorization"] = "Bearer invalid-wham-key" + + response = await async_client.get( + "/backend-api/wham/agent-identities/jwks", + headers=headers, + ) + + if auth_state == "valid": + assert response.status_code == 200 + assert response.json() == {} + assert calls == 1 + return + assert response.status_code == 401 + assert response.json()["error"]["code"] == "invalid_api_key" + assert calls == 0 + + +@pytest.mark.parametrize( + "path", + [ + "/v1/live/rtc_daybreak_guard", + "/backend-api/codex/rtc_daybreak_guard", + "/backend-api/codex/v1/rtc_daybreak_guard", + "/v1/realtime?call_id=rtc_daybreak_guard", + ], + ids=["v1-live", "native-live", "native-v1-alias-live", "v1-realtime"], +) +@pytest.mark.parametrize("auth_state", ["valid", "missing", "invalid"]) +def test_daybreak_capability_fails_closed_on_non_responses_websockets( + app_instance, + monkeypatch: pytest.MonkeyPatch, + path: str, + auth_state: str, +) -> None: + async def fail_before_owner_lookup(*_args: Any, **_kwargs: Any) -> None: + pytest.fail("capability-bearing Live WebSocket must fail before owner lookup") + + monkeypatch.setattr(ProxyService, "proxy_realtime_live_websocket", fail_before_owner_lookup) + with TestClient(app_instance, client=("127.0.0.1", 50000)) as client: + assert client.portal is not None + headers = dict(_CAPABILITY_HEADERS) + if auth_state == "valid": + key = client.portal.call(_create_api_key, f"Daybreak live guard {path}") + headers["Authorization"] = f"Bearer {key}" + elif auth_state == "invalid": + headers["Authorization"] = "Bearer invalid-daybreak-key" + + with pytest.raises(WebSocketDenialResponse) as denial: + with client.websocket_connect(path, headers=headers): + pytest.fail("unsupported capability-bearing WebSocket must not connect") + + if auth_state == "valid": + assert denial.value.status_code == 400 + assert denial.value.json() == _TRANSPORT_DENIAL + else: + assert denial.value.status_code == 401 + assert denial.value.json()["error"]["code"] == "invalid_api_key" + + +def test_headerless_live_websocket_keeps_existing_owner_lookup_behavior(app_instance) -> None: + with TestClient(app_instance, client=("127.0.0.1", 50000)) as client: + assert client.portal is not None + key = client.portal.call(_create_api_key, "Ordinary live control") + + with pytest.raises(WebSocketDenialResponse) as denial: + with client.websocket_connect( + "/backend-api/codex/rtc_ordinary_missing", + headers={"Authorization": f"Bearer {key}"}, + ): + pytest.fail("an unbound ordinary live call must not connect") + + assert denial.value.status_code == 404 + assert denial.value.json()["error"]["code"] != "required_capability_transport_unsupported" diff --git a/tests/integration/test_detached_persistence.py b/tests/integration/test_detached_persistence.py index cb18ef800b..732f0c66d4 100644 --- a/tests/integration/test_detached_persistence.py +++ b/tests/integration/test_detached_persistence.py @@ -144,6 +144,7 @@ async def test_failed_detached_settlement_retries_failed_release_until_persisted output_tokens=6, ), ) + assert reservation is not None original_get_reservation = ApiKeysRepository.get_usage_reservation reservation_read_attempts = 0 @@ -212,6 +213,76 @@ async def fail_first_two_reservation_reads( assert retry_was_tracked is True +@pytest.mark.asyncio +async def test_settlement_release_and_heartbeat_noop_without_reservation(): + """A limit-free admission yields no reservation; settlement, release, and + heartbeat must no-op without opening a repository session.""" + from contextlib import asynccontextmanager + from typing import cast + + from app.core.utils.time import utcnow + from app.modules.api_keys.service import ApiKeyData + + factory_uses = 0 + + @asynccontextmanager + async def repo_factory(): + nonlocal factory_uses + factory_uses += 1 + yield object() + + service = proxy_service_module.ProxyService(cast(proxy_service_module.ProxyRepoFactory, repo_factory)) + api_key = ApiKeyData( + id="key_unlimited", + name="unlimited", + key_prefix="sk-clb-test", + allowed_models=None, + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + settlement = proxy_service_module._StreamSettlement( + status="success", + model="gpt-5.5", + input_tokens=4, + output_tokens=6, + ) + + assert ( + await service._settle_stream_api_key_usage( + api_key, + None, + settlement, + request_id="req_no_reservation", + ) + is True + ) + assert settlement.usage_settlement_transferred is False + await service._settle_compact_api_key_usage( + api_key=api_key, + api_key_reservation=None, + response=None, + request_service_tier=None, + ) + await service._release_websocket_reservation(None) + assert ( + await service._maybe_touch_api_key_reservation( + api_key=api_key, + reservation=None, + last_touch_at=123.0, + request_id="req_no_reservation", + surface="stream", + ) + == 123.0 + ) + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert factory_uses == 0 + + @pytest.mark.asyncio async def test_drain_ignores_stuck_non_persistence_cleanup_tasks(): """A stuck bridge-close cleanup in _background_cleanup_tasks must not diff --git a/tests/integration/test_health_and_errors.py b/tests/integration/test_health_and_errors.py index f0b2609d54..dfa66f01b8 100644 --- a/tests/integration/test_health_and_errors.py +++ b/tests/integration/test_health_and_errors.py @@ -98,3 +98,50 @@ async def test_missing_static_asset_returns_not_found(async_client): assert response.status_code == 404 assert response.json()["detail"] == "Not Found" assert response.headers["X-App-Version"] == __version__ + + +def test_ensure_web_asset_mime_types_overrides_poisoned_registry(): + """Simulates the Windows HKCR poisoning from issue #1698. + + ``mimetypes.add_type`` mutates the global table, so this test re-runs the + startup registration after poisoning and leaves the correct mappings in + place for the rest of the suite. + """ + import mimetypes + + from app.main import _WEB_ASSET_MIME_TYPES, _ensure_web_asset_mime_types + + for extension in _WEB_ASSET_MIME_TYPES: + mimetypes.add_type("text/plain", extension) + assert mimetypes.guess_type("x.js")[0] == "text/plain" + + _ensure_web_asset_mime_types() + + for extension, expected in _WEB_ASSET_MIME_TYPES.items(): + assert mimetypes.guess_type(f"x{extension}")[0] == expected, extension + + +@pytest.mark.asyncio +async def test_assets_js_served_as_javascript_despite_poisoned_registry(async_client): + """Product-path regression for issue #1698: /assets/*.js must serve + text/javascript even when the OS mimetypes sources map .js to text/plain, + or strict browser MIME checking rejects every dashboard module script.""" + import mimetypes + + from app.main import _ensure_web_asset_mime_types + + asset_name = next( + (candidate.name for candidate in sorted((_STATIC_DIR / "assets").glob("*.js"))), + None, + ) + assert asset_name is not None, "built dashboard assets missing; run cd frontend && bun run build" + + mimetypes.add_type("text/plain", ".js") + try: + _ensure_web_asset_mime_types() + response = await async_client.get(f"/assets/{asset_name}") + finally: + _ensure_web_asset_mime_types() + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/javascript") diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index 3a558de7b2..909fed0586 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -3,8 +3,8 @@ import asyncio import base64 import contextlib -import copy import json +import socket import time from collections import deque from collections.abc import AsyncGenerator @@ -21,14 +21,7 @@ from sqlalchemy import select, update import app.modules.proxy.load_balancer as load_balancer_module -import app.modules.proxy.replay_safety as replay_safety_module import app.modules.proxy.service as proxy_module -from app.core.auth.dashboard_access import admin_principal -from app.core.auth.dashboard_mode import DashboardAuthMode -from app.core.clients.proxy_websocket import ( - UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - UpstreamWebSocketTransportError, -) from app.core.config.settings import Settings from app.core.openai.model_registry import ModelRegistry from app.core.utils.request_id import ( @@ -38,26 +31,14 @@ set_request_scope_id, ) from app.core.utils.time import utcnow -from app.db.models import ( - Account, - AccountStatus, - DashboardSettings, - HttpBridgeRecoveryAttemptRecord, - HttpBridgeRecoveryAttemptState, - HttpBridgeRowlessRecoveryAuthority, - HttpBridgeRowlessRecoveryState, - HttpBridgeSessionAlias, - HttpBridgeSessionRecord, - HttpBridgeSessionState, - RequestLog, -) +from app.db.models import Account, AccountStatus, DashboardSettings, HttpBridgeSessionState, RequestLog, StickySession from app.db.session import SessionLocal from app.dependencies import get_proxy_service_for_app from app.modules.proxy._service import support as proxy_support from app.modules.proxy._service.http_bridge import quarantine as http_bridge_quarantine_module from app.modules.proxy._service.http_bridge import streaming as http_bridge_streaming_module -from app.modules.proxy._service.http_bridge import upstream_events as http_bridge_upstream_events_module from app.modules.proxy._service.http_bridge.helpers import ( + _make_http_bridge_session_header_fallback_key, _release_http_bridge_unanchored_handoff, _reserve_http_bridge_unanchored_handoff, ) @@ -67,13 +48,6 @@ AccountSelection, CatalogOmissionQuotaAdmission, ) -from app.modules.proxy.rowless_recovery import ROWLESS_AUTHORIZATION_MODE_AUTOMATIC -from app.modules.proxy.rowless_recovery_api import require_authenticated_rebase_admin -from app.modules.proxy.rowless_recovery_repository import ( - RowlessCheckpointReceipt, - RowlessRecoveryRepository, - RowlessRecoveryStateError, -) from app.modules.proxy.sticky_repository import StickySessionsRepository from app.modules.usage.repository import AdditionalUsageRepository @@ -81,3627 +55,1442 @@ _TEST_SYNC_TIMEOUT_SECONDS = 5.0 -@pytest.fixture(autouse=True) -def _enable_legacy_rowless_recovery_contract_tests(monkeypatch: pytest.MonkeyPatch) -> None: - """Keep compatibility coverage while production defaults to the upstream path.""" +@pytest_asyncio.fixture(autouse=True) +async def _cleanup_http_bridge_sessions(app_instance): + yield + service = get_proxy_service_for_app(app_instance) + async with service._http_bridge_lock: + sessions = list(service._http_bridge_sessions.values()) + inflight_sessions = list(service._http_bridge_inflight_sessions.values()) + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() + service._http_bridge_previous_response_index.clear() + for session in sessions: + await service._close_http_bridge_session(session) + for inflight_future in inflight_sessions: + if not inflight_future.done(): + inflight_future.cancel() + + +def _encode_jwt(payload: dict) -> str: + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + body = base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") + return f"header.{body}.sig" + - monkeypatch.setattr( - http_bridge_streaming_module, - "_ROWLESS_SEMANTIC_REBASE_REQUEST_PATH_ENABLED", - True, - ) - monkeypatch.setattr( - http_bridge_streaming_module, - "_DURABLE_RECOVERY_MARKER_REQUEST_PATH_ENABLED", - True, - ) - - -def _official_codex_turn_carriers(task_id: str, turn_id: str) -> tuple[dict[str, str], dict[str, str]]: - turn_metadata = { - "installation_id": "installation-before-account-selection", - "session_id": task_id, - "thread_id": task_id, - "turn_id": turn_id, - "root_turn_id": turn_id, - "window_id": "window-a", - "workspace_kind": "projectless", - "request_kind": "turn", - } - turn_metadata_json = json.dumps(turn_metadata, separators=(",", ":")) - headers = { - "session-id": task_id, - "thread-id": task_id, - "x-client-request-id": task_id, - "x-codex-installation-id": "installation-before-account-selection", - "x-codex-window-id": "window-a", - "x-codex-turn-metadata": turn_metadata_json, +def _make_auth_json(account_id: str, email: str, *, plan_type: str = "plus") -> dict: + payload = { + "email": email, + "chatgpt_account_id": account_id, + "https://api.openai.com/auth": {"chatgpt_plan_type": plan_type}, } - client_metadata = { - "session_id": task_id, - "thread_id": task_id, - "turn_id": turn_id, - "root_turn_id": turn_id, - "x-codex-installation-id": "installation-before-account-selection", - "x-codex-window-id": "window-a", - "x-codex-turn-metadata": turn_metadata_json, + return { + "tokens": { + "idToken": _encode_jwt(payload), + "accessToken": "access-token", + "refreshToken": "refresh-token", + "accountId": account_id, + }, } - return headers, client_metadata -def _complete_automatic_recovery_input() -> list[dict[str, object]]: +async def _collect_sse_events( + async_client, + path: str, + *, + json_body: dict, + headers: dict[str, str] | None = None, +) -> list[dict]: + async with async_client.stream("POST", path, json=json_body, headers=headers) as response: + assert response.status_code == 200 + lines = [line async for line in response.aiter_lines() if line.startswith("data: ")] return [ - { - "role": "user", - "content": [{"type": "input_text", "text": "original task"}], - }, - { - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "original result"}], - }, - { - "role": "user", - "content": [{"type": "input_text", "text": "continue the original task"}], - }, + event + for line in lines + if line[6:] != "[DONE]" + if (event := json.loads(line[6:])).get("type") != "codex.keepalive" ] -def test_forked_official_codex_turn_is_not_eligible_for_automatic_recovery() -> None: - task_id = "task-forked" - headers, client_metadata = _official_codex_turn_carriers(task_id, "turn-forked") - body_metadata = json.loads(client_metadata["x-codex-turn-metadata"]) - direct_metadata = json.loads(headers["x-codex-turn-metadata"]) - body_metadata["forked_from_thread_id"] = "task-parent" - direct_metadata["forked_from_thread_id"] = "task-parent" - client_metadata["x-codex-turn-metadata"] = json.dumps(body_metadata, separators=(",", ":")) - headers["x-codex-turn-metadata"] = json.dumps(direct_metadata, separators=(",", ":")) - - valid, child_signal, metadata_thread_present, automatic_live_recovery = ( - http_bridge_streaming_module._rowless_client_metadata_evidence( - raw_client_metadata=client_metadata, - normalized_headers=headers, - session_id=task_id, - task_identity=task_id, - ) - ) +async def _collect_sse_events_with_headers( + async_client, + path: str, + *, + json_body: dict, + headers: dict[str, str] | None = None, +) -> tuple[list[dict], dict[str, str]]: + async with async_client.stream("POST", path, json=json_body, headers=headers) as response: + assert response.status_code == 200 + response_headers = dict(response.headers) + lines = [line async for line in response.aiter_lines() if line.startswith("data: ")] + return [ + event + for line in lines + if line[6:] != "[DONE]" + if (event := json.loads(line[6:])).get("type") != "codex.keepalive" + ], response_headers - assert valid - assert not child_signal - assert metadata_thread_present - assert not automatic_live_recovery +def _assert_created_text_delta_completed(events: list[dict]) -> None: + assert [event["type"] for event in events] == [ + "response.created", + "response.output_text.delta", + "response.completed", + ] + assert events[1]["delta"] == "OK" -@pytest.mark.asyncio -@pytest.mark.parametrize( - "session_exit_failure", - (None, "cancel", "runtime"), - ids=("normal", "cancel-during-session-exit", "error-during-session-exit"), -) -async def test_official_codex_turn_automatically_rebases_stale_anchor_in_place( - async_client, - app_instance, - monkeypatch, - session_exit_failure, -) -> None: - """A proven official root turn recovers without a dashboard round trip.""" - del app_instance - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc-rowless-automatic", - "rowless-automatic@example.com", - ) - account = await _get_account(account_id) - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_auto_purged") - recovered_upstream = _FakeBridgeUpstreamWebSocket("resp_auto_recovered") - upstreams = [stale_upstream, recovered_upstream] - connect_count = 0 - selection_count = 0 +async def _import_account(async_client, account_id: str, email: str, *, plan_type: str = "plus") -> str: + auth_json = _make_auth_json(account_id, email, plan_type=plan_type) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + response = await async_client.post("/api/accounts/import", files=files) + assert response.status_code == 200 + return response.json()["accountId"] - async def fake_select_account_with_budget(self, deadline, **kwargs): - nonlocal selection_count - del self, deadline, kwargs - selection_count += 1 - return AccountSelection(account=account, error_message=None, error_code=None) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target +async def _get_account(account_id: str) -> Account: + async with SessionLocal() as session: + result = await session.execute(select(Account).where(Account.id == account_id)) + account = result.scalar_one() + session.expunge(account) + return account - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - nonlocal connect_count - del headers, access_token, account_id_header, base_url, session - upstream = upstreams[connect_count] - connect_count += 1 - return upstream - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) +async def _wait_for_event(event: asyncio.Event, *, timeout: float = _TEST_SYNC_TIMEOUT_SECONDS) -> None: + await asyncio.wait_for(event.wait(), timeout=timeout) - claim_returned = False - session_exit_failed = False - if session_exit_failure is not None: - original_claim = RowlessRecoveryRepository.capture_and_claim_automatic_preflight - original_session_factory = http_bridge_upstream_events_module.SessionLocal - - async def record_returned_claim(repository, **kwargs): - nonlocal claim_returned - authority = await original_claim(repository, **kwargs) - claim_returned = True - return authority - - class FailClaimSessionExit: - def __init__(self): - self._context = original_session_factory() - - async def __aenter__(self): - return await self._context.__aenter__() - - async def __aexit__(self, exc_type, exc, traceback): - nonlocal session_exit_failed - result = await self._context.__aexit__(exc_type, exc, traceback) - if claim_returned and not session_exit_failed: - session_exit_failed = True - if session_exit_failure == "cancel": - raise asyncio.CancelledError - raise RuntimeError("injected claim session exit failure") - return result - - monkeypatch.setattr( - RowlessRecoveryRepository, - "capture_and_claim_automatic_preflight", - record_returned_claim, - ) - monkeypatch.setattr( - http_bridge_upstream_events_module, - "SessionLocal", - FailClaimSessionExit, - ) - task_id = "01a0322c-0c11-7780-b68e-061ace9161a4" - turn_id = "01a034b7-9758-7253-8125-5167789a2bde" - headers, client_metadata = _official_codex_turn_carriers(task_id, turn_id) - complete_input = _complete_automatic_recovery_input() - request: dict[str, Any] = { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "previous_response_id": "resp_auto_purged", - "prompt_cache_key": task_id, - "client_metadata": client_metadata, - "input": complete_input, - } +async def _replace_http_bridge_upstream_reader( + service: proxy_module.ProxyService, + session: proxy_module._HTTPBridgeSession, + upstream: proxy_module.UpstreamWebSocket, +) -> None: + reader = session.upstream_reader + if reader is not None: + reader.cancel() + with contextlib.suppress(asyncio.CancelledError): + await reader + session.upstream = upstream + session.closed = False + session.upstream_control = proxy_module._WebSocketUpstreamControl() + session.upstream_reader = asyncio.create_task(service._relay_http_bridge_upstream_messages(session)) - if session_exit_failure is not None: - failed_response = await async_client.post("/v1/responses", headers=headers, json=request) - assert failed_response.status_code == 502, failed_response.text - expected_code = ( - "upstream_stream_truncated" if session_exit_failure == "cancel" else "bridge_continuity_persistence_failed" - ) - assert failed_response.json()["error"]["code"] == expected_code - assert claim_returned - assert session_exit_failed - assert connect_count == 1 - assert selection_count == 1 - assert len(stale_upstream.sent_text) == 1 - assert not recovered_upstream.sent_text - async with SessionLocal() as db_session: - authority = await db_session.scalar(select(HttpBridgeRowlessRecoveryAuthority)) - assert authority is not None - assert authority.state == HttpBridgeRowlessRecoveryState.APPROVED - assert authority.dispatch_request_id is None - assert authority.wire_request_fingerprint is None - assert authority.dispatch_send_started_at is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord.id)) is None - return - - response = await async_client.post("/v1/responses", headers=headers, json=request) - assert response.status_code == 200, response.text - assert response.json()["id"] == "resp_auto_recovered_1" - assert connect_count == 2 - assert selection_count == 2 - assert len(recovered_upstream.sent_text) == 1 - recovered_wire = json.loads(recovered_upstream.sent_text[0]) - assert "previous_response_id" not in recovered_wire - assert recovered_wire["input"] == complete_input - async with SessionLocal() as db_session: - authority = await db_session.scalar(select(HttpBridgeRowlessRecoveryAuthority)) - assert authority is not None - assert authority.state == HttpBridgeRowlessRecoveryState.CONSUMED - assert authority.authorization_mode == ROWLESS_AUTHORIZATION_MODE_AUTOMATIC - assert authority.checkpoint_receipt_sha256 is None - attempt = await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord)) - assert attempt is not None - assert attempt.state == HttpBridgeRecoveryAttemptState.REPLAYED - assert attempt.response_id == response.json()["id"] +class _SettingsCache: + def __init__(self, settings: DashboardSettings) -> None: + self._settings = settings + async def get(self) -> DashboardSettings: + return self._settings -@pytest.mark.asyncio -async def test_official_incremental_turn_keeps_operator_recovery_gate( - async_client, - app_instance, - monkeypatch, -) -> None: - """An anchor-only incremental turn cannot prove retained history.""" - del app_instance - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc-rowless-incremental", - "rowless-incremental@example.com", +def _make_app_settings( + *, + enabled: bool, + max_sessions: int = 128, + queue_limit: int = 8, + admission_wait_timeout_seconds: float = 0.05, + codex_idle_ttl_seconds: float = 900.0, + codex_prewarm_enabled: bool = False, + instance_id: str = "instance-a", + instance_ring: list[str] | None = None, +) -> Settings: + return Settings( + http_responses_session_bridge_enabled=enabled, + http_responses_session_bridge_idle_ttl_seconds=120.0, + http_responses_session_bridge_codex_idle_ttl_seconds=codex_idle_ttl_seconds, + http_responses_session_bridge_codex_prewarm_enabled=codex_prewarm_enabled, + http_responses_session_bridge_max_sessions=max_sessions, + http_responses_session_bridge_queue_limit=queue_limit, + http_responses_session_bridge_instance_id=instance_id, + http_responses_session_bridge_instance_ring=list(instance_ring or []), + proxy_admission_wait_timeout_seconds=admission_wait_timeout_seconds, + proxy_request_budget_seconds=75.0, + compact_request_budget_seconds=75.0, + transcription_request_budget_seconds=120.0, + upstream_compact_timeout_seconds=None, + upstream_stream_transport="auto", + stream_idle_timeout_seconds=300.0, + openai_prompt_cache_key_derivation_enabled=True, ) - account = await _get_account(account_id) - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_incremental_purged") - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline, kwargs - return AccountSelection(account=account, error_message=None, error_code=None) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target +def _make_dashboard_settings( + *, + prefer_earlier_reset_accounts: bool = False, + gateway_safe_mode: bool = False, + prompt_cache_idle_ttl_seconds: int | float = 3600, +) -> DashboardSettings: + return DashboardSettings( + id=1, + sticky_threads_enabled=False, + upstream_stream_transport="auto", + prefer_earlier_reset_accounts=prefer_earlier_reset_accounts, + routing_strategy="usage_weighted", + openai_cache_affinity_max_age_seconds=300, + import_without_overwrite=False, + totp_required_on_login=False, + api_key_auth_enabled=False, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=int(prompt_cache_idle_ttl_seconds), + http_responses_session_bridge_gateway_safe_mode=gateway_safe_mode, + sticky_reallocation_budget_threshold_pct=95.0, + ) - async def fake_connect_responses_websocket(*args, **kwargs): - del args, kwargs - return stale_upstream - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) +def _install_proxy_settings( + monkeypatch: pytest.MonkeyPatch, + *, + app_settings: Settings, + dashboard_settings: DashboardSettings, +) -> None: + monkeypatch.setattr(proxy_module, "get_settings_cache", lambda: _SettingsCache(dashboard_settings)) + monkeypatch.setattr(proxy_module, "get_settings", lambda: app_settings) - task_id = "task-rowless-incremental" - headers, client_metadata = _official_codex_turn_carriers(task_id, "turn-rowless-incremental") - response = await async_client.post( - "/v1/responses", - headers=headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "previous_response_id": "resp_incremental_purged", - "prompt_cache_key": task_id, - "client_metadata": client_metadata, - "input": [{"role": "user", "content": "continue the original task"}], - }, - ) - assert response.status_code == 400, response.text - assert response.json()["error"]["code"] == "previous_response_recovery_authorization_required" - assert len(stale_upstream.sent_text) == 1 - async with SessionLocal() as db_session: - authority = await db_session.scalar(select(HttpBridgeRowlessRecoveryAuthority)) - assert authority is not None - assert authority.state == HttpBridgeRowlessRecoveryState.CAPTURED - assert authority.authorization_mode is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord.id)) is None +def _install_bridge_settings(monkeypatch: pytest.MonkeyPatch, *, enabled: bool) -> None: + _install_bridge_settings_with_limits(monkeypatch, enabled=enabled) -@pytest.mark.asyncio -async def test_official_codex_turn_restores_unsent_claim_when_retry_is_not_submitted( - async_client, - app_instance, - monkeypatch, +def _install_bridge_settings_with_limits( + monkeypatch: pytest.MonkeyPatch, + *, + enabled: bool, + max_sessions: int = 128, + queue_limit: int = 8, + admission_wait_timeout_seconds: float = 0.05, + codex_idle_ttl_seconds: float = 900.0, + prompt_cache_idle_ttl_seconds: float = 3600.0, + codex_prewarm_enabled: bool = False, + gateway_safe_mode: bool = False, + prefer_earlier_reset_accounts: bool = False, + instance_id: str = "instance-a", + instance_ring: list[str] | None = None, ) -> None: - """A local retry refusal must not strand an unsent authority in UNKNOWN.""" - - del app_instance - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc-rowless-retry-refused", - "rowless-retry-refused@example.com", + _install_proxy_settings( + monkeypatch, + app_settings=_make_app_settings( + enabled=enabled, + max_sessions=max_sessions, + queue_limit=queue_limit, + admission_wait_timeout_seconds=admission_wait_timeout_seconds, + codex_idle_ttl_seconds=codex_idle_ttl_seconds, + codex_prewarm_enabled=codex_prewarm_enabled, + instance_id=instance_id, + instance_ring=instance_ring, + ), + dashboard_settings=_make_dashboard_settings( + prefer_earlier_reset_accounts=prefer_earlier_reset_accounts, + gateway_safe_mode=gateway_safe_mode, + prompt_cache_idle_ttl_seconds=prompt_cache_idle_ttl_seconds, + ), ) - account = await _get_account(account_id) - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_retry_refused") - - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline, kwargs - return AccountSelection(account=account, error_message=None, error_code=None) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, +class _FakeUpstreamMessage: + def __init__( + self, + kind: str, *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - return stale_upstream + text: str | None = None, + close_code: int | None = None, + error: str | None = None, + error_code: str | None = None, + ) -> None: + self.kind = kind + self.text = text + self.close_code = close_code + self.error = error + self.error_code = error_code + self.data = None - async def refuse_precreated_retry(self, session, *, request_state=None, restart_reader=False): - del self, session, request_state, restart_reader - return False - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_retry_http_bridge_precreated_request", refuse_precreated_retry) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) +class _FakeBridgeUpstreamWebSocket: + def __init__(self, response_id_prefix: str = "resp_bridge") -> None: + self.sent_text: list[str] = [] + self.closed = False + self.response_id_prefix = response_id_prefix + self._messages: asyncio.Queue[_FakeUpstreamMessage] = asyncio.Queue() - task_id = "01a0322c-0c11-7780-b68e-061ace9161a4" - headers, client_metadata = _official_codex_turn_carriers( - task_id, - "01a034b7-9758-7253-8125-5167789a2bde", - ) - response = await async_client.post( - "/v1/responses", - headers=headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "previous_response_id": "resp_retry_refused", - "prompt_cache_key": task_id, - "client_metadata": client_metadata, - "input": _complete_automatic_recovery_input(), - }, - ) + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + response_id = f"{self.response_id_prefix}_{len(self.sent_text)}" + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": {"id": response_id, "object": "response", "status": "in_progress"}, + }, + separators=(",", ":"), + ), + ) + ) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": response_id, + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) + ) - assert response.status_code == 502, response.text - assert response.json()["error"]["code"] == "bridge_continuity_persistence_failed" - async with SessionLocal() as db_session: - authority = await db_session.scalar(select(HttpBridgeRowlessRecoveryAuthority)) - assert authority is not None - assert authority.state == HttpBridgeRowlessRecoveryState.APPROVED - assert authority.dispatch_request_id is None - assert authority.wire_request_fingerprint is None - assert authority.dispatch_send_started_at is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord.id)) is None + async def send_bytes(self, data: bytes) -> None: + raise AssertionError(f"Unexpected binary frame: {data!r}") + async def receive(self) -> _FakeUpstreamMessage: + return await self._messages.get() -@pytest.mark.asyncio -async def test_official_codex_turn_cancellation_during_retry_setup_restores_unsent_claim( - async_client, - app_instance, - monkeypatch, -) -> None: - """Cancellation after the claim commits must restore the unsent authority.""" + async def close(self) -> None: + self.closed = True - del app_instance - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc-rowless-retry-cancelled", - "rowless-retry-cancelled@example.com", - ) - account = await _get_account(account_id) - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_retry_cancelled") + def response_header(self, name: str) -> str | None: + del name + return None - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline, kwargs - return AccountSelection(account=account, error_message=None, error_code=None) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target +class _InterruptedCustomToolUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + """First response completes with an unresolved ``custom_tool_call``.""" - async def fake_connect_responses_websocket(*args, **kwargs): - del args, kwargs - return stale_upstream + def __init__(self, response_id_prefix: str = "resp_bridge", *, emit_added: bool = False) -> None: + super().__init__(response_id_prefix) + self._emit_added = emit_added - async def cancel_precreated_retry(self, session, *, request_state=None, restart_reader=False): - del self, session, request_state, restart_reader - raise asyncio.CancelledError + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + response_id = f"resp_bridge_custom_{len(self.sent_text)}" + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": {"id": response_id, "object": "response", "status": "in_progress"}, + }, + separators=(",", ":"), + ), + ) + ) + if len(self.sent_text) == 1: + if self._emit_added: + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.output_item.added", + "response_id": response_id, + "item": { + "id": "ctc_shell", + "type": "custom_tool_call", + "status": "in_progress", + "call_id": "call_custom_shell", + "name": "shell", + "input": "", + }, + "output_index": 0, + }, + separators=(",", ":"), + ), + ) + ) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.output_item.done", + "response_id": response_id, + "item": { + "id": "ctc_shell", + "type": "custom_tool_call", + "status": "completed", + "call_id": "call_custom_shell", + "name": "shell", + "input": "pwd", + }, + "output_index": 0, + }, + separators=(",", ":"), + ), + ) + ) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": response_id, + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) + ) - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_retry_http_bridge_precreated_request", cancel_precreated_retry) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - task_id = "task-rowless-retry-cancelled" - headers, client_metadata = _official_codex_turn_carriers(task_id, "turn-rowless-retry-cancelled") - response = await async_client.post( - "/v1/responses", - headers=headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "previous_response_id": "resp_retry_cancelled", - "prompt_cache_key": task_id, - "client_metadata": client_metadata, - "input": _complete_automatic_recovery_input(), - }, - ) +class _ClosingInterruptedCustomToolUpstreamWebSocket(_InterruptedCustomToolUpstreamWebSocket): + def __init__(self, response_id_prefix: str = "resp_bridge") -> None: + super().__init__(response_id_prefix, emit_added=True) - assert response.status_code == 502, response.text - async with SessionLocal() as db_session: - authority = await db_session.scalar(select(HttpBridgeRowlessRecoveryAuthority)) - assert authority is not None - assert authority.state == HttpBridgeRowlessRecoveryState.APPROVED - assert authority.dispatch_request_id is None - assert authority.wire_request_fingerprint is None - assert authority.dispatch_send_started_at is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord.id)) is None + async def send_text(self, text: str) -> None: + await super().send_text(text) + await self._messages.put(_FakeUpstreamMessage("close", close_code=1000)) -@pytest.mark.asyncio -async def test_official_codex_turn_retry_transport_failure_settles_attached_request( - async_client, - app_instance, - monkeypatch, -) -> None: - """A retry transport failure must reach the reader's terminal settlement.""" - - del app_instance - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc-rowless-retry-transport-error", - "rowless-retry-transport-error@example.com", - ) - account = await _get_account(account_id) - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_retry_transport_error") +class _ClosingBridgeUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + await super().send_text(text) + await self._messages.put(_FakeUpstreamMessage("close", close_code=1000)) - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline, kwargs - return AccountSelection(account=account, error_message=None, error_code=None) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target +class _PrecreatedCloseUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + await self._messages.put(_FakeUpstreamMessage("close", close_code=1011)) - async def fake_connect_responses_websocket(*args, **kwargs): - del args, kwargs - return stale_upstream - async def fail_precreated_retry(self, session, *, request_state=None, restart_reader=False): - del self, session, request_state, restart_reader - raise UpstreamWebSocketTransportError( - "upstream closed before automatic retry dispatch", - error_code=UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, +class _PrecreatedOverloadUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.failed", + "response": { + "error": { + "code": "server_is_overloaded", + "message": "Our servers are currently overloaded. Please try again later.", + } + }, + }, + separators=(",", ":"), + ), + ) ) - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_retry_http_bridge_precreated_request", fail_precreated_retry) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - task_id = "task-rowless-retry-transport-error" - headers, client_metadata = _official_codex_turn_carriers(task_id, "turn-rowless-retry-transport-error") - with anyio.fail_after(_TEST_SYNC_TIMEOUT_SECONDS): - response = await async_client.post( - "/v1/responses", - headers=headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "previous_response_id": "resp_retry_transport_error", - "prompt_cache_key": task_id, - "client_metadata": client_metadata, - "input": _complete_automatic_recovery_input(), - }, +class _CreatedOnlyUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + response_id = f"resp_created_only_{len(self.sent_text)}" + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": {"id": response_id, "object": "response", "status": "in_progress"}, + }, + separators=(",", ":"), + ), + ) ) - assert response.status_code == 502, response.text - assert response.json()["error"]["code"] == UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE - async with SessionLocal() as db_session: - authority = await db_session.scalar(select(HttpBridgeRowlessRecoveryAuthority)) - assert authority is not None - assert authority.state == HttpBridgeRowlessRecoveryState.APPROVED - assert authority.dispatch_request_id is None - assert authority.wire_request_fingerprint is None - assert authority.dispatch_send_started_at is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord.id)) is None - - -@pytest.mark.asyncio -async def test_official_codex_turn_automatic_claim_conflict_returns_non_retryable_error( - async_client, - app_instance, - monkeypatch, -) -> None: - """A concurrent automatic claim is a 400 proof conflict, not a retryable 502.""" - - del app_instance - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc-rowless-claim-conflict", - "rowless-claim-conflict@example.com", - ) - account = await _get_account(account_id) - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_claim_conflict") - selection_count = 0 - async def fake_select_account_with_budget(self, deadline, **kwargs): - nonlocal selection_count - del self, deadline, kwargs - selection_count += 1 - return AccountSelection(account=account, error_message=None, error_code=None) +class _SilentUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - async def fake_connect_responses_websocket(*args, **kwargs): - del args, kwargs - return stale_upstream +class _AccountScopedAnchorUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + """Upstream that only resolves ``previous_response_id`` values it issued. - async def reject_concurrent_claim(self, **kwargs): - del self, kwargs - raise RowlessRecoveryStateError("automatic_live_request_claim_conflict") + A ``previous_response_id`` is account-scoped upstream: only the account that + created the response can resume it. A ``response.create`` carrying a foreign + anchor is accepted by the socket but never answered with ``response.created``. + Modelling that here makes a cross-account anchor observable as the production + symptom instead of a silent assertion: the turn never settles and the + per-bridge ``response_create_gate`` stays held. + """ - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr( - RowlessRecoveryRepository, - "capture_and_claim_automatic_preflight", - reject_concurrent_claim, - ) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + async def send_text(self, text: str) -> None: + anchor = json.loads(text).get("previous_response_id") + if isinstance(anchor, str) and not anchor.startswith(self.response_id_prefix): + self.sent_text.append(text) + return + await super().send_text(text) - task_id = "task-rowless-claim-conflict" - headers, client_metadata = _official_codex_turn_carriers(task_id, "turn-rowless-claim-conflict") - response = await async_client.post( - "/v1/responses", - headers=headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "previous_response_id": "resp_claim_conflict", - "prompt_cache_key": task_id, - "client_metadata": client_metadata, - "input": _complete_automatic_recovery_input(), - }, - ) - assert response.status_code == 400, response.text - assert response.json()["error"]["code"] == "rowless_automatic_recovery_proof_rejected" - assert selection_count == 1 - assert len(stale_upstream.sent_text) == 1 - async with SessionLocal() as db_session: - assert await db_session.scalar(select(HttpBridgeRowlessRecoveryAuthority.id)) is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord.id)) is None +class _RecordingUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + pass -@pytest.mark.asyncio -async def test_legacy_rowless_capture_state_error_keeps_persistence_failure_contract( - async_client, - app_instance, - monkeypatch, -) -> None: - """Legacy capture limits must not be mislabeled as automatic proof failures.""" +class _CreatedThenCloseUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + response_id = f"resp_created_then_close_{len(self.sent_text)}" + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": {"id": response_id, "object": "response", "status": "in_progress"}, + }, + separators=(",", ":"), + ), + ) + ) + await self._messages.put(_FakeUpstreamMessage("close", close_code=1011)) - del app_instance - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc-rowless-legacy-capture-limit", - "rowless-legacy-capture-limit@example.com", - ) - account = await _get_account(account_id) - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_legacy_capture_limit") - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline, kwargs - return AccountSelection(account=account, error_message=None, error_code=None) +class _ReasoningThenAbruptCloseUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + response_id = f"resp_reasoning_then_close_{len(self.sent_text)}" + reasoning_id = f"rs_reasoning_then_close_{len(self.sent_text)}" + events = [ + { + "type": "response.created", + "sequence_number": 0, + "response": {"id": response_id, "object": "response", "status": "in_progress"}, + }, + { + "type": "response.output_item.added", + "sequence_number": 1, + "response_id": response_id, + "output_index": 0, + "item": { + "id": reasoning_id, + "type": "reasoning", + "summary": [], + "encrypted_content": None, + }, + }, + { + "type": "response.reasoning_summary_part.added", + "sequence_number": 2, + "response_id": response_id, + "item_id": reasoning_id, + "output_index": 0, + "summary_index": 0, + "part": {"type": "summary_text", "text": ""}, + }, + { + "type": "response.reasoning_summary_text.delta", + "sequence_number": 3, + "response_id": response_id, + "item_id": reasoning_id, + "output_index": 0, + "summary_index": 0, + "delta": "Reviewing the final integration result.", + }, + ] + for event in events: + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps(event, separators=(",", ":")), + ) + ) + await self._messages.put( + _FakeUpstreamMessage( + "error", + error="no close frame received or sent", + ) + ) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - async def fake_connect_responses_websocket(*args, **kwargs): - del args, kwargs - return stale_upstream +class _CompleteThenReasoningAbruptCloseUpstreamWebSocket(_ReasoningThenAbruptCloseUpstreamWebSocket): + async def send_text(self, text: str) -> None: + if not self.sent_text: + await _FakeBridgeUpstreamWebSocket.send_text(self, text) + return + await super().send_text(text) - async def reject_legacy_capture(self, **kwargs): - del self, kwargs - raise RowlessRecoveryStateError("rowless_capture_scope_limit_reached") - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(RowlessRecoveryRepository, "capture", reject_legacy_capture) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) +class _CompleteThenPrecreatedCloseUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + if not self.sent_text: + await super().send_text(text) + return + self.sent_text.append(text) + await self._messages.put(_FakeUpstreamMessage("close", close_code=1011)) - task_id = "task-rowless-legacy-capture-limit" - response = await async_client.post( - "/v1/responses", - headers={ - "session-id": task_id, - "thread-id": task_id, - "x-client-request-id": task_id, - }, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "previous_response_id": "resp_legacy_capture_limit", - "prompt_cache_key": task_id, - "input": [{"role": "user", "content": "continue the original task"}], - }, - ) - assert response.status_code == 502, response.text - assert response.json()["error"]["code"] == "bridge_continuity_persistence_failed" +class _ErrorOnlyUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "invalid_request_error", + "message": ( + "The 'gpt-5.3-codex-spark' model is not supported when using Codex " + "with a ChatGPT account." + ), + }, + }, + separators=(",", ":"), + ), + ) + ) -@pytest.mark.asyncio -@pytest.mark.parametrize( - "fail_claim_session_exit", - (False, True), - ids=("normal", "error-during-session-exit"), -) -async def test_official_codex_turn_supersedes_unsent_approved_context_drift( - async_client, - app_instance, - monkeypatch, - fail_claim_session_exit, -) -> None: - """A safe live turn replaces an obsolete, physically-unsent approval.""" +class _RateLimitErrorUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "error", + "status": 429, + "error": { + "type": "rate_limit_error", + "code": "rate_limit_exceeded", + "message": "Rate limit reached for gpt-4o on tokens per day", + "plan_type": "team", + "resets_at": 1700000000, + "resets_in_seconds": 3600, + }, + }, + separators=(",", ":"), + ), + ) + ) - del app_instance - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc-rowless-approved-drift", - "rowless-approved-drift@example.com", - ) - account = await _get_account(account_id) - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_approved_purged") - recovered_upstream = _FakeBridgeUpstreamWebSocket("resp_approved_recovered") - upstreams = [stale_upstream, recovered_upstream] - connect_count = 0 - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline, kwargs - return AccountSelection(account=account, error_message=None, error_code=None) +class _PreviousResponseNotFoundUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + payload = json.loads(text) + previous_response_id = payload.get("previous_response_id") + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": f"Previous response with id '{previous_response_id}' not found.", + "param": "previous_response_id", + }, + }, + separators=(",", ":"), + ), + ) + ) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - nonlocal connect_count - del headers, access_token, account_id_header, base_url, session - upstream = upstreams[connect_count] - connect_count += 1 - return upstream +class _AnonymousPreviousResponseNotFoundWithInflightUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + def __init__(self) -> None: + super().__init__() + self.first_request_created = asyncio.Event() + self._anchored_followup_failed = False - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + if len(self.sent_text) == 1: + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": { + "id": "resp_bridge_inflight", + "object": "response", + "status": "in_progress", + }, + }, + separators=(",", ":"), + ), + ) + ) + self.first_request_created.set() + return - task_id = "01a0322c-0c11-7780-b68e-061ace9161a4" - legacy_headers = { - "session-id": task_id, - "thread-id": task_id, - "x-client-request-id": task_id, - } - request: dict[str, Any] = { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "previous_response_id": "resp_approved_purged", - "prompt_cache_key": task_id, - "input": [{"role": "user", "content": "continue the original task"}], - } - captured_response = await async_client.post("/v1/responses", headers=legacy_headers, json=request) - assert captured_response.status_code == 400, captured_response.text - assert captured_response.json()["error"]["code"] == "previous_response_recovery_authorization_required" - - async with SessionLocal() as db_session: - repository = RowlessRecoveryRepository(db_session) - authority = await db_session.scalar(select(HttpBridgeRowlessRecoveryAuthority)) - assert authority is not None - challenge = await repository.issue_challenge( - authority_id=authority.id, - generation=authority.generation, - ) - receipt = RowlessCheckpointReceipt( - schema="qk_http_bridge_rowless_checkpoint_receipt_v1", - remote_session_jsonl_sha256="a" * 64, - remote_session_jsonl_size_bytes=4096, - remote_session_jsonl_last_offset=4096, - full_checkpoint_tool_ledger_digest="b" * 64, - unresolved_count=0, - task_identity=task_id, - session_identity=task_id, - strong_session_hash=authority.strong_session_hash, - task_authority_digest=authority.captured_task_authority_digest, - captured_input_item_count=authority.captured_input_item_count, - captured_input_fingerprint=authority.captured_input_fingerprint, - non_input_contract_fingerprint=authority.non_input_contract_fingerprint, - retained_request_direct_call_ledger_digest=authority.settled_direct_call_ledger_digest, - captured_projected_payload_fingerprint=authority.projected_payload_fingerprint, - captured_actual_wire_fingerprint=authority.actual_wire_fingerprint, - captured_request_binding_provenance="server_challenge", - ) - approved = await repository.approve( - authority_id=authority.id, - generation=authority.generation, - challenge=challenge.challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement="operator_acknowledged_semantic_rebase", - approved_actor="trusted-dashboard-operator", - request_id="historical-approval", - ) - authority_id = approved.id - approved_generation = approved.generation - - claim_returned = False - claim_session_exit_failed = False - if fail_claim_session_exit: - original_claim = RowlessRecoveryRepository.capture_and_claim_automatic_preflight - original_session_factory = http_bridge_streaming_module.SessionLocal - - async def record_returned_claim(repository, **kwargs): - nonlocal claim_returned - claimed = await original_claim(repository, **kwargs) - claim_returned = True - return claimed - - class FailClaimSessionExit: - def __init__(self): - self._context = original_session_factory() - - async def __aenter__(self): - return await self._context.__aenter__() - - async def __aexit__(self, exc_type, exc, traceback): - nonlocal claim_session_exit_failed - result = await self._context.__aexit__(exc_type, exc, traceback) - if claim_returned and not claim_session_exit_failed: - claim_session_exit_failed = True - raise RuntimeError("injected request-start claim session exit failure") - return result - - monkeypatch.setattr( - RowlessRecoveryRepository, - "capture_and_claim_automatic_preflight", - record_returned_claim, + payload = json.loads(text) + previous_response_id = payload.get("previous_response_id") + if self._anchored_followup_failed: + response_id = f"{self.response_id_prefix}_{len(self.sent_text)}" + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": { + "id": response_id, + "object": "response", + "status": "in_progress", + }, + }, + separators=(",", ":"), + ), + ) + ) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": response_id, + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) + ) + return + + self._anchored_followup_failed = True + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": f"Previous response with id '{previous_response_id}' not found.", + "param": "previous_response_id", + }, + }, + separators=(",", ":"), + ), + ) ) - monkeypatch.setattr( - http_bridge_streaming_module, - "SessionLocal", - FailClaimSessionExit, + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_bridge_inflight", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) ) - headers, client_metadata = _official_codex_turn_carriers( - task_id, - "01a034d2-b7bb-74f0-9326-8ad06f40000d", - ) - drifted_request = copy.deepcopy(request) - drifted_request["client_metadata"] = client_metadata - drifted_request["input"] = _complete_automatic_recovery_input() - recovered = await async_client.post("/v1/responses", headers=headers, json=drifted_request) - - if fail_claim_session_exit: - assert recovered.status_code == 502, recovered.text - assert recovered.json()["error"]["code"] == "bridge_continuity_persistence_failed" - assert claim_returned - assert claim_session_exit_failed - assert connect_count == 1 - assert not recovered_upstream.sent_text - async with SessionLocal() as db_session: - authority = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority_id) - assert authority is not None - assert authority.state == HttpBridgeRowlessRecoveryState.APPROVED - assert authority.dispatch_request_id is None - assert authority.wire_request_fingerprint is None - assert authority.dispatch_send_started_at is None - assert authority.replacement_session_id is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord.id)) is None - return - - assert recovered.status_code == 200, recovered.text - assert recovered.json()["id"] == "resp_approved_recovered_1" - assert connect_count == 2 - assert len(recovered_upstream.sent_text) == 1 - recovered_wire = json.loads(recovered_upstream.sent_text[0]) - assert "previous_response_id" not in recovered_wire - async with SessionLocal() as db_session: - authority = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority_id) - assert authority is not None - assert authority.generation == approved_generation + 1 - assert authority.state == HttpBridgeRowlessRecoveryState.CONSUMED - assert authority.authorization_mode == ROWLESS_AUTHORIZATION_MODE_AUTOMATIC - assert authority.checkpoint_receipt_sha256 is None - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("recovery_mode", "incident_shape_index"), - ( - ("success", 0), - ("success", 1), - ("success", 2), - ("approved_no_anchor", 1), - ("approved_wrong_anchor", 1), - ("cancel_before_send_marker", 1), - ("cancel_after_send_marker", 1), - ("cancel_during_fresh_reconnect", 1), - ("cancel_during_fresh_reconnect_rollback", 1), - ("socket_reconnect_does_not_prove_ambiguous_send_unsent", 1), - ("two_closed_before_send", 1), - ("cancel_during_two_close_rollback", 1), - ("fresh_setup_failure_before_send", 1), - ("ambiguous_post_send", 1), - ("live_publication_failure", 1), - ("installation_id_drift", 1), - ("late_wire_drift", 1), - ), -) -async def test_rowless_stale_anchor_semantic_rebase_end_to_end( - async_client, - app_instance, - monkeypatch, - recovery_mode, - incident_shape_index, -): - """A purged checkpoint requires one admin-bound same-turn semantic rebase.""" - - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc-rowless-rebase", - "rowless-rebase@example.com", - ) - account = await _get_account(account_id) - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_purged") - recovered_upstream = _FakeBridgeUpstreamWebSocket("resp_rowless_recovered") - ambiguous_upstream = _AmbiguousAcceptedReceiveErrorUpstreamWebSocket() - accepted_then_cancelled_upstream = _AcceptedThenCancelledSendUpstreamWebSocket() - closed_upstreams = [_ClosedBeforeSendUpstreamWebSocket(), _ClosedBeforeSendUpstreamWebSocket()] - upstreams: list[_FakeBridgeUpstreamWebSocket] - if recovery_mode in { - "success", - "approved_no_anchor", - "approved_wrong_anchor", - "cancel_before_send_marker", - "cancel_after_send_marker", - "live_publication_failure", - "installation_id_drift", - "late_wire_drift", - }: - upstreams = [stale_upstream, recovered_upstream] - elif recovery_mode in {"two_closed_before_send", "cancel_during_two_close_rollback"}: - upstreams = [stale_upstream, *closed_upstreams, recovered_upstream] - elif recovery_mode in { - "fresh_setup_failure_before_send", - "cancel_during_fresh_reconnect", - "cancel_during_fresh_reconnect_rollback", - }: - upstreams = [stale_upstream, closed_upstreams[0]] - elif recovery_mode == "socket_reconnect_does_not_prove_ambiguous_send_unsent": - upstreams = [ - stale_upstream, - _FakeBridgeUpstreamWebSocket("resp_unused_before_socket_reconnect"), - accepted_then_cancelled_upstream, - ] - else: - upstreams = [stale_upstream, ambiguous_upstream] - connect_count = 0 - selection_count = 0 - async def fake_select_account_with_budget(self, deadline, **kwargs): - nonlocal selection_count - del self, deadline, kwargs - selection_count += 1 - return AccountSelection(account=account, error_message=None, error_code=None) +class _InvalidRequestPreviousResponseUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + payload = json.loads(text) + previous_response_id = payload.get("previous_response_id") + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "invalid_request_error", + "message": f"Previous response with id '{previous_response_id}' not found.", + "param": "previous_response_id", + }, + }, + separators=(",", ":"), + ), + ) + ) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - nonlocal connect_count - del headers, access_token, account_id_header, base_url, session - upstream = upstreams[connect_count] - connect_count += 1 - return upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - if recovery_mode == "live_publication_failure": - service = get_proxy_service_for_app(app_instance) - - async def fail_live_alias_publication(*args, **kwargs): - del args, kwargs - return False - - monkeypatch.setattr( - service, - "_register_http_bridge_previous_response_id", - fail_live_alias_publication, - ) - task_id = "01a02f21-77a1-7cc2-a892-b6abac317deb" - process_session_id = task_id - from tests.unit.test_replay_safety import _rehydrate_sanitized_pending_settlement_shapes - - agent_message_input = copy.deepcopy(_rehydrate_sanitized_pending_settlement_shapes()[incident_shape_index][0]) - headers = { - "session-id": process_session_id, - "thread-id": task_id, - "x-client-request-id": task_id, - } - request = { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "previous_response_id": "resp_purged", - "prompt_cache_key": task_id, - "input": agent_message_input, - } - captured_response = await async_client.post("/v1/responses", headers=headers, json=request) - - assert captured_response.status_code == 400, ( - captured_response.text, - connect_count, - stale_upstream.sent_text, - ) - captured_error = captured_response.json()["error"] - assert captured_error["code"] == "previous_response_recovery_authorization_required" - assert captured_error["action"] == "retry_same_turn_after_admin_approval" - assert connect_count == 1 - assert selection_count == 1 - - for anchor_mutation in (None, "resp_different_stale_anchor"): - drifted_request = copy.deepcopy(request) - if anchor_mutation is None: - drifted_request.pop("previous_response_id") - else: - drifted_request["previous_response_id"] = anchor_mutation - drifted_capture = await async_client.post( - "/v1/responses", - headers=headers, - json=drifted_request, - ) - assert drifted_capture.status_code == 400 - assert drifted_capture.json()["error"]["code"] == ("previous_response_recovery_authorization_required") - assert connect_count == 1 - assert selection_count == 1 - - for drift_headers in ( - {**headers, "x-codex-turn-state": "arbitrary-drift"}, - {**headers, "x-client-request-id": "different-client-request"}, - {**headers, "x-codex-session-id": "conflicting-session-alias"}, - ): - drifted_capture = await async_client.post( - "/v1/responses", - headers=drift_headers, - json=request, - ) - assert drifted_capture.status_code == 400 - assert drifted_capture.json()["error"]["code"] == ("previous_response_recovery_authorization_required") - assert connect_count == 1 - assert selection_count == 1 - - for anchor_mutation, routing_drift in ( - (None, {"x-codex-turn-state": "combined-turn-state-drift"}), - ("resp_different_stale_anchor", {"x-client-request-id": "combined-client-drift"}), - (None, {"x-codex-session-id": "combined-session-alias-drift"}), - ): - drifted_request = copy.deepcopy(request) - if anchor_mutation is None: - drifted_request.pop("previous_response_id") - else: - drifted_request["previous_response_id"] = anchor_mutation - drifted_capture = await async_client.post( - "/v1/responses", - headers={**headers, **routing_drift}, - json=drifted_request, - ) - assert drifted_capture.status_code == 400 - assert drifted_capture.json()["error"]["code"] == ("previous_response_recovery_authorization_required") - assert connect_count == 1 - assert selection_count == 1 - - unresolved_retry = copy.deepcopy(request) - unresolved_input = unresolved_retry["input"] - assert isinstance(unresolved_input, list) - unresolved_input.append( - { - "type": "function_call", - "call_id": "call_unresolved_after_capture", - "name": "irreversible_action", - "arguments": "{}", - } - ) - suppressed_unresolved = await async_client.post( - "/v1/responses", - headers=headers, - json=unresolved_retry, - ) - assert suppressed_unresolved.status_code == 400 - assert suppressed_unresolved.json()["error"]["code"] == ("previous_response_recovery_authorization_required") - assert connect_count == 1 - assert selection_count == 1 - - async with SessionLocal() as db_session: - authority = await db_session.scalar(select(HttpBridgeRowlessRecoveryAuthority)) - assert authority is not None - assert authority.state == HttpBridgeRowlessRecoveryState.CAPTURED - receipt = RowlessCheckpointReceipt( - schema="qk_http_bridge_rowless_checkpoint_receipt_v1", - remote_session_jsonl_sha256="a" * 64, - remote_session_jsonl_size_bytes=1153, - remote_session_jsonl_last_offset=1153, - full_checkpoint_tool_ledger_digest="b" * 64, - unresolved_count=0, - task_identity=task_id, - session_identity=process_session_id, - strong_session_hash=authority.strong_session_hash, - task_authority_digest=authority.captured_task_authority_digest, - captured_input_item_count=authority.captured_input_item_count, - captured_input_fingerprint=authority.captured_input_fingerprint, - non_input_contract_fingerprint=authority.non_input_contract_fingerprint, - retained_request_direct_call_ledger_digest=authority.settled_direct_call_ledger_digest, - captured_projected_payload_fingerprint=authority.projected_payload_fingerprint, - captured_actual_wire_fingerprint=authority.actual_wire_fingerprint, - captured_request_binding_provenance="server_challenge", - ) - app_instance.dependency_overrides[require_authenticated_rebase_admin] = lambda: admin_principal( - auth_mode=DashboardAuthMode.TRUSTED_HEADER, - actor="test-dashboard-admin", - ) - challenge_response = await async_client.post( - f"/api/http-bridge/rowless-recovery/{authority.id}/challenge", - json={"generation": authority.generation}, - ) - assert challenge_response.status_code == 200, challenge_response.text - challenge_payload = challenge_response.json() - assert challenge_payload["capturedInputItemCount"] == authority.captured_input_item_count - assert challenge_payload["capturedInputFingerprint"] == authority.captured_input_fingerprint - assert challenge_payload["nonInputContractFingerprint"] == authority.non_input_contract_fingerprint - assert challenge_payload["retainedRequestLedgerDigest"] == authority.settled_direct_call_ledger_digest - assert challenge_payload["projectedPayloadFingerprint"] == authority.projected_payload_fingerprint - assert challenge_payload["actualWireFingerprint"] == authority.actual_wire_fingerprint - assert challenge_payload["retainedUnresolvedCount"] == 0 - assert challenge_payload["requestSelfContained"] is True - assert challenge_payload["requestAccountNeutral"] is True - assert "selectedAccountIntent" not in challenge_payload - approval_payload = { - "generation": authority.generation, - "challenge": challenge_payload["challenge"], - "receipt_sha256": receipt.sha256(), - "receipt": receipt.canonical_payload(), - } - missing_ack = await async_client.post( - f"/api/http-bridge/rowless-recovery/{authority.id}/approve", - json=approval_payload, - ) - assert missing_ack.status_code == 422 - wrong_ack = await async_client.post( - f"/api/http-bridge/rowless-recovery/{authority.id}/approve", - json={**approval_payload, "acknowledgement": "OPERATOR_ACKNOWLEDGED_SEMANTIC_REBASE"}, - ) - assert wrong_ack.status_code == 422 - approve_response = await async_client.post( - f"/api/http-bridge/rowless-recovery/{authority.id}/approve", - json={ - **approval_payload, - "acknowledgement": "operator_acknowledged_semantic_rebase", - }, - ) - app_instance.dependency_overrides.pop(require_authenticated_rebase_admin, None) - assert approve_response.status_code == 200, approve_response.text - assert approve_response.json()["state"] == HttpBridgeRowlessRecoveryState.APPROVED.value - - approved_identity_drift = await async_client.post( - "/v1/responses", - headers={**headers, "x-codex-turn-state": "approved-drift"}, - json=request, - ) - assert approved_identity_drift.status_code == 400 - assert approved_identity_drift.json()["error"]["code"] == "rowless_recovery_exact_identity_required" - assert connect_count == 1 - assert selection_count == 1 - - approved_anchorless_identity_drift = copy.deepcopy(request) - approved_anchorless_identity_drift.pop("previous_response_id") - approved_anchorless_drift = await async_client.post( - "/v1/responses", - headers={**headers, "x-codex-turn-state": "approved-anchorless-drift"}, - json=approved_anchorless_identity_drift, - ) - assert approved_anchorless_drift.status_code == 400 - assert approved_anchorless_drift.json()["error"]["code"] == "rowless_recovery_exact_identity_required" - assert connect_count == 1 - assert selection_count == 1 - - approved_unresolved = await async_client.post( - "/v1/responses", - headers=headers, - json=unresolved_retry, - ) - assert approved_unresolved.status_code == 400 - assert approved_unresolved.json()["error"]["code"] == "rowless_recovery_exact_same_turn_required" - assert connect_count == 1 - assert selection_count == 1 - - if recovery_mode in {"cancel_before_send_marker", "cancel_after_send_marker"}: - original_mark_send_started = RowlessRecoveryRepository.mark_dispatch_send_started - - async def cancel_at_send_marker(repository, **kwargs): - if recovery_mode == "cancel_after_send_marker": - assert await original_mark_send_started(repository, **kwargs) - raise asyncio.CancelledError - - monkeypatch.setattr( - RowlessRecoveryRepository, - "mark_dispatch_send_started", - cancel_at_send_marker, - ) - # ASGITransport may surface an endpoint cancellation as either the - # original cancellation or Starlette's "No response returned" wrapper. - with pytest.raises((asyncio.CancelledError, RuntimeError)): - await async_client.post("/v1/responses", headers=headers, json=request) - assert connect_count == 2 - assert selection_count == 2 - assert len(recovered_upstream.sent_text) == 0 - async with SessionLocal() as db_session: - restored = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - assert restored is not None - assert restored.state == HttpBridgeRowlessRecoveryState.APPROVED - assert restored.replacement_session_id is None - assert restored.dispatch_request_id is None - assert restored.dispatch_send_started_at is None - assert restored.wire_request_fingerprint is None - if restored.origin_marker_session_id is not None: - marker = await db_session.get(HttpBridgeSessionRecord, restored.origin_marker_session_id) - assert marker is not None - assert marker.recovery_required_attempt_fingerprint is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord)) is None - return - - if recovery_mode in {"cancel_during_fresh_reconnect", "cancel_during_fresh_reconnect_rollback"}: - service = get_proxy_service_for_app(app_instance) - reconnect_entered = asyncio.Event() - rollback_entered = asyncio.Event() - allow_rollback = asyncio.Event() - original_preflight_rollback = service._rollback_rowless_preflight_setup_failure_if_unbound - - async def cancel_reconnect_before_fresh_send(*args, **kwargs): - del args, kwargs - if recovery_mode == "cancel_during_fresh_reconnect": - raise asyncio.CancelledError - reconnect_entered.set() - await asyncio.Future() - - async def block_rollback_after_second_cancellation(request_state): - rollback_entered.set() - await allow_rollback.wait() - await original_preflight_rollback(request_state) - - monkeypatch.setattr( - service, - "_reconnect_http_bridge_session", - cancel_reconnect_before_fresh_send, - ) - if recovery_mode == "cancel_during_fresh_reconnect_rollback": - monkeypatch.setattr( - service, - "_rollback_rowless_preflight_setup_failure_if_unbound", - block_rollback_after_second_cancellation, - ) - recovery_task = asyncio.create_task(async_client.post("/v1/responses", headers=headers, json=request)) - await asyncio.wait_for(reconnect_entered.wait(), timeout=_TEST_SYNC_TIMEOUT_SECONDS) - recovery_task.cancel() - await asyncio.wait_for(rollback_entered.wait(), timeout=_TEST_SYNC_TIMEOUT_SECONDS) - recovery_task.cancel() - allow_rollback.set() - with pytest.raises(asyncio.CancelledError): - await recovery_task - else: - with pytest.raises((asyncio.CancelledError, RuntimeError)): - await async_client.post("/v1/responses", headers=headers, json=request) - assert connect_count == 2 - assert selection_count == 2 - assert not closed_upstreams[0].sent_text - assert len(recovered_upstream.sent_text) == 0 - async with SessionLocal() as db_session: - restored = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - assert restored is not None - assert restored.state == HttpBridgeRowlessRecoveryState.APPROVED - assert restored.replacement_session_id is None - assert restored.dispatch_request_id is None - assert restored.dispatch_send_started_at is None - assert restored.wire_request_fingerprint is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord)) is None - return - - if recovery_mode == "socket_reconnect_does_not_prove_ambiguous_send_unsent": - service = get_proxy_service_for_app(app_instance) - original_get_or_create = service._get_or_create_http_bridge_session - - async def close_created_session_before_submit(*args, **kwargs): - session = await original_get_or_create(*args, **kwargs) - session.closed = True - return session - - monkeypatch.setattr( - service, - "_get_or_create_http_bridge_session", - close_created_session_before_submit, - ) - with pytest.raises((asyncio.CancelledError, RuntimeError)): - await async_client.post("/v1/responses", headers=headers, json=request) - assert connect_count == 3 - assert len(accepted_then_cancelled_upstream.sent_text) == 1 - assert accepted_then_cancelled_upstream.irreversible_effect_count == 1 - async with SessionLocal() as db_session: - unknown = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - assert unknown is not None - assert unknown.state == HttpBridgeRowlessRecoveryState.UNKNOWN - assert unknown.replacement_session_id is not None - assert unknown.dispatch_send_started_at is not None - journal = await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord)) - assert journal is not None - assert journal.state == HttpBridgeRecoveryAttemptState.UNKNOWN - connect_count_after_ambiguous_send = connect_count - selection_count_after_ambiguous_send = selection_count - suppressed_retry = await async_client.post( - "/v1/responses", - headers=headers, - json=request, - ) - assert suppressed_retry.status_code == 400 - assert suppressed_retry.json()["error"]["code"] == "rowless_recovery_dispatch_outcome_unknown" - assert connect_count == connect_count_after_ambiguous_send - assert selection_count == selection_count_after_ambiguous_send - assert len(accepted_then_cancelled_upstream.sent_text) == 1 - assert accepted_then_cancelled_upstream.irreversible_effect_count == 1 - return - - if recovery_mode == "installation_id_drift": - async with SessionLocal() as db_session: - await db_session.execute( - update(Account) - .where(Account.id == account_id) - .values(codex_installation_id="00000000-0000-4000-8000-000000000001") +class _ForeignPreviousResponseNotFoundAfterCreatedUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + if len(self.sent_text) == 1: + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": { + "id": "resp_bridge_prev_anchor", + "object": "response", + "status": "in_progress", + }, + }, + separators=(",", ":"), + ), + ) ) - await db_session.commit() - drifted = await async_client.post("/v1/responses", headers=headers, json=request) - assert drifted.status_code == 400 - assert drifted.json()["error"]["code"] == "rowless_recovery_actual_wire_changed" - assert connect_count == 1 - assert selection_count == 1 - assert len(recovered_upstream.sent_text) == 0 - async with SessionLocal() as db_session: - approved = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - assert approved is not None - assert approved.state == HttpBridgeRowlessRecoveryState.APPROVED - return - - if recovery_mode == "late_wire_drift": - original_transform = proxy_module.ProxyService._http_bridge_text_with_account_installation_id - - def drift_after_account_selection(self, session, request_state, text_data): - transformed = original_transform(self, session, request_state, text_data) - if request_state.rowless_recovery_authority_id is not None: - return f"{transformed} " - return transformed - - monkeypatch.setattr( - proxy_module.ProxyService, - "_http_bridge_text_with_account_installation_id", - drift_after_account_selection, - ) - drifted = await async_client.post("/v1/responses", headers=headers, json=request) - assert drifted.status_code == 400 - assert drifted.json()["error"]["code"] == "rowless_recovery_actual_wire_changed" - assert connect_count == 2 - assert selection_count == 2 - assert len(recovered_upstream.sent_text) == 0 - async with SessionLocal() as db_session: - approved = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - assert approved is not None - assert approved.state == HttpBridgeRowlessRecoveryState.APPROVED - assert approved.replacement_session_id is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord)) is None - return - - if recovery_mode == "two_closed_before_send": - proven_unsent = await async_client.post("/v1/responses", headers=headers, json=request) - assert proven_unsent.status_code == 502, proven_unsent.text - assert connect_count == 3 - assert all(not upstream.sent_text for upstream in closed_upstreams) - async with SessionLocal() as db_session: - restored = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - assert restored is not None - assert restored.state == HttpBridgeRowlessRecoveryState.APPROVED - assert restored.dispatch_send_started_at is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord)) is None - recovered_after_unsent = await async_client.post("/v1/responses", headers=headers, json=request) - assert recovered_after_unsent.status_code == 200, recovered_after_unsent.text - assert connect_count == 4 - assert len(recovered_upstream.sent_text) == 1 - async with SessionLocal() as db_session: - consumed = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - assert consumed is not None - assert consumed.state == HttpBridgeRowlessRecoveryState.CONSUMED - return - - if recovery_mode == "cancel_during_two_close_rollback": - original_rollback = RowlessRecoveryRepository.rollback_physically_unsent_after_send_marker - rollback_entered = asyncio.Event() - allow_rollback = asyncio.Event() - - async def block_rollback_until_cancelled(repository, **kwargs): - rollback_entered.set() - await allow_rollback.wait() - return await original_rollback(repository, **kwargs) - - monkeypatch.setattr( - RowlessRecoveryRepository, - "rollback_physically_unsent_after_send_marker", - block_rollback_until_cancelled, - ) - recovery_task = asyncio.create_task(async_client.post("/v1/responses", headers=headers, json=request)) - await asyncio.wait_for(rollback_entered.wait(), timeout=_TEST_SYNC_TIMEOUT_SECONDS) - recovery_task.cancel() - allow_rollback.set() - with pytest.raises(asyncio.CancelledError): - await recovery_task - assert connect_count == 3 - assert all(not upstream.sent_text for upstream in closed_upstreams) - async with SessionLocal() as db_session: - restored = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - assert restored is not None - assert restored.state == HttpBridgeRowlessRecoveryState.APPROVED - assert restored.dispatch_send_started_at is None - assert restored.replacement_session_id is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord)) is None - return - - if recovery_mode == "fresh_setup_failure_before_send": - proven_unsent = await async_client.post("/v1/responses", headers=headers, json=request) - assert proven_unsent.status_code == 502, proven_unsent.text - assert connect_count == 2 - assert not closed_upstreams[0].sent_text - async with SessionLocal() as db_session: - restored = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - assert restored is not None - assert restored.state == HttpBridgeRowlessRecoveryState.APPROVED - assert restored.dispatch_send_started_at is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord)) is None - upstreams.append(recovered_upstream) - recovered_after_unsent = await async_client.post("/v1/responses", headers=headers, json=request) - assert recovered_after_unsent.status_code == 200, recovered_after_unsent.text - assert connect_count == 3 - assert len(recovered_upstream.sent_text) == 1 - return - - if recovery_mode == "ambiguous_post_send": - ambiguous = await async_client.post("/v1/responses", headers=headers, json=request) - assert ambiguous.status_code == 502 - assert ambiguous.json()["error"]["code"] == "stream_incomplete" - assert connect_count == 2 - assert selection_count == 2 - assert len(ambiguous_upstream.sent_text) == 1 - assert ambiguous_upstream.irreversible_effect_count == 1 - async with SessionLocal() as db_session: - unknown = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - assert unknown is not None - assert unknown.state == HttpBridgeRowlessRecoveryState.UNKNOWN - journal = await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord)) - assert journal is not None - assert journal.state == HttpBridgeRecoveryAttemptState.UNKNOWN - local_retry = await async_client.post("/v1/responses", headers=headers, json=unresolved_retry) - assert local_retry.status_code == 400 - assert local_retry.json()["error"]["code"] == "rowless_recovery_dispatch_outcome_unknown" - assert connect_count == 2 - assert selection_count == 2 - assert len(ambiguous_upstream.sent_text) == 1 - assert ambiguous_upstream.irreversible_effect_count == 1 - unknown_identity_drift = await async_client.post( - "/v1/responses", - headers={**headers, "x-codex-turn-state": "unknown-drift"}, - json=request, - ) - assert unknown_identity_drift.status_code == 400 - assert unknown_identity_drift.json()["error"]["code"] == "rowless_recovery_dispatch_outcome_unknown" - assert connect_count == 2 - assert selection_count == 2 - for anchor_mutation in (None, "resp_different_stale_anchor"): - drifted_request = copy.deepcopy(request) - if anchor_mutation is None: - drifted_request.pop("previous_response_id") - else: - drifted_request["previous_response_id"] = anchor_mutation - suppressed = await async_client.post( - "/v1/responses", - headers=headers, - json=drifted_request, + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_bridge_prev_anchor", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) ) - assert suppressed.status_code == 400 - assert suppressed.json()["error"]["code"] == "rowless_recovery_dispatch_outcome_unknown" - assert connect_count == 2 - assert selection_count == 2 - assert ambiguous_upstream.irreversible_effect_count == 1 - for anchor_mutation, routing_drift in ( - (None, {"x-codex-turn-state": "unknown-combined-turn-state"}), - ("resp_different_stale_anchor", {"x-client-request-id": "unknown-combined-client"}), - (None, {"x-codex-session-id": "unknown-combined-alias"}), - ): - drifted_request = copy.deepcopy(request) - if anchor_mutation is None: - drifted_request.pop("previous_response_id") - else: - drifted_request["previous_response_id"] = anchor_mutation - suppressed = await async_client.post( - "/v1/responses", - headers={**headers, **routing_drift}, - json=drifted_request, + return + + if len(self.sent_text) == 2: + payload = json.loads(text) + previous_response_id = payload.get("previous_response_id") + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": { + "id": "resp_bridge_followup_created", + "object": "response", + "status": "in_progress", + }, + }, + separators=(",", ":"), + ), + ) ) - assert suppressed.status_code == 400 - assert suppressed.json()["error"]["code"] == "rowless_recovery_dispatch_outcome_unknown" - assert connect_count == 2 - assert selection_count == 2 - assert ambiguous_upstream.irreversible_effect_count == 1 - return - - dispatch_request = copy.deepcopy(request) - if recovery_mode == "approved_no_anchor": - dispatch_request.pop("previous_response_id") - elif recovery_mode == "approved_wrong_anchor": - dispatch_request["previous_response_id"] = "resp_different_stale_anchor" - retry_a, retry_b = await asyncio.gather( - async_client.post("/v1/responses", headers=headers, json=dispatch_request), - async_client.post("/v1/responses", headers=headers, json=dispatch_request), - ) - recovered = next(response for response in (retry_a, retry_b) if response.status_code == 200) - rejected_concurrent = next(response for response in (retry_a, retry_b) if response.status_code != 200) - assert rejected_concurrent.status_code == 400 - assert rejected_concurrent.json()["error"]["code"] in { - "rowless_recovery_dispatch_already_claimed", - "rowless_recovery_dispatch_outcome_unknown", - "rowless_recovery_already_consumed", - } - assert recovered.status_code == 200, recovered.text - assert recovered.json()["id"] == "resp_rowless_recovered_1" - assert connect_count == 2 - assert len(stale_upstream.sent_text) == 1 - assert len(recovered_upstream.sent_text) == 1 - recovered_wire = json.loads(recovered_upstream.sent_text[0]) - assert "previous_response_id" not in recovered_wire - expected_projection = replay_safety_module.project_responses_input_for_account_neutral_fresh_replay( - request["input"], - stored_count=len(request["input"]), - ) - assert expected_projection is not None - expected_rowless_input = replay_safety_module.normalize_responses_input_for_rowless_replay( - expected_projection.input_items - ) - assert expected_rowless_input is not None - assert recovered_wire["input"] == expected_rowless_input - - async with SessionLocal() as db_session: - authority = await db_session.scalar(select(HttpBridgeRowlessRecoveryAuthority)) - assert authority is not None - assert authority.state == HttpBridgeRowlessRecoveryState.CONSUMED - if recovery_mode == "live_publication_failure": - durable_alias = await db_session.scalar( - select(HttpBridgeSessionAlias).where(HttpBridgeSessionAlias.alias_value == recovered.json()["id"]) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.failed", + "response": { + "id": "resp_bridge_foreign_prev_nf", + "object": "response", + "status": "failed", + "error": { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": f"Previous response with id '{previous_response_id}' not found.", + "param": "previous_response_id", + }, + }, + }, + separators=(",", ":"), + ), + ) ) - assert durable_alias is not None - - if recovery_mode == "live_publication_failure": - assert recovered.status_code == 200 - assert len(recovered_upstream.sent_text) == 1 - assert connect_count == 2 - assert selection_count == 2 - return - - repeated_old_turn = await async_client.post("/v1/responses", headers=headers, json=request) - assert repeated_old_turn.status_code == 400 - assert repeated_old_turn.json()["error"]["code"] == "rowless_recovery_already_consumed" - assert connect_count == 2 - assert selection_count == 2 + return - for anchor_mutation in (None, "resp_different_stale_anchor"): - drifted_request = copy.deepcopy(request) - if anchor_mutation is None: - drifted_request.pop("previous_response_id") - else: - drifted_request["previous_response_id"] = anchor_mutation - suppressed = await async_client.post( - "/v1/responses", - headers=headers, - json=drifted_request, + response_id = "resp_bridge_after_error" + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": {"id": response_id, "object": "response", "status": "in_progress"}, + }, + separators=(",", ":"), + ), + ) ) - assert suppressed.status_code == 400 - assert suppressed.json()["error"]["code"] == "rowless_recovery_already_consumed" - assert connect_count == 2 - assert selection_count == 2 - assert len(recovered_upstream.sent_text) == 1 - - for anchor_mutation, routing_drift in ( - (None, {"x-codex-turn-state": "consumed-combined-turn-state"}), - ("resp_different_stale_anchor", {"x-client-request-id": "consumed-combined-client"}), - (None, {"x-codex-session-id": "consumed-combined-alias"}), - ): - drifted_request = copy.deepcopy(request) - if anchor_mutation is None: - drifted_request.pop("previous_response_id") - else: - drifted_request["previous_response_id"] = anchor_mutation - suppressed = await async_client.post( - "/v1/responses", - headers={**headers, **routing_drift}, - json=drifted_request, + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": response_id, + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) ) - assert suppressed.status_code == 400 - assert suppressed.json()["error"]["code"] == "rowless_recovery_already_consumed" - assert connect_count == 2 - assert selection_count == 2 - assert len(recovered_upstream.sent_text) == 1 - - consumed_identity_drift = await async_client.post( - "/v1/responses", - headers={**headers, "x-codex-turn-state": "consumed-drift"}, - json=request, - ) - assert consumed_identity_drift.status_code == 400 - assert consumed_identity_drift.json()["error"]["code"] == "rowless_recovery_already_consumed" - assert connect_count == 2 - assert selection_count == 2 - assert len(recovered_upstream.sent_text) == 1 - - consumed_unresolved = await async_client.post( - "/v1/responses", - headers=headers, - json=unresolved_retry, - ) - assert consumed_unresolved.status_code == 400 - assert consumed_unresolved.json()["error"]["code"] == "rowless_recovery_already_consumed" - assert connect_count == 2 - assert selection_count == 2 - - follow_up = await async_client.post( - "/v1/responses", - headers=headers, - json={ - "model": request["model"], - "instructions": request["instructions"], - "previous_response_id": recovered.json()["id"], - "prompt_cache_key": task_id, - "input": [{"role": "user", "content": "ordinary follow-up"}], - }, - ) - assert follow_up.status_code == 200, follow_up.text - assert len(recovered_upstream.sent_text) == 2 - assert selection_count == 2 -@pytest.mark.parametrize( - "identity_case", - [ - "child_both", - "client_only", - "explicit_subagent", - "metadata_subagent", - "metadata_session_drift", - "metadata_thread_drift", - "turn_metadata_subagent", - "turn_metadata_session_drift", - "turn_metadata_thread_drift", - "turn_metadata_header_conflict", - "turn_metadata_header_conflict_with_explicit_thread", - "turn_metadata_projection_conflict_with_explicit_thread", - "turn_metadata_workspace_kind_conflict_with_explicit_thread", - "turn_metadata_workspace_kind_missing_body_with_explicit_thread", - "turn_metadata_workspace_kind_missing_direct_with_explicit_thread", - "turn_metadata_malformed", - "turn_metadata_non_turn", - "turn_metadata_oversized", - "turn_metadata_oversized_with_explicit_thread", - "turn_metadata_wrong_type", - ], -) -@pytest.mark.asyncio -async def test_rowless_child_thread_sharing_root_bridge_identity_fails_closed_without_capture( - async_client, - monkeypatch, - identity_case, -): - """A child thread cannot rebase through its root's durable bridge row.""" +class _AnonymousPreviousResponseNotFoundAfterCreatedUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + def __init__(self) -> None: + super().__init__() + self.first_request_created = asyncio.Event() - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc-rowless-child-reject", - "rowless-child-reject@example.com", - ) - account = await _get_account(account_id) - upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_child_stale") - selection_count = 0 - connect_count = 0 - - async def fake_select_account_with_budget(self, deadline, **kwargs): - nonlocal selection_count - del self, deadline, kwargs - selection_count += 1 - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket(*args, **kwargs): - nonlocal connect_count - del args, kwargs - connect_count += 1 - return upstream + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + if len(self.sent_text) == 1: + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": { + "id": "resp_bridge_inflight", + "object": "response", + "status": "in_progress", + }, + }, + separators=(",", ":"), + ), + ) + ) + self.first_request_created.set() + return - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - root_session = "root-task" - child_thread = "child-task" - child_headers = {"session-id": root_session} - if identity_case == "child_both": - child_headers["thread-id"] = child_thread - child_headers["x-client-request-id"] = child_thread - elif identity_case == "client_only": - child_headers["x-client-request-id"] = child_thread - elif identity_case == "explicit_subagent": - child_headers["x-client-request-id"] = root_session - child_headers["x-openai-subagent"] = "reviewer" - else: - child_headers["x-client-request-id"] = root_session - if identity_case == "turn_metadata_header_conflict_with_explicit_thread": - child_headers["thread-id"] = root_session - if identity_case == "turn_metadata_oversized_with_explicit_thread": - child_headers["thread-id"] = root_session - if identity_case == "turn_metadata_projection_conflict_with_explicit_thread": - child_headers["thread-id"] = root_session - if identity_case in { - "turn_metadata_workspace_kind_conflict_with_explicit_thread", - "turn_metadata_workspace_kind_missing_body_with_explicit_thread", - "turn_metadata_workspace_kind_missing_direct_with_explicit_thread", - }: - child_headers["thread-id"] = root_session - request_json = { - "model": "gpt-5.1", - "previous_response_id": "resp_child_stale", - "prompt_cache_key": root_session, - "input": [{"role": "user", "content": "same-turn retry"}], - } - if identity_case == "metadata_subagent": - child_headers["x-client-request-id"] = root_session - request_json["client_metadata"] = {"x-openai-subagent": "reviewer"} - elif identity_case == "metadata_session_drift": - request_json["client_metadata"] = {"session_id": child_thread} - elif identity_case == "metadata_thread_drift": - child_headers["x-client-request-id"] = root_session - request_json["client_metadata"] = {"thread_id": child_thread} - elif identity_case == "turn_metadata_subagent": - request_json["client_metadata"] = { - "x-codex-turn-metadata": json.dumps( - {"thread_id": root_session, "parent_thread_id": root_session, "subagent_kind": "review"} + if len(self.sent_text) == 2: + payload = json.loads(text) + previous_response_id = payload.get("previous_response_id") + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": { + "id": "resp_bridge_followup_created", + "object": "response", + "status": "in_progress", + }, + }, + separators=(",", ":"), + ), + ) ) - } - elif identity_case == "turn_metadata_session_drift": - request_json["client_metadata"] = {"x-codex-turn-metadata": json.dumps({"session_id": child_thread})} - elif identity_case == "turn_metadata_thread_drift": - request_json["client_metadata"] = {"x-codex-turn-metadata": json.dumps({"thread_id": child_thread})} - elif identity_case in { - "turn_metadata_header_conflict", - "turn_metadata_header_conflict_with_explicit_thread", - }: - request_json["client_metadata"] = { - "x-codex-turn-metadata": json.dumps( - { - "session_id": root_session, - "thread_id": root_session, - "turn_id": "turn-root", - "request_kind": "turn", - } + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": f"Previous response with id '{previous_response_id}' not found.", + "param": "previous_response_id", + }, + }, + separators=(",", ":"), + ), + ) + ) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_bridge_inflight", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) + ) + return + + response_id = "resp_bridge_after_error" + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": {"id": response_id, "object": "response", "status": "in_progress"}, + }, + separators=(",", ":"), + ), ) - } - child_headers["x-codex-turn-metadata"] = json.dumps( - { - "session_id": root_session, - "thread_id": child_thread, - "turn_id": "turn-child", - "request_kind": "turn", - "parent_thread_id": root_session, - } ) - elif identity_case == "turn_metadata_projection_conflict_with_explicit_thread": - request_json["client_metadata"] = { - "session_id": root_session, - "thread_id": root_session, - "turn_id": "turn-root", - "root_turn_id": "turn-root", - "x-codex-installation-id": "installation-body", - "x-codex-window-id": "window-body", - "x-codex-turn-metadata": json.dumps( - { - "installation_id": "installation-body", - "session_id": root_session, - "thread_id": root_session, - "turn_id": "turn-root", - "root_turn_id": "turn-root", - "window_id": "window-body", - "request_kind": "turn", - } - ), - } - child_headers["x-codex-turn-metadata"] = json.dumps( - { - "installation_id": "installation-direct", - "session_id": root_session, - "thread_id": root_session, - "turn_id": "turn-root", - "root_turn_id": "turn-root", - "window_id": "window-direct", - "request_kind": "turn", - } + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": response_id, + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) ) - elif identity_case in { - "turn_metadata_workspace_kind_conflict_with_explicit_thread", - "turn_metadata_workspace_kind_missing_body_with_explicit_thread", - "turn_metadata_workspace_kind_missing_direct_with_explicit_thread", - }: - body_metadata = { - "session_id": root_session, - "thread_id": root_session, - "turn_id": "turn-root", - "request_kind": "turn", - "workspace_kind": "projectless", - } - direct_metadata = { - **body_metadata, - "workspace_kind": "project", - } - if identity_case == "turn_metadata_workspace_kind_missing_body_with_explicit_thread": - body_metadata.pop("workspace_kind") - direct_metadata["workspace_kind"] = "projectless" - elif identity_case == "turn_metadata_workspace_kind_missing_direct_with_explicit_thread": - direct_metadata.pop("workspace_kind") - request_json["client_metadata"] = { - "session_id": root_session, - "thread_id": root_session, - "turn_id": "turn-root", - "x-codex-turn-metadata": json.dumps(body_metadata), - } - child_headers["x-codex-turn-metadata"] = json.dumps(direct_metadata) - elif identity_case == "turn_metadata_malformed": - request_json["client_metadata"] = {"x-codex-turn-metadata": "not-json"} - elif identity_case == "turn_metadata_non_turn": - request_json["client_metadata"] = { - "x-codex-turn-metadata": json.dumps({"thread_id": root_session, "request_kind": "compact"}) - } - elif identity_case in {"turn_metadata_oversized", "turn_metadata_oversized_with_explicit_thread"}: - oversized_metadata = { - "session_id": root_session, - "thread_id": root_session, - "turn_id": "turn-root", - "request_kind": "turn", - "padding": "x" * (17 * 1024), - } - if identity_case == "turn_metadata_oversized_with_explicit_thread": - child_headers["x-codex-turn-metadata"] = json.dumps(oversized_metadata) - else: - request_json["client_metadata"] = {"x-codex-turn-metadata": json.dumps(oversized_metadata)} - elif identity_case == "turn_metadata_wrong_type": - request_json["client_metadata"] = {"x-codex-turn-metadata": 7} - response = await async_client.post( - "/v1/responses", - headers=child_headers, - json=request_json, - ) - if identity_case in { - "turn_metadata_header_conflict_with_explicit_thread", - "turn_metadata_oversized_with_explicit_thread", - "turn_metadata_projection_conflict_with_explicit_thread", - "turn_metadata_workspace_kind_conflict_with_explicit_thread", - "turn_metadata_workspace_kind_missing_body_with_explicit_thread", - "turn_metadata_workspace_kind_missing_direct_with_explicit_thread", - }: - assert response.status_code == 400 - assert response.json()["error"]["code"] == "rowless_recovery_identity_metadata_invalid" - assert selection_count == 0 - assert connect_count == 0 - assert upstream.sent_text == [] - async with SessionLocal() as db_session: - authorities = list((await db_session.scalars(select(HttpBridgeRowlessRecoveryAuthority))).all()) - assert authorities == [] - return - assert response.status_code == 502 - assert response.json()["error"]["code"] != "previous_response_recovery_authorization_required" - async with SessionLocal() as db_session: - authorities = list((await db_session.scalars(select(HttpBridgeRowlessRecoveryAuthority))).all()) - assert authorities == [] - -@pytest.mark.parametrize( - "resolution_mode", - [ - "administrator", - "administrator_client_request_id_fallback", - "administrator_responses_lite_0149", - "administrator_child_client_request_id", - "automatic", - "automatic_legacy_unknown", - "automatic_legacy_consumed", - "automatic_legacy_unknown_missing_thread", - "automatic_legacy_consumed_missing_thread", - "automatic_session_creation_failure", - "automatic_cooldown_before_submit", - "automatic_incremental", - "automatic_terminal_persistence_failure", - "automatic_terminal_invalid_tool_manifest", - ], -) -@pytest.mark.asyncio -async def test_marker_backed_rowless_rebase_recovers_mismatched_pending_call_without_clearing_fence( - async_client, - app_instance, - monkeypatch, - resolution_mode, -): - """A retained marker admits exactly one safe recovery owner.""" - - administrator_mode = resolution_mode.startswith("administrator") - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc-marker-rowless-rebase", - "marker-rowless-rebase@example.com", - ) - account = await _get_account(account_id) - source_upstream = _ClosingInterruptedCustomToolUpstreamWebSocket( - "resp_marker_rowless_source", - tool_call_type="custom_tool_call", - ) - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_bridge_custom_1") - recovered_upstream = _FakeBridgeUpstreamWebSocket( - "resp_marker_rowless_recovered", - completed_output=( - [ - { - "type": "computer_call", - "call_id": "call_computer", - "action": {"type": "screenshot"}, - } - ] - if resolution_mode == "automatic_terminal_invalid_tool_manifest" - else None - ), - ) - upstreams = [source_upstream, stale_upstream, recovered_upstream] - connect_count = 0 - selection_count = 0 - preflight_rollback_count = 0 +class _TwoFollowupsPreviousResponseNotFoundUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + def __init__(self) -> None: + super().__init__() + self.first_followup_created = asyncio.Event() - async def fake_select_account_with_budget(self, deadline, **kwargs): - nonlocal selection_count - del self, deadline, kwargs - selection_count += 1 - return AccountSelection(account=account, error_message=None, error_code=None) + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + if len(self.sent_text) == 1: + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": { + "id": "resp_bridge_prev_anchor_a", + "object": "response", + "status": "in_progress", + }, + }, + separators=(",", ":"), + ), + ) + ) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_bridge_prev_anchor_a", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) + ) + return - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - nonlocal connect_count - del headers, access_token, account_id_header, base_url, session - upstream = upstreams[connect_count] - connect_count += 1 - return upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - task_id = "01a02edf-376c-7c13-a7de-08715e492fab" - headers = { - "session-id": task_id, - "thread-id": task_id, - "x-client-request-id": task_id, - } - stored_input = [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": f"sanitized stored item {index}"}], - } - for index in range(8) - ] - first = await async_client.post( - "/v1/responses", - headers=headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "prompt_cache_key": task_id, - "input": stored_input, - }, - ) - assert first.status_code == 200, first.text - - rejected_anchor = await async_client.post( - "/v1/responses", - headers=headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "prompt_cache_key": task_id, - "input": [{"role": "user", "content": "scheduled continue"}], - }, - ) - assert rejected_anchor.status_code == 400, rejected_anchor.text - assert rejected_anchor.json()["error"]["code"] == "previous_response_pending_call_resolution_required" - assert connect_count == 2 - selection_count_after_marker = selection_count - - async with SessionLocal() as db_session: - marker = await db_session.scalar( - select(HttpBridgeSessionRecord).where(HttpBridgeSessionRecord.recovery_required_anchor_hash.is_not(None)) - ) - assert marker is not None - marker_id = marker.id - assert marker.recovery_required_account_id == account.id - assert marker.recovery_required_attempt_fingerprint is None - - from tests.unit.test_replay_safety import _rehydrate_sanitized_pending_settlement_shapes - - mismatched_complete_input = copy.deepcopy(_rehydrate_sanitized_pending_settlement_shapes()[1][0]) - if administrator_mode: - historical_output = next( - item - for item in mismatched_complete_input - if isinstance(item, dict) and item.get("type") in {"custom_tool_call_output", "function_call_output"} - ) - historical_output["output"] = [ - {"type": "input_text", "text": "settled fixture output"}, - {"type": "input_text", "text": ""}, - ] - historical_agent = next( - item for item in mismatched_complete_input if isinstance(item, dict) and item.get("type") == "agent_message" - ) - historical_agent_content = cast(list[object], historical_agent["content"]) - historical_agent_content.append({"type": "encrypted_content", "encrypted_content": "opaque-response-state"}) - historical_function_call = next( - ( - item - for item in mismatched_complete_input - if isinstance(item, dict) and item.get("type") == "function_call" - ), - None, - ) - if historical_function_call is not None: - historical_function_call["namespace"] = "collaboration" - if resolution_mode == "administrator_responses_lite_0149": - mismatched_complete_input = [ - { - "type": "additional_tools", - "role": "developer", - "tools": [ - { - "type": "namespace", - "name": "functions", - "description": "", - "tools": [ - { - "type": "function", - "name": "lookup", - "description": "Lookup a fixture.", - "strict": False, - "defer_loading": True, - "parameters": {"type": "object", "properties": {}}, - } - ], - }, - { - "type": "tool_search", - "execution": "client", - "description": "Search deferred tools.", - "parameters": { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "Search query for deferred tools.", - }, - "limit": { - "type": "number", - "description": "Maximum number of tools to return. Defaults to 8.", + if len(self.sent_text) == 2: + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": { + "id": "resp_bridge_prev_anchor_b", + "object": "response", + "status": "in_progress", + }, + }, + separators=(",", ":"), + ), + ) + ) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_bridge_prev_anchor_b", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, }, }, - "required": ["query"], - "additionalProperties": False, }, - }, - ], - }, - {"role": "developer", "content": "Use the declared tools."}, - *mismatched_complete_input, - ] - request = { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "prompt_cache_key": task_id, - "input": mismatched_complete_input, - } - if resolution_mode == "administrator_responses_lite_0149": - request["reasoning"] = { - "context": "all_turns", - "effort": "high", - "summary": "auto", - } - body_turn_metadata = { - "installation_id": "installation-before-account-selection", - "session_id": task_id, - "thread_id": task_id, - "turn_id": "019f-turn-id", - "root_turn_id": "019f-turn-id", - "window_id": "window-a", - "workspace_kind": "projectless", - "request_kind": "turn", - "tool_namespaces_info": { - "functions": { - "name": "functions", - "functions": { - f"tool_{index:03d}": { - "name": f"tool_{index:03d}", - "direct": False, - "code_mode_name": None, - "deferred": True, - "source": {"kind": "harness"}, - } - for index in range(160) - }, - } - }, - } - body_turn_metadata_json = json.dumps(body_turn_metadata, separators=(",", ":")) - assert len(body_turn_metadata_json.encode()) > 16 * 1024 - request["client_metadata"] = { - "session_id": task_id, - "thread_id": task_id, - "turn_id": "019f-turn-id", - "root_turn_id": "019f-turn-id", - "x-codex-installation-id": "installation-before-account-selection", - "x-codex-window-id": "window-a", - "x-codex-turn-metadata": body_turn_metadata_json, - } - recovery_headers = ( - {**headers, "x-codex-turn-state": "turn-state-marker-rowless-capture"} if administrator_mode else headers - ) - if resolution_mode == "administrator_responses_lite_0149": - recovery_headers["x-codex-installation-id"] = "installation-before-account-selection" - recovery_headers["x-codex-window-id"] = "window-a" - recovery_headers["x-codex-turn-metadata"] = json.dumps( - { - "installation_id": "installation-before-account-selection", - "session_id": task_id, - "thread_id": task_id, - "turn_id": "019f-turn-id", - "root_turn_id": "019f-turn-id", - "window_id": "window-a", - "workspace_kind": "projectless", - "request_kind": "turn", - }, - separators=(",", ":"), - ) - if resolution_mode in { - "administrator_client_request_id_fallback", - "administrator_child_client_request_id", - }: - recovery_headers.pop("thread-id") - if resolution_mode == "administrator_client_request_id_fallback": - request["client_metadata"] = { - "session_id": task_id, - "thread_id": task_id, - "turn_id": "turn-fallback", - "x-codex-turn-metadata": json.dumps( - { - "session_id": task_id, - "thread_id": task_id, - "turn_id": "turn-fallback", - "request_kind": "turn", - } - ), - } - recovery_headers["x-codex-turn-metadata"] = json.dumps( - { - "session_id": task_id, - "thread_id": task_id, - "turn_id": "turn-fallback", - "request_kind": "turn", - } - ) - if resolution_mode == "administrator_child_client_request_id": - recovery_headers["x-client-request-id"] = "child-task" - request_model = proxy_module.ResponsesRequest.model_validate(request) - request_affinity = proxy_module._sticky_key_for_responses_request( - request_model, - recovery_headers, - codex_session_affinity=True, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - request_key = proxy_module._make_http_bridge_session_key( - request_model, - headers=recovery_headers, - affinity=request_affinity, - api_key=None, - request_id="marker-rowless-capture", - explicit_prompt_cache_key=task_id, - ) - if administrator_mode: - assert request_key.affinity_kind == "turn_state_header" - assert request_key.affinity_key != marker.session_key_value - else: - assert request_key.affinity_kind == marker.session_key_kind - assert request_key.affinity_key == marker.session_key_value - service = get_proxy_service_for_app(app_instance) - if not administrator_mode: - lookup = await service._durable_bridge.lookup_request_targets( - session_key_kind=request_key.affinity_kind, - session_key_value=request_key.affinity_key, - api_key_id=None, - turn_state=None, - session_header=task_id, - previous_response_id=None, - ) - assert lookup is not None - assert lookup.session_id == marker_id - assert lookup.recovery_is_required_for_latest_anchor() - captured = await async_client.post("/v1/responses", headers=recovery_headers, json=request) - assert captured.status_code == 400, captured.text - if resolution_mode == "administrator_child_client_request_id": - assert captured.json()["error"]["code"] == "previous_response_pending_call_resolution_required" - assert connect_count == 2 - assert selection_count == selection_count_after_marker - async with SessionLocal() as db_session: - assert list((await db_session.scalars(select(HttpBridgeRowlessRecoveryAuthority))).all()) == [] - retained_marker = await db_session.get(HttpBridgeSessionRecord, marker_id) - assert retained_marker is not None - assert retained_marker.recovery_required_attempt_fingerprint is None - return - assert captured.json()["error"]["code"] == "previous_response_recovery_authorization_required" - assert captured.json()["error"]["action"] == "retry_same_turn_after_admin_approval" - assert connect_count == 2 - assert selection_count == selection_count_after_marker - - repeated_capture_headers = recovery_headers - if resolution_mode == "administrator_client_request_id_fallback": - repeated_capture_headers = {**recovery_headers, "thread-id": task_id} - repeated_capture = await async_client.post("/v1/responses", headers=repeated_capture_headers, json=request) - assert repeated_capture.status_code == 400, repeated_capture.text - assert repeated_capture.json()["error"]["code"] == "previous_response_recovery_authorization_required" - assert connect_count == 2 - assert selection_count == selection_count_after_marker - - async with SessionLocal() as db_session: - authorities = list((await db_session.scalars(select(HttpBridgeRowlessRecoveryAuthority))).all()) - assert len(authorities) == 1 - authority = authorities[0] - assert authority is not None - assert authority.state == HttpBridgeRowlessRecoveryState.CAPTURED - assert authority.origin_marker_session_id == marker_id - assert authority.request_account_neutral - - if resolution_mode.startswith("automatic"): - if resolution_mode.startswith("automatic_legacy_"): - async with SessionLocal() as db_session: - legacy_authority = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - assert legacy_authority is not None - legacy_authority.origin_marker_session_id = None - if "unknown" in resolution_mode: - legacy_authority.state = HttpBridgeRowlessRecoveryState.UNKNOWN - legacy_authority.replacement_session_id = marker_id - legacy_authority.dispatch_request_id = "legacy-rowless-unknown" - legacy_authority.wire_request_fingerprint = legacy_authority.actual_wire_fingerprint - legacy_authority.dispatch_send_started_at = utcnow() - db_session.add( - HttpBridgeRecoveryAttemptRecord( - session_id=marker_id, - request_fingerprint=legacy_authority.actual_wire_fingerprint, - request_id="legacy-rowless-unknown", - account_id=account.id, - model="gpt-5.1", - replay_safe=False, - state=HttpBridgeRecoveryAttemptState.UNKNOWN, - ) - ) - else: - legacy_authority.state = HttpBridgeRowlessRecoveryState.CONSUMED - legacy_authority.consumed_response_id_hash = "c" * 64 - legacy_authority.consumed_at = utcnow() - await db_session.commit() - await db_session.refresh(legacy_authority) - assert legacy_authority.origin_marker_session_id is None - assert legacy_authority.state == ( - HttpBridgeRowlessRecoveryState.UNKNOWN - if "unknown" in resolution_mode - else HttpBridgeRowlessRecoveryState.CONSUMED + separators=(",", ":"), + ), ) - - if resolution_mode == "automatic_terminal_persistence_failure": - - async def fail_rowless_terminal_settlement(*args, **kwargs): - del args, kwargs - raise RuntimeError("injected rowless terminal persistence failure") - - monkeypatch.setattr( - RowlessRecoveryRepository, - "settle_completed", - fail_rowless_terminal_settlement, ) + return - if resolution_mode == "automatic_session_creation_failure": - - async def fail_recovery_session_creation(*args, **kwargs): - del args, kwargs - raise proxy_module.ProxyResponseError( - 502, - proxy_module.openai_error("upstream_unavailable", "injected recovery session creation failure"), + if len(self.sent_text) == 3: + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": { + "id": "resp_bridge_followup_a", + "object": "response", + "status": "in_progress", + }, + }, + separators=(",", ":"), + ), ) - - monkeypatch.setattr( - proxy_module.ProxyService, - "_get_or_create_http_bridge_session", - fail_recovery_session_creation, ) + self.first_followup_created.set() + return - automatic_full_resend = [ - *stored_input, - { - "type": "reasoning", - "id": "rs_08639659bba14680016a8a0902e9a887d090946ff27b898f05", - "content": None, - "encrypted_content": "opaque", - "summary": [], - }, - { - "type": "custom_tool_call", - "call_id": "call_custom_shell", - "name": "shell", - "input": "pwd", - }, - { - "type": "custom_tool_call_output", - "call_id": "call_custom_shell", - "output": "verified pending-call settlement", - }, - { - "type": "message", - "role": "assistant", - "status": "completed", - "phase": "final_answer", - "content": [{"type": "output_text", "text": "completed prior result"}], - }, - {"role": "user", "content": "continue the original task"}, - ] - if resolution_mode == "automatic_incremental": - automatic_full_resend = [{"role": "user", "content": "continue the original task"}] - automatic_headers = dict(headers) - automatic_client_metadata = None - if resolution_mode.endswith("_missing_thread"): - automatic_headers.pop("thread-id") - if resolution_mode.startswith("automatic") and not resolution_mode.startswith("automatic_legacy_"): - official_headers, automatic_client_metadata = _official_codex_turn_carriers( - task_id, - f"turn-marker-rowless-{resolution_mode}", + if len(self.sent_text) == 4: + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": { + "id": "resp_bridge_followup_b", + "object": "response", + "status": "in_progress", + }, + }, + separators=(",", ":"), + ), + ) ) - automatic_headers.update(official_headers) - if resolution_mode == "automatic_cooldown_before_submit": - automatic_headers["x-codex-turn-state"] = "turn-state-marker-rowless-cooldown" - - async def active_retry_circuit(self, session): - del self, session - return SimpleNamespace( - retry_after_seconds=60.0, - last_detail="stream_idle_timeout", + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": ( + "Cannot continue conversation because upstream lost resp_bridge_prev_anchor_a." + ), + "param": "previous_response_id", + }, + }, + separators=(",", ":"), + ), ) - - monkeypatch.setattr( - proxy_module.ProxyService, - "_http_bridge_retry_circuit_snapshot", - active_retry_circuit, ) - monkeypatch.setattr( - http_bridge_streaming_module, - "_http_bridge_continuity_bound_without_safe_replay", - lambda request_state: True, + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_bridge_followup_b", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) ) - original_preflight_rollback = proxy_module.ProxyService._rollback_rowless_preflight_setup_failure_if_unbound - - async def record_preflight_rollback(self, request_state): - nonlocal preflight_rollback_count - preflight_rollback_count += 1 - await original_preflight_rollback(self, request_state) + return - monkeypatch.setattr( - proxy_module.ProxyService, - "_rollback_rowless_preflight_setup_failure_if_unbound", - record_preflight_rollback, - ) - automatic_request = { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "previous_response_id": "resp_bridge_custom_1", - "prompt_cache_key": task_id, - "input": automatic_full_resend, - } - if automatic_client_metadata is not None: - automatic_request["client_metadata"] = automatic_client_metadata - automatic_request_task = asyncio.create_task( - async_client.post( - "/v1/responses", - headers=automatic_headers, - json=automatic_request, - ) - ) - automatic_recovery = await automatic_request_task - if resolution_mode.startswith("automatic_legacy_"): - assert automatic_recovery.status_code == 400, automatic_recovery.text - expected_code = ( - "rowless_recovery_dispatch_outcome_unknown" - if "unknown" in resolution_mode - else "rowless_recovery_already_consumed" - ) - assert automatic_recovery.json()["error"]["code"] == expected_code - assert connect_count == 2 - assert selection_count == selection_count_after_marker - assert not recovered_upstream.sent_text - async with SessionLocal() as db_session: - marker = await db_session.get(HttpBridgeSessionRecord, marker_id) - assert marker is not None - assert marker.recovery_required_attempt_fingerprint is None - return - if resolution_mode == "automatic_incremental": - assert automatic_recovery.status_code == 400, automatic_recovery.text - assert automatic_recovery.json()["error"]["code"] == ("previous_response_recovery_authorization_required") - assert connect_count == 2 - assert selection_count == selection_count_after_marker - assert not recovered_upstream.sent_text - async with SessionLocal() as db_session: - retained_authority = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - retained_marker = await db_session.get(HttpBridgeSessionRecord, marker_id) - assert retained_authority is not None - assert retained_authority.state == HttpBridgeRowlessRecoveryState.CAPTURED - assert retained_authority.authorization_mode is None - assert retained_authority.dispatch_request_id is None - assert retained_authority.wire_request_fingerprint is None - assert retained_authority.dispatch_send_started_at is None - assert retained_authority.replacement_session_id is None - assert retained_marker is not None - assert retained_marker.recovery_required_attempt_fingerprint is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord.id)) is None - return - if resolution_mode == "automatic_session_creation_failure": - assert automatic_recovery.status_code == 502, automatic_recovery.text - assert automatic_recovery.json()["error"]["code"] == "upstream_unavailable" - assert connect_count == 2 - assert not recovered_upstream.sent_text - async with SessionLocal() as db_session: - retained_authority = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - retained_marker = await db_session.get(HttpBridgeSessionRecord, marker_id) - assert retained_authority is not None - assert retained_authority.state == HttpBridgeRowlessRecoveryState.APPROVED - assert retained_marker is not None - assert retained_marker.recovery_required_attempt_fingerprint is None - assert retained_marker.recovery_required_attempt_request_id is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord.id)) is None - return - if resolution_mode == "automatic_cooldown_before_submit": - assert automatic_recovery.status_code == 503, automatic_recovery.text - assert automatic_recovery.json()["error"]["code"] == "upstream_request_timeout" - # The startup cooldown is checked after the replacement socket is - # opened, but before any request body or dispatch marker is sent. - assert connect_count == 3 - assert not recovered_upstream.sent_text - assert preflight_rollback_count == 1 - async with SessionLocal() as db_session: - retained_authority = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - retained_marker = await db_session.get(HttpBridgeSessionRecord, marker_id) - assert retained_authority is not None - assert retained_authority.state == HttpBridgeRowlessRecoveryState.APPROVED - assert retained_authority.authorization_mode == ROWLESS_AUTHORIZATION_MODE_AUTOMATIC - assert retained_authority.dispatch_request_id is None - assert retained_authority.wire_request_fingerprint is None - assert retained_authority.dispatch_send_started_at is None - assert retained_authority.replacement_session_id is None - assert retained_marker is not None - assert retained_marker.recovery_required_attempt_fingerprint is None - assert await db_session.scalar(select(HttpBridgeRecoveryAttemptRecord.id)) is None - return - if resolution_mode in { - "automatic_terminal_persistence_failure", - "automatic_terminal_invalid_tool_manifest", - }: - assert automatic_recovery.status_code == 502, automatic_recovery.text - assert automatic_recovery.json()["error"]["code"] == "bridge_continuity_persistence_failed" - assert connect_count == 3 - assert len(recovered_upstream.sent_text) == 1 - async with SessionLocal() as db_session: - marker = await db_session.get(HttpBridgeSessionRecord, marker_id) - assert marker is not None - assert marker.latest_response_id == "resp_bridge_custom_1" - assert marker.recovery_required_anchor_hash is not None - assert marker.recovery_required_attempt_fingerprint is not None - attempt = await db_session.scalar( - select(HttpBridgeRecoveryAttemptRecord).where( - HttpBridgeRecoveryAttemptRecord.session_id == marker_id, - HttpBridgeRecoveryAttemptRecord.request_id.is_not(None), - ) - ) - assert attempt is not None - assert attempt.state == HttpBridgeRecoveryAttemptState.UNKNOWN - assert attempt.response_id is None - replacement_alias = await db_session.scalar( - select(HttpBridgeSessionAlias).where( - HttpBridgeSessionAlias.alias_kind == "previous_response_id", - HttpBridgeSessionAlias.alias_value == "resp_marker_rowless_recovered", - ) - ) - assert replacement_alias is None - return - assert automatic_recovery.status_code == 200, automatic_recovery.text - assert connect_count == 3 - assert selection_count == selection_count_after_marker + 1 - assert len(recovered_upstream.sent_text) == 1 - automatic_wire = json.loads(recovered_upstream.sent_text[0]) - assert "previous_response_id" not in automatic_wire - async with SessionLocal() as db_session: - retained_authority = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - marker = await db_session.get(HttpBridgeSessionRecord, marker_id) - assert retained_authority is not None - assert retained_authority.state == HttpBridgeRowlessRecoveryState.CONSUMED - assert retained_authority.authorization_mode == ROWLESS_AUTHORIZATION_MODE_AUTOMATIC - assert retained_authority.consumed_response_id_hash is not None - assert marker is not None - assert marker.latest_response_id == automatic_recovery.json()["id"] - assert marker.recovery_required_anchor_hash is None - assert marker.recovery_required_attempt_fingerprint is None - attempt = await db_session.scalar( - select(HttpBridgeRecoveryAttemptRecord).where(HttpBridgeRecoveryAttemptRecord.session_id == marker_id) + response_id = "resp_bridge_after_error" + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": {"id": response_id, "object": "response", "status": "in_progress"}, + }, + separators=(",", ":"), + ), ) - assert attempt is not None - assert attempt.state == HttpBridgeRecoveryAttemptState.REPLAYED - return - - async with SessionLocal() as db_session: - repository = RowlessRecoveryRepository(db_session) - authority = await db_session.get(HttpBridgeRowlessRecoveryAuthority, authority.id) - assert authority is not None - challenge = await repository.issue_challenge( - authority_id=authority.id, - generation=authority.generation, - ) - receipt = RowlessCheckpointReceipt( - schema="qk_http_bridge_rowless_checkpoint_receipt_v1", - remote_session_jsonl_sha256="a" * 64, - remote_session_jsonl_size_bytes=2048, - remote_session_jsonl_last_offset=2048, - full_checkpoint_tool_ledger_digest="b" * 64, - unresolved_count=0, - task_identity=task_id, - session_identity=task_id, - strong_session_hash=authority.strong_session_hash, - task_authority_digest=authority.captured_task_authority_digest, - captured_input_item_count=authority.captured_input_item_count, - captured_input_fingerprint=authority.captured_input_fingerprint, - non_input_contract_fingerprint=authority.non_input_contract_fingerprint, - retained_request_direct_call_ledger_digest=authority.settled_direct_call_ledger_digest, - captured_projected_payload_fingerprint=authority.projected_payload_fingerprint, - captured_actual_wire_fingerprint=authority.actual_wire_fingerprint, - captured_request_binding_provenance="server_challenge", - ) - approved = await repository.approve( - authority_id=authority.id, - generation=authority.generation, - challenge=challenge.challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement="operator_acknowledged_semantic_rebase", - approved_actor="trusted-dashboard-operator", - request_id="marker-rowless-admin-approval", - ) - assert approved.state == HttpBridgeRowlessRecoveryState.APPROVED - - recovered = await async_client.post("/v1/responses", headers=recovery_headers, json=request) - assert recovered.status_code == 200, recovered.text - assert connect_count == 3 - assert selection_count == selection_count_after_marker + 1 - assert len(recovered_upstream.sent_text) == 1 - recovered_wire = json.loads(recovered_upstream.sent_text[0]) - assert "previous_response_id" not in recovered_wire - assert recovered_wire["input"] != stored_input - if administrator_mode: - assert "encrypted_content" not in json.dumps(recovered_wire["input"]) - assert all( - part != {"type": "input_text", "text": ""} - for item in recovered_wire["input"] - if isinstance(item, dict) and isinstance(item.get("output"), list) - for part in item["output"] ) - - async with SessionLocal() as db_session: - authority = await db_session.scalar(select(HttpBridgeRowlessRecoveryAuthority)) - marker = await db_session.get(HttpBridgeSessionRecord, marker_id) - assert authority is not None - assert authority.state == HttpBridgeRowlessRecoveryState.CONSUMED - assert authority.replacement_session_id == marker_id - assert marker is not None - assert marker.latest_response_id == recovered.json()["id"] - assert marker.recovery_required_anchor_hash is None - assert marker.recovery_required_account_id is None - assert marker.recovery_required_attempt_fingerprint is None - attempt = await db_session.scalar( - select(HttpBridgeRecoveryAttemptRecord).where(HttpBridgeRecoveryAttemptRecord.session_id == marker_id) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": response_id, + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) ) - assert attempt is not None - assert attempt.state == HttpBridgeRecoveryAttemptState.REPLAYED - assert attempt.response_id == recovered.json()["id"] - - -@pytest_asyncio.fixture(autouse=True) -async def _cleanup_http_bridge_sessions(app_instance): - yield - service = get_proxy_service_for_app(app_instance) - async with service._http_bridge_lock: - sessions = list(service._http_bridge_sessions.values()) - inflight_sessions = list(service._http_bridge_inflight_sessions.values()) - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() - service._http_bridge_previous_response_index.clear() - for session in sessions: - await service._close_http_bridge_session(session) - for inflight_future in inflight_sessions: - if not inflight_future.done(): - inflight_future.cancel() - - -def _encode_jwt(payload: dict) -> str: - raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") - body = base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") - return f"header.{body}.sig" - - -def _make_auth_json(account_id: str, email: str, *, plan_type: str = "plus") -> dict: - payload = { - "email": email, - "chatgpt_account_id": account_id, - "https://api.openai.com/auth": {"chatgpt_plan_type": plan_type}, - } - return { - "tokens": { - "idToken": _encode_jwt(payload), - "accessToken": "access-token", - "refreshToken": "refresh-token", - "accountId": account_id, - }, - } - - -async def _collect_sse_events( - async_client, - path: str, - *, - json_body: dict, - headers: dict[str, str] | None = None, -) -> list[dict]: - async with async_client.stream("POST", path, json=json_body, headers=headers) as response: - assert response.status_code == 200 - lines = [line async for line in response.aiter_lines() if line.startswith("data: ")] - return [ - event - for line in lines - if line[6:] != "[DONE]" - if (event := json.loads(line[6:])).get("type") != "codex.keepalive" - ] - - -async def _collect_sse_events_with_headers( - async_client, - path: str, - *, - json_body: dict, - headers: dict[str, str] | None = None, -) -> tuple[list[dict], dict[str, str]]: - async with async_client.stream("POST", path, json=json_body, headers=headers) as response: - assert response.status_code == 200 - response_headers = dict(response.headers) - lines = [line async for line in response.aiter_lines() if line.startswith("data: ")] - return [ - event - for line in lines - if line[6:] != "[DONE]" - if (event := json.loads(line[6:])).get("type") != "codex.keepalive" - ], response_headers - - -def _assert_created_text_delta_completed(events: list[dict]) -> None: - assert [event["type"] for event in events] == [ - "response.created", - "response.output_text.delta", - "response.completed", - ] - assert events[1]["delta"] == "OK" - - -async def _import_account(async_client, account_id: str, email: str, *, plan_type: str = "plus") -> str: - auth_json = _make_auth_json(account_id, email, plan_type=plan_type) - files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} - response = await async_client.post("/api/accounts/import", files=files) - assert response.status_code == 200 - return response.json()["accountId"] - - -async def _get_account(account_id: str) -> Account: - async with SessionLocal() as session: - result = await session.execute(select(Account).where(Account.id == account_id)) - account = result.scalar_one() - session.expunge(account) - return account - -async def _wait_for_event(event: asyncio.Event, *, timeout: float = _TEST_SYNC_TIMEOUT_SECONDS) -> None: - await asyncio.wait_for(event.wait(), timeout=timeout) +class _TwoSameAnchorFollowupsPreviousResponseNotFoundUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + def __init__(self) -> None: + super().__init__() + self.first_followup_created = asyncio.Event() -async def _run_concurrent_marker_recoveries_at_response_create_gate( - *, - service: proxy_module.ProxyService, - monkeypatch: pytest.MonkeyPatch, - recover_once, - recover_peer=None, -) -> tuple[list[Any], int, list[tuple[str, str, str | None, str | None]]]: - """Hold one gate until its peer owns the UNKNOWN recovery journal.""" - - original_submit = service._submit_http_bridge_request_with_handoff - original_record_attempt = service._durable_bridge.record_recovery_attempt - second_submit_arrived = asyncio.Event() - other_request_completed_before_submit = asyncio.Event() - recovery_attempt_recorded = asyncio.Event() - marker_submit_count = 0 - record_observations: list[tuple[str, str, str | None, str | None]] = [] - - async def monitored_record_attempt(**kwargs): - attempt = await original_record_attempt(**kwargs) - record_observations.append( - ( - kwargs["request_fingerprint"], - kwargs["request_id"], - getattr(attempt.state, "value", None) if attempt is not None else None, - attempt.request_id if attempt is not None else None, + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + if len(self.sent_text) == 1: + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": { + "id": "resp_bridge_prev_anchor_shared", + "object": "response", + "status": "in_progress", + }, + }, + separators=(",", ":"), + ), + ) ) - ) - if ( - attempt is not None - and attempt.state == HttpBridgeRecoveryAttemptState.UNKNOWN - and attempt.request_id == kwargs["request_id"] - ): - recovery_attempt_recorded.set() - return attempt - - async def submit_with_barrier(session, **kwargs): - nonlocal marker_submit_count - request_state = kwargs["request_state"] - if not ( - request_state.previous_response_id is None - and request_state.fresh_upstream_request_is_retry_safe - and request_state.fresh_upstream_request_text - ): - return await original_submit(session, **kwargs) - marker_submit_count += 1 - if marker_submit_count == 1: - await session.response_create_gate.acquire() - try: - second_arrival = asyncio.create_task(second_submit_arrived.wait()) - peer_completion = asyncio.create_task(other_request_completed_before_submit.wait()) - done, pending = await asyncio.wait( - {second_arrival, peer_completion}, - timeout=_TEST_SYNC_TIMEOUT_SECONDS, - return_when=asyncio.FIRST_COMPLETED, + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_bridge_prev_anchor_shared", + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), ) - for waiter in pending: - waiter.cancel() - if not done: - raise TimeoutError("peer recovery did not reach a durable decision") - if second_submit_arrived.is_set(): - await _wait_for_event(recovery_attempt_recorded) - finally: - session.response_create_gate.release() - elif marker_submit_count == 2: - second_submit_arrived.set() - return await original_submit(session, **kwargs) - - monkeypatch.setattr(service._durable_bridge, "record_recovery_attempt", monitored_record_attempt) - monkeypatch.setattr(service, "_submit_http_bridge_request_with_handoff", submit_with_barrier) - - async def run_recovery(recover): - try: - return await recover() - finally: - if not second_submit_arrived.is_set(): - other_request_completed_before_submit.set() - - results = list(await asyncio.gather(run_recovery(recover_once), run_recovery(recover_peer or recover_once))) - return results, marker_submit_count, record_observations + ) + return + if len(self.sent_text) == 2: + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": { + "id": "resp_bridge_followup_same_anchor_a", + "object": "response", + "status": "in_progress", + }, + }, + separators=(",", ":"), + ), + ) + ) + self.first_followup_created.set() + return -async def _replace_http_bridge_upstream_reader( - service: proxy_module.ProxyService, - session: proxy_module._HTTPBridgeSession, - upstream: proxy_module.UpstreamWebSocket, -) -> None: - reader = session.upstream_reader - if reader is not None: - reader.cancel() - with contextlib.suppress(asyncio.CancelledError): - await reader - session.upstream = upstream - session.closed = False - session.upstream_control = proxy_module._WebSocketUpstreamControl() - session.upstream_reader = asyncio.create_task(service._relay_http_bridge_upstream_messages(session)) - - -class _SettingsCache: - def __init__(self, settings: DashboardSettings) -> None: - self._settings = settings - - async def get(self) -> DashboardSettings: - return self._settings - - -def _make_app_settings( - *, - enabled: bool, - max_sessions: int = 128, - queue_limit: int = 8, - admission_wait_timeout_seconds: float = 0.05, - codex_idle_ttl_seconds: float = 900.0, - codex_prewarm_enabled: bool = False, - instance_id: str = "instance-a", - instance_ring: list[str] | None = None, -) -> Settings: - return Settings( - http_responses_session_bridge_enabled=enabled, - http_responses_session_bridge_idle_ttl_seconds=120.0, - http_responses_session_bridge_codex_idle_ttl_seconds=codex_idle_ttl_seconds, - http_responses_session_bridge_codex_prewarm_enabled=codex_prewarm_enabled, - http_responses_session_bridge_max_sessions=max_sessions, - http_responses_session_bridge_queue_limit=queue_limit, - http_responses_session_bridge_instance_id=instance_id, - http_responses_session_bridge_instance_ring=list(instance_ring or []), - proxy_admission_wait_timeout_seconds=admission_wait_timeout_seconds, - proxy_request_budget_seconds=75.0, - compact_request_budget_seconds=75.0, - transcription_request_budget_seconds=120.0, - upstream_compact_timeout_seconds=None, - upstream_stream_transport="auto", - stream_idle_timeout_seconds=300.0, - openai_prompt_cache_key_derivation_enabled=True, - ) - - -def _make_dashboard_settings( - *, - prefer_earlier_reset_accounts: bool = False, - gateway_safe_mode: bool = False, - prompt_cache_idle_ttl_seconds: int | float = 3600, -) -> DashboardSettings: - return DashboardSettings( - id=1, - sticky_threads_enabled=False, - upstream_stream_transport="auto", - prefer_earlier_reset_accounts=prefer_earlier_reset_accounts, - routing_strategy="usage_weighted", - openai_cache_affinity_max_age_seconds=300, - import_without_overwrite=False, - totp_required_on_login=False, - api_key_auth_enabled=False, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=int(prompt_cache_idle_ttl_seconds), - http_responses_session_bridge_gateway_safe_mode=gateway_safe_mode, - sticky_reallocation_budget_threshold_pct=95.0, - ) - - -def _install_proxy_settings( - monkeypatch: pytest.MonkeyPatch, - *, - app_settings: Settings, - dashboard_settings: DashboardSettings, -) -> None: - monkeypatch.setattr(proxy_module, "get_settings_cache", lambda: _SettingsCache(dashboard_settings)) - monkeypatch.setattr(proxy_module, "get_settings", lambda: app_settings) - - -def _install_bridge_settings(monkeypatch: pytest.MonkeyPatch, *, enabled: bool) -> None: - _install_bridge_settings_with_limits(monkeypatch, enabled=enabled) - - -def _install_bridge_settings_with_limits( - monkeypatch: pytest.MonkeyPatch, - *, - enabled: bool, - max_sessions: int = 128, - queue_limit: int = 8, - admission_wait_timeout_seconds: float = 0.05, - codex_idle_ttl_seconds: float = 900.0, - prompt_cache_idle_ttl_seconds: float = 3600.0, - codex_prewarm_enabled: bool = False, - gateway_safe_mode: bool = False, - prefer_earlier_reset_accounts: bool = False, - instance_id: str = "instance-a", - instance_ring: list[str] | None = None, -) -> None: - _install_proxy_settings( - monkeypatch, - app_settings=_make_app_settings( - enabled=enabled, - max_sessions=max_sessions, - queue_limit=queue_limit, - admission_wait_timeout_seconds=admission_wait_timeout_seconds, - codex_idle_ttl_seconds=codex_idle_ttl_seconds, - codex_prewarm_enabled=codex_prewarm_enabled, - instance_id=instance_id, - instance_ring=instance_ring, - ), - dashboard_settings=_make_dashboard_settings( - prefer_earlier_reset_accounts=prefer_earlier_reset_accounts, - gateway_safe_mode=gateway_safe_mode, - prompt_cache_idle_ttl_seconds=prompt_cache_idle_ttl_seconds, - ), - ) - - -class _FakeUpstreamMessage: - def __init__( - self, - kind: str, - *, - text: str | None = None, - close_code: int | None = None, - error: str | None = None, - error_code: str | None = None, - ) -> None: - self.kind = kind - self.text = text - self.close_code = close_code - self.error = error - self.error_code = error_code - self.data = None - - -class _FakeBridgeUpstreamWebSocket: - def __init__( - self, - response_id_prefix: str = "resp_bridge", - *, - completed_output: list[dict[str, Any]] | None = None, - ) -> None: - self.sent_text: list[str] = [] - self.closed = False - self.response_id_prefix = response_id_prefix - self.completed_output = completed_output - self._messages: asyncio.Queue[_FakeUpstreamMessage] = asyncio.Queue() - - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - response_id = f"{self.response_id_prefix}_{len(self.sent_text)}" - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": {"id": response_id, "object": "response", "status": "in_progress"}, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": response_id, - "object": "response", - "status": "completed", - "output": self.completed_output - or [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ], - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - - async def send_bytes(self, data: bytes) -> None: - raise AssertionError(f"Unexpected binary frame: {data!r}") - - async def receive(self) -> _FakeUpstreamMessage: - return await self._messages.get() - - async def close(self) -> None: - self.closed = True - - def response_header(self, name: str) -> str | None: - del name - return None - - -class _InterruptedCustomToolUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - """First response completes with an unresolved direct tool call.""" - - def __init__( - self, - response_id_prefix: str = "resp_bridge", - *, - emit_added: bool = False, - tool_call_type: str = "custom_tool_call", - include_full_transition_output: bool = False, - empty_terminal_output: bool = False, - ) -> None: - super().__init__(response_id_prefix) - self._emit_added = emit_added - self._tool_call_type = tool_call_type - self._include_full_transition_output = include_full_transition_output - self._empty_terminal_output = empty_terminal_output - - def _pending_call_item(self, *, status: str) -> dict[str, Any]: - item: dict[str, Any] = { - "id": "ctc_shell", - "type": self._tool_call_type, - "status": status, - "call_id": "call_custom_shell", - "name": "shell", - } - if self._tool_call_type == "custom_tool_call": - item["input"] = "" if status == "in_progress" else "pwd" - elif self._tool_call_type == "function_call": - item["arguments"] = "" if status == "in_progress" else "{}" - else: # pragma: no cover - test helper is closed to direct call types - raise AssertionError(f"unsupported pending call type: {self._tool_call_type}") - return item - - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - response_id = f"resp_bridge_custom_{len(self.sent_text)}" - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": {"id": response_id, "object": "response", "status": "in_progress"}, - }, - separators=(",", ":"), - ), - ) - ) - if len(self.sent_text) == 1: - if self._emit_added: - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.output_item.added", - "response_id": response_id, - "item": self._pending_call_item(status="in_progress"), - "output_index": 0, - }, - separators=(",", ":"), - ), - ) - ) + if len(self.sent_text) == 3: await self._messages.put( _FakeUpstreamMessage( "text", text=json.dumps( { - "type": "response.output_item.done", - "response_id": response_id, - "item": self._pending_call_item(status="completed"), - "output_index": 0, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": response_id, - "object": "response", - "status": "completed", - "output": ( - [] - if self._empty_terminal_output and len(self.sent_text) == 1 - else [ - { - "type": "reasoning", - "id": "rs_transition", - "encrypted_content": "opaque", - "summary": [], - "status": "completed", - }, - { - "type": "message", - "id": "msg_transition", - "role": "assistant", - "phase": "commentary", - "content": [{"type": "output_text", "text": "Checking the task."}], - }, - self._pending_call_item(status="completed"), - ] - if self._include_full_transition_output and len(self.sent_text) == 1 - else [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ] - ), - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - - -class _ClosingInterruptedCustomToolUpstreamWebSocket(_InterruptedCustomToolUpstreamWebSocket): - def __init__( - self, - response_id_prefix: str = "resp_bridge", - *, - tool_call_type: str = "custom_tool_call", - include_full_transition_output: bool = False, - ) -> None: - super().__init__( - response_id_prefix, - emit_added=True, - tool_call_type=tool_call_type, - include_full_transition_output=include_full_transition_output, - ) - - async def send_text(self, text: str) -> None: - await super().send_text(text) - await self._messages.put(_FakeUpstreamMessage("close", close_code=1000)) - - -class _ClosingBridgeUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - await super().send_text(text) - await self._messages.put(_FakeUpstreamMessage("close", close_code=1000)) - - -class _PrecreatedCloseUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - await self._messages.put(_FakeUpstreamMessage("close", close_code=1011)) - - -class _AmbiguousAcceptedReceiveErrorUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - """Accept response.create, then lose receive provenance before any event.""" - - def __init__(self) -> None: - super().__init__() - self.irreversible_effect_count = 0 - - async def send_text(self, text: str) -> None: - self.irreversible_effect_count += 1 - self.sent_text.append(text) - await self._messages.put( - _FakeUpstreamMessage( - "error", - error="upstream receive failed after transport accepted the frame", - error_code=None, - ) - ) - - -class _PrecreatedOverloadUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.failed", - "response": { - "error": { - "code": "server_is_overloaded", - "message": "Our servers are currently overloaded. Please try again later.", - } - }, - }, - separators=(",", ":"), - ), - ) - ) - - -class _CreatedOnlyUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - response_id = f"resp_created_only_{len(self.sent_text)}" - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": {"id": response_id, "object": "response", "status": "in_progress"}, - }, - separators=(",", ":"), - ), - ) - ) - - -class _SilentUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - - -class _AccountScopedAnchorUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - """Upstream that only resolves ``previous_response_id`` values it issued. - - A ``previous_response_id`` is account-scoped upstream: only the account that - created the response can resume it. A ``response.create`` carrying a foreign - anchor is accepted by the socket but never answered with ``response.created``. - Modelling that here makes a cross-account anchor observable as the production - symptom instead of a silent assertion: the turn never settles and the - per-bridge ``response_create_gate`` stays held. - """ - - async def send_text(self, text: str) -> None: - anchor = json.loads(text).get("previous_response_id") - if isinstance(anchor, str) and not anchor.startswith(self.response_id_prefix): - self.sent_text.append(text) - return - await super().send_text(text) - - -class _RecordingUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - pass - - -class _CreatedThenCloseUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - response_id = f"resp_created_then_close_{len(self.sent_text)}" - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": {"id": response_id, "object": "response", "status": "in_progress"}, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put(_FakeUpstreamMessage("close", close_code=1011)) - - -class _ReasoningThenAbruptCloseUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - response_id = f"resp_reasoning_then_close_{len(self.sent_text)}" - reasoning_id = f"rs_reasoning_then_close_{len(self.sent_text)}" - events = [ - { - "type": "response.created", - "sequence_number": 0, - "response": {"id": response_id, "object": "response", "status": "in_progress"}, - }, - { - "type": "response.output_item.added", - "sequence_number": 1, - "response_id": response_id, - "output_index": 0, - "item": { - "id": reasoning_id, - "type": "reasoning", - "summary": [], - "encrypted_content": None, - }, - }, - { - "type": "response.reasoning_summary_part.added", - "sequence_number": 2, - "response_id": response_id, - "item_id": reasoning_id, - "output_index": 0, - "summary_index": 0, - "part": {"type": "summary_text", "text": ""}, - }, - { - "type": "response.reasoning_summary_text.delta", - "sequence_number": 3, - "response_id": response_id, - "item_id": reasoning_id, - "output_index": 0, - "summary_index": 0, - "delta": "Reviewing the final integration result.", - }, - ] - for event in events: - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps(event, separators=(",", ":")), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "error", - error="no close frame received or sent", - ) - ) - - -class _CompleteThenReasoningAbruptCloseUpstreamWebSocket(_ReasoningThenAbruptCloseUpstreamWebSocket): - async def send_text(self, text: str) -> None: - if not self.sent_text: - await _FakeBridgeUpstreamWebSocket.send_text(self, text) - return - await super().send_text(text) - - -class _ErrorOnlyUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "invalid_request_error", - "message": ( - "The 'gpt-5.3-codex-spark' model is not supported when using Codex " - "with a ChatGPT account." - ), - }, - }, - separators=(",", ":"), - ), - ) - ) - - -class _RateLimitErrorUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "error", - "status": 429, - "error": { - "type": "rate_limit_error", - "code": "rate_limit_exceeded", - "message": "Rate limit reached for gpt-4o on tokens per day", - "plan_type": "team", - "resets_at": 1700000000, - "resets_in_seconds": 3600, - }, - }, - separators=(",", ":"), - ), - ) - ) - - -class _PreviousResponseNotFoundUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - payload = json.loads(text) - previous_response_id = payload.get("previous_response_id") - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "previous_response_not_found", - "message": f"Previous response with id '{previous_response_id}' not found.", - "param": "previous_response_id", - }, - }, - separators=(",", ":"), - ), - ) - ) - - -class _AnonymousPreviousResponseNotFoundWithInflightUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - def __init__(self) -> None: - super().__init__() - self.first_request_created = asyncio.Event() - - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - if len(self.sent_text) == 1: - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": { - "id": "resp_bridge_inflight", - "object": "response", - "status": "in_progress", - }, - }, - separators=(",", ":"), - ), - ) - ) - self.first_request_created.set() - return - - payload = json.loads(text) - previous_response_id = payload.get("previous_response_id") - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "previous_response_not_found", - "message": f"Previous response with id '{previous_response_id}' not found.", - "param": "previous_response_id", - }, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": "resp_bridge_inflight", - "object": "response", - "status": "completed", - "output": [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ], - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - - -class _InvalidRequestPreviousResponseUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - payload = json.loads(text) - previous_response_id = payload.get("previous_response_id") - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "invalid_request_error", - "message": f"Previous response with id '{previous_response_id}' not found.", - "param": "previous_response_id", - }, - }, - separators=(",", ":"), - ), - ) - ) - - -class _RejectStalePreviousResponseUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - """Reject one stale anchor but accept a proved full unanchored resend.""" - - def __init__(self, stale_response_id: str, *, canonical_invalid_shape: bool = False) -> None: - super().__init__(response_id_prefix="resp_recovered") - self._stale_response_id = stale_response_id - self._canonical_invalid_shape = canonical_invalid_shape - - async def send_text(self, text: str) -> None: - payload = json.loads(text) - if payload.get("previous_response_id") != self._stale_response_id: - await super().send_text(text) - return - self.sent_text.append(text) - error: dict[str, object] = { - "type": "invalid_request_error", - "code": "invalid_request_error", - "message": f"Previous response with id '{self._stale_response_id}' not found.", - "param": "previous_response_id", - } - if self._canonical_invalid_shape: - error = { - "type": "invalid_request_error", - "code": "invalid_request_error", - "message": "Invalid `previous_response_id`.", - } - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "error", - "status": 400, - "error": error, - }, - separators=(",", ":"), - ), - ) - ) - - -class _CompleteThenRejectStalePreviousResponseUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - """Complete one turn, then reject its proxy-injected continuation anchor.""" - - async def send_text(self, text: str) -> None: - if not self.sent_text: - await super().send_text(text) - return - payload = json.loads(text) - stale_response_id = f"{self.response_id_prefix}_1" - assert payload.get("previous_response_id") == stale_response_id - self.sent_text.append(text) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "invalid_request_error", - "message": f"Previous response with id '{stale_response_id}' not found.", - "param": "previous_response_id", - }, - }, - separators=(",", ":"), - ), - ) - ) - - -class _ClosedBeforeSendUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - """Prove that no response.create bytes reached this physical socket.""" - - async def send_text(self, text: str) -> None: - del text - raise UpstreamWebSocketTransportError( - "upstream closed before response.create dispatch", - error_code=UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - ) - - -class _AcceptedThenCancelledSendUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - """Accept one irreversible send, then cancel before a response is observed.""" - - def __init__(self) -> None: - super().__init__() - self.irreversible_effect_count = 0 - - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - self.irreversible_effect_count += 1 - raise asyncio.CancelledError - - -class _ForeignPreviousResponseNotFoundAfterCreatedUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - if len(self.sent_text) == 1: - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": { - "id": "resp_bridge_prev_anchor", - "object": "response", - "status": "in_progress", - }, + "type": "response.created", + "response": { + "id": "resp_bridge_followup_same_anchor_b", + "object": "response", + "status": "in_progress", + }, }, separators=(",", ":"), ), @@ -3712,1911 +1501,400 @@ async def send_text(self, text: str) -> None: "text", text=json.dumps( { - "type": "response.completed", - "response": { - "id": "resp_bridge_prev_anchor", - "object": "response", - "status": "completed", - "output": [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ], - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - return - - if len(self.sent_text) == 2: - payload = json.loads(text) - previous_response_id = payload.get("previous_response_id") - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": { - "id": "resp_bridge_followup_created", - "object": "response", - "status": "in_progress", - }, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.failed", - "response": { - "id": "resp_bridge_foreign_prev_nf", - "object": "response", - "status": "failed", - "error": { - "type": "invalid_request_error", - "code": "previous_response_not_found", - "message": f"Previous response with id '{previous_response_id}' not found.", - "param": "previous_response_id", - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - return - - response_id = "resp_bridge_after_error" - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": {"id": response_id, "object": "response", "status": "in_progress"}, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": response_id, - "object": "response", - "status": "completed", - "output": [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ], - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - - -class _AnonymousPreviousResponseNotFoundAfterCreatedUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - def __init__(self) -> None: - super().__init__() - self.first_request_created = asyncio.Event() - - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - if len(self.sent_text) == 1: - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": { - "id": "resp_bridge_inflight", - "object": "response", - "status": "in_progress", - }, - }, - separators=(",", ":"), - ), - ) - ) - self.first_request_created.set() - return - - if len(self.sent_text) == 2: - payload = json.loads(text) - previous_response_id = payload.get("previous_response_id") - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": { - "id": "resp_bridge_followup_created", - "object": "response", - "status": "in_progress", - }, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "previous_response_not_found", - "message": f"Previous response with id '{previous_response_id}' not found.", - "param": "previous_response_id", - }, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": "resp_bridge_inflight", - "object": "response", - "status": "completed", - "output": [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ], - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - return - - response_id = "resp_bridge_after_error" - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": {"id": response_id, "object": "response", "status": "in_progress"}, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": response_id, - "object": "response", - "status": "completed", - "output": [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ], - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - - -class _TwoFollowupsPreviousResponseNotFoundUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - def __init__(self) -> None: - super().__init__() - self.first_followup_created = asyncio.Event() - - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - if len(self.sent_text) == 1: - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": { - "id": "resp_bridge_prev_anchor_a", - "object": "response", - "status": "in_progress", - }, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": "resp_bridge_prev_anchor_a", - "object": "response", - "status": "completed", - "output": [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ], - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - return - - if len(self.sent_text) == 2: - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": { - "id": "resp_bridge_prev_anchor_b", - "object": "response", - "status": "in_progress", - }, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": "resp_bridge_prev_anchor_b", - "object": "response", - "status": "completed", - "output": [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ], - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - return - - if len(self.sent_text) == 3: - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": { - "id": "resp_bridge_followup_a", - "object": "response", - "status": "in_progress", - }, - }, - separators=(",", ":"), - ), - ) - ) - self.first_followup_created.set() - return - - if len(self.sent_text) == 4: - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": { - "id": "resp_bridge_followup_b", - "object": "response", - "status": "in_progress", - }, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "previous_response_not_found", - "message": ( - "Cannot continue conversation because upstream lost resp_bridge_prev_anchor_a." - ), - "param": "previous_response_id", - }, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": "resp_bridge_followup_b", - "object": "response", - "status": "completed", - "output": [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ], - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - return - - response_id = "resp_bridge_after_error" - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": {"id": response_id, "object": "response", "status": "in_progress"}, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": response_id, - "object": "response", - "status": "completed", - "output": [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ], - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - - -class _TwoSameAnchorFollowupsPreviousResponseNotFoundUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - def __init__(self) -> None: - super().__init__() - self.first_followup_created = asyncio.Event() - - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - if len(self.sent_text) == 1: - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": { - "id": "resp_bridge_prev_anchor_shared", - "object": "response", - "status": "in_progress", - }, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": "resp_bridge_prev_anchor_shared", - "object": "response", - "status": "completed", - "output": [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ], - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - return - - if len(self.sent_text) == 2: - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": { - "id": "resp_bridge_followup_same_anchor_a", - "object": "response", - "status": "in_progress", - }, - }, - separators=(",", ":"), - ), - ) - ) - self.first_followup_created.set() - return - - if len(self.sent_text) == 3: - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": { - "id": "resp_bridge_followup_same_anchor_b", - "object": "response", - "status": "in_progress", - }, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "previous_response_not_found", - "message": "Previous response with id 'resp_bridge_prev_anchor_shared' not found.", - "param": "previous_response_id", - }, - }, - separators=(",", ":"), - ), - ) - ) - return - - response_id = "resp_bridge_after_same_anchor_error" - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": {"id": response_id, "object": "response", "status": "in_progress"}, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": response_id, - "object": "response", - "status": "completed", - "output": [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ], - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - - -class _FailingSendThenCloseUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - await self._messages.put(_FakeUpstreamMessage("close", close_code=1011)) - raise RuntimeError("socket closed during send") - - -def _make_dummy_bridge_session(session_key: proxy_module._HTTPBridgeSessionKey) -> proxy_module._HTTPBridgeSession: - async def _close() -> None: - return None - - return proxy_module._HTTPBridgeSession( - key=session_key, - headers={}, - affinity=proxy_module._AffinityPolicy(), - request_model="gpt-5.4", - account=cast(Account, SimpleNamespace(id=None, status=AccountStatus.ACTIVE, plan_type="plus")), - upstream=cast(proxy_module.UpstreamWebSocket, SimpleNamespace(close=_close)), - upstream_control=proxy_module._WebSocketUpstreamControl(), - pending_lock=anyio.Lock(), - pending_requests=deque(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=time.monotonic(), - idle_ttl_seconds=120.0, - ) - - -class _PrewarmingBridgeUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - payload = json.loads(text) - response_id = f"resp_prewarm_{len(self.sent_text)}" - output = [] - usage = { - "input_tokens": 12, - "output_tokens": 0, - "total_tokens": 12, - "input_tokens_details": {"cached_tokens": 0}, - "output_tokens_details": {"reasoning_tokens": 0}, - } - if payload.get("generate") is not False: - response_id = f"resp_actual_{len(self.sent_text)}" - output = [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "OK"}], - } - ] - usage = { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - } - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": {"id": response_id, "object": "response", "status": "in_progress"}, - }, - separators=(",", ":"), - ), - ) - ) - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": response_id, - "object": "response", - "status": "completed", - "output": output, - "usage": usage, - }, - }, - separators=(",", ":"), - ), - ) - ) - - -class _TurnStateBridgeUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - def __init__(self, turn_state: str) -> None: - super().__init__() - self._turn_state = turn_state - - def response_header(self, name: str) -> str | None: - if name.lower() == "x-codex-turn-state": - return self._turn_state - return None - - -def _make_api_key_data( - *, - key_id: str, - assigned_account_ids: list[str], - account_assignment_scope_enabled: bool | None = None, -) -> proxy_module.ApiKeyData: - return proxy_module.ApiKeyData( - id=key_id, - name="bridge-key", - key_prefix="sk-bridge", - allowed_models=None, - enforced_model=None, - enforced_reasoning_effort=None, - enforced_service_tier=None, - expires_at=None, - is_active=True, - created_at=datetime(2025, 1, 1, tzinfo=timezone.utc), - last_used_at=None, - account_assignment_scope_enabled=( - bool(assigned_account_ids) if account_assignment_scope_enabled is None else account_assignment_scope_enabled - ), - assigned_account_ids=assigned_account_ids, - ) - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_fails_over_account_routed_proxy_before_dispatch( - async_client, - monkeypatch, -): - _install_bridge_settings(monkeypatch, enabled=True) - first_account_id = await _import_account( - async_client, - "acc_http_bridge_proxy_connect_a", - "http-bridge-proxy-connect-a@example.com", - ) - second_account_id = await _import_account( - async_client, - "acc_http_bridge_proxy_connect_b", - "http-bridge-proxy-connect-b@example.com", - ) - first_account = await _get_account(first_account_id) - second_account = await _get_account(second_account_id) - upstream = _FakeBridgeUpstreamWebSocket() - connect_calls: list[str | None] = [] - selection_exclusions: list[set[str]] = [] - backed_off_accounts: list[str] = [] - handle_stream_error = AsyncMock() - - async def fake_select_account_with_budget(self, deadline, *, exclude_account_ids=None, **kwargs): - del self, deadline, kwargs - excluded = set(exclude_account_ids or set()) - selection_exclusions.append(excluded) - account = second_account if first_account.id in excluded else first_account - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - **kwargs, - ): - del headers, access_token, kwargs - connect_calls.append(account_id_header) - if len(connect_calls) == 1: - raise proxy_module.ProxyResponseError( - 502, - proxy_module.openai_error("upstream_unavailable", "sanitized bridge proxy failure"), - failure_phase="connect", - retryable_same_contract=True, - failure_detail="proxy_connect_pre_dispatch", - failure_exception_type="ClientProxyConnectionError", - ) - return upstream - - async def fake_record_error_backoff(self, account): - del self - backed_off_accounts.append(account.id) - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_handle_stream_error", handle_stream_error) - monkeypatch.setattr(proxy_module.LoadBalancer, "record_error_backoff", fake_record_error_backoff) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - events = await _collect_sse_events( - async_client, - "/v1/responses", - json_body={ - "model": "gpt-5.4", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-proxy-connect-failover-key", - "stream": True, - }, - ) - - _assert_created_text_delta_completed(events) - assert len(connect_calls) == 2 - assert selection_exclusions == [set(), {first_account.id}] - assert backed_off_accounts == [first_account.id] - assert len(upstream.sent_text) == 1 - handle_stream_error.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_shared_proxy_exhaustion_does_not_penalize_account( - async_client, - monkeypatch, -): - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_shared_proxy", - "http-bridge-shared-proxy@example.com", - ) - account = await _get_account(account_id) - selection_exclusions: list[set[str]] = [] - connect_calls: list[str | None] = [] - backed_off_accounts: list[str] = [] - - async def fake_select_account_with_budget(self, deadline, *, exclude_account_ids=None, **kwargs): - del self, deadline, kwargs - selection_exclusions.append(set(exclude_account_ids or set())) - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket(headers, access_token, account_id_header, **kwargs): - del headers, access_token, kwargs - connect_calls.append(account_id_header) - raise proxy_module.ProxyResponseError( - 502, - proxy_module.openai_error("upstream_unavailable", "shared proxy setup failed"), - failure_phase="connect", - retryable_same_contract=False, - failure_detail="shared_proxy_connect_pre_dispatch_exhausted", - failure_exception_type="ConnectionResetError", - ) - - async def fake_record_error_backoff(self, selected_account): - del self - backed_off_accounts.append(selected_account.id) - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module.LoadBalancer, "record_error_backoff", fake_record_error_backoff) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - response = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.4", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-shared-proxy-exhaustion-key", - "stream": True, - }, - ) - - assert response.status_code == 502 - assert response.json()["error"]["code"] == "upstream_unavailable" - assert len(connect_calls) == 1 - assert selection_exclusions == [set()] - assert backed_off_accounts == [] - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_codex_session_uses_extended_idle_ttl(async_client, app_instance, monkeypatch): - _install_bridge_settings_with_limits(monkeypatch, enabled=True, codex_idle_ttl_seconds=600.0) - account_id = await _import_account(async_client, "acc_http_bridge_codex_ttl", "http-bridge-codex-ttl@example.com") - account = await _get_account(account_id) - service = get_proxy_service_for_app(app_instance) - fake_upstream = _FakeBridgeUpstreamWebSocket() - - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - ) - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - return fake_upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - payload = proxy_module.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {"x-codex-turn-state": "turn_state_1"}, - codex_session_affinity=True, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={"x-codex-turn-state": "turn_state_1"}, - affinity=affinity, - api_key=None, - request_id="req_1", - ) - - session = await service._get_or_create_http_bridge_session( - key, - headers={"x-codex-turn-state": "turn_state_1"}, - affinity=affinity, - api_key=None, - request_model=payload.model, - idle_ttl_seconds=proxy_module._effective_http_bridge_idle_ttl_seconds( - affinity=affinity, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=600.0, - ), - max_sessions=8, - ) - - session.last_used_at = time.monotonic() - 300.0 - async with service._http_bridge_lock: - stale_sessions = service._prune_http_bridge_sessions_locked() - assert key in service._http_bridge_sessions - assert stale_sessions == [] - - session.last_used_at = time.monotonic() - 601.0 - async with service._http_bridge_lock: - stale_sessions = service._prune_http_bridge_sessions_locked() - assert key not in service._http_bridge_sessions - for stale_session in stale_sessions: - await service._close_http_bridge_session(stale_session) - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_creation_honors_prefer_earlier_reset(async_client, app_instance, monkeypatch): - _install_bridge_settings_with_limits(monkeypatch, enabled=True, prefer_earlier_reset_accounts=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_prefer_earlier_reset", - "http-bridge-prefer-earlier-reset@example.com", - ) - account = await _get_account(account_id) - service = get_proxy_service_for_app(app_instance) - fake_upstream = _FakeBridgeUpstreamWebSocket() - select_calls: list[tuple[bool, str | None]] = [] - - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - service_tier=None, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - ) - select_calls.append((prefer_earlier_reset_accounts, service_tier)) - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_open_upstream_websocket_with_budget(self, target, headers, *, timeout_seconds): - del self, target, headers, timeout_seconds - return fake_upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr( - proxy_module.ProxyService, - "_open_upstream_websocket_with_budget", - fake_open_upstream_websocket_with_budget, - ) - - payload = proxy_module.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "", - "input": "hello", - "prompt_cache_key": "bridge_prefer_earlier_reset", - "service_tier": "priority", - } - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={}, - affinity=affinity, - api_key=None, - request_id="req_bridge_prefer_earlier_reset", - ) - - session = await service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=affinity, - api_key=None, - request_model=payload.model, - request_service_tier=payload.service_tier, - idle_ttl_seconds=120.0, - max_sessions=8, - gateway_safe_mode=True, - ) - - assert select_calls == [(True, "priority")] - await service._close_http_bridge_session(session) - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_codex_session_prewarms_first_request(async_client, monkeypatch): - _install_bridge_settings_with_limits( - monkeypatch, - enabled=True, - codex_idle_ttl_seconds=600.0, - codex_prewarm_enabled=True, - ) - account_id = await _import_account(async_client, "acc_http_bridge_prewarm", "http-bridge-prewarm@example.com") - account = await _get_account(account_id) - fake_upstream = _PrewarmingBridgeUpstreamWebSocket() - - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - ) - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - return fake_upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - response = await async_client.post( - "/v1/responses", - headers={"x-codex-turn-state": "turn_state_prewarm"}, - json={ - "model": "gpt-5.4", - "instructions": "hi", - "input": [{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}], - }, - ) - - assert response.status_code == 200 - assert response.json()["id"] == "resp_actual_2" - assert len(fake_upstream.sent_text) == 2 - assert json.loads(fake_upstream.sent_text[0])["generate"] is False - assert "generate" not in json.loads(fake_upstream.sent_text[1]) - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_codex_session_does_not_prewarm_by_default(async_client, monkeypatch): - _install_bridge_settings_with_limits(monkeypatch, enabled=True, codex_idle_ttl_seconds=600.0) - account_id = await _import_account(async_client, "acc_http_bridge_no_prewarm", "http-bridge-no-prewarm@example.com") - account = await _get_account(account_id) - fake_upstream = _PrewarmingBridgeUpstreamWebSocket() - - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - ) - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - return fake_upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - response = await async_client.post( - "/v1/responses", - headers={"x-codex-turn-state": "turn_state_no_prewarm"}, - json={ - "model": "gpt-5.4", - "instructions": "hi", - "input": [{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}], - }, - ) - - assert response.status_code == 200 - assert response.json()["id"] == "resp_actual_1" - assert len(fake_upstream.sent_text) == 1 - assert "generate" not in json.loads(fake_upstream.sent_text[0]) - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_non_owner_instance_falls_back_to_local_session( - async_client, - app_instance, - monkeypatch, -): - _install_bridge_settings_with_limits( - monkeypatch, - enabled=True, - gateway_safe_mode=True, - instance_id="instance-b", - instance_ring=["instance-a", "instance-b"], - ) - account_id = await _import_account(async_client, "acc_http_bridge_owner", "http-bridge-owner@example.com") - account = await _get_account(account_id) - service = get_proxy_service_for_app(app_instance) - service._ring_membership = cast( - proxy_module.RingMembershipService, - SimpleNamespace(list_active=AsyncMock(return_value=["instance-a", "instance-b"])), - ) - - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - ) - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - return _FakeBridgeUpstreamWebSocket() - - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - candidate_suffix = 0 - while True: - payload = proxy_module.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "hi", - "input": [{"role": "user", "content": "hi"}], - "prompt_cache_key": f"owner-check-{candidate_suffix}", - } - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={}, - affinity=affinity, - api_key=None, - request_id="req_owner", - ) - owner = await proxy_module._http_bridge_owner_instance(key, proxy_module.get_settings()) - if owner != "instance-b": - break - candidate_suffix += 1 - - session = await service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=affinity, - api_key=None, - request_model=payload.model, - idle_ttl_seconds=120.0, - max_sessions=8, - gateway_safe_mode=True, - ) - - assert session is not None - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_non_owner_prompt_cache_rebinds_locally_when_gateway_safe_mode_disabled( - async_client, - app_instance, - monkeypatch, -): - _install_bridge_settings_with_limits( - monkeypatch, - enabled=True, - gateway_safe_mode=False, - instance_id="instance-b", - instance_ring=["instance-a", "instance-b"], - ) - account_id = await _import_account( - async_client, - "acc_http_bridge_owner_strict", - "http-bridge-owner-strict@example.com", - ) - account = await _get_account(account_id) - service = get_proxy_service_for_app(app_instance) - service._ring_membership = cast( - proxy_module.RingMembershipService, - SimpleNamespace(list_active=AsyncMock(return_value=["instance-a", "instance-b"])), - ) - - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - api_key=None, - preferred_account_id=None, - exclude_account_ids=None, - additional_limit_name=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - api_key, - exclude_account_ids, - additional_limit_name, - ) - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr( - proxy_module, - "connect_responses_websocket", - AsyncMock(return_value=_FakeBridgeUpstreamWebSocket()), - ) - - candidate_suffix = 0 - while True: - payload = proxy_module.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "hi", - "input": [{"role": "user", "content": "hi"}], - "prompt_cache_key": f"owner-check-strict-{candidate_suffix}", - } - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={}, - affinity=affinity, - api_key=None, - request_id="req_owner_strict", - ) - owner = await proxy_module._http_bridge_owner_instance(key, proxy_module.get_settings()) - if owner != "instance-b": - break - candidate_suffix += 1 - - session = await service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=affinity, - api_key=None, - request_model=payload.model, - idle_ttl_seconds=120.0, - max_sessions=8, - gateway_safe_mode=False, - ) - - assert session.account.id == account.id - assert session.key == key - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_missing_turn_state_alias_with_previous_response_id_fails_closed( - app_instance, - monkeypatch, -): - _install_bridge_settings_with_limits(monkeypatch, enabled=True) - service = get_proxy_service_for_app(app_instance) - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() - - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._get_or_create_http_bridge_session( - proxy_module._HTTPBridgeSessionKey("turn_state_header", "http_turn_missing_alias", None), - headers={"x-codex-turn-state": "http_turn_missing_alias"}, - affinity=proxy_module._AffinityPolicy( - key="http_turn_missing_alias", - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ), - api_key=None, - request_model="gpt-5.1", - idle_ttl_seconds=120.0, - max_sessions=128, - previous_response_id="resp_missing_alias", - ) - - exc = exc_info.value - assert exc.status_code == 502 - assert exc.payload["error"] == { - "message": "Upstream websocket closed before response.completed", - "type": "server_error", - "code": "stream_incomplete", - } - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_stale_previous_response_alias_same_model_fails_closed( - app_instance, - monkeypatch, -): - _install_bridge_settings_with_limits(monkeypatch, enabled=True) - service = get_proxy_service_for_app(app_instance) - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() - service._http_bridge_previous_response_index.clear() - - previous_response_id = "resp_stale_same_model_alias" - stale_key = proxy_module._HTTPBridgeSessionKey("prompt_cache", "bridge-stale-prev-owner", None) - stale_session = _make_dummy_bridge_session(stale_key) - stale_session.request_model = "gpt-5.1" - stale_session.account = cast(Account, SimpleNamespace(id="acc-stale-prev-owner", status=AccountStatus.PAUSED)) - stale_session.previous_response_ids.add(previous_response_id) - service._http_bridge_sessions[stale_key] = stale_session - service._http_bridge_previous_response_index[(previous_response_id, None)] = stale_key - - async def fail_create_http_bridge_session(self, *args, **kwargs): - del self, args, kwargs - raise AssertionError("stale same-model previous_response_id must fail closed before replacement creation") - - monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fail_create_http_bridge_session) + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": "Previous response with id 'resp_bridge_prev_anchor_shared' not found.", + "param": "previous_response_id", + }, + }, + separators=(",", ":"), + ), + ) + ) + return - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._get_or_create_http_bridge_session( - proxy_module._HTTPBridgeSessionKey("request", "bridge-stale-prev-request", None), - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.1", - idle_ttl_seconds=120.0, - max_sessions=128, - previous_response_id=previous_response_id, + response_id = "resp_bridge_after_same_anchor_error" + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": {"id": response_id, "object": "response", "status": "in_progress"}, + }, + separators=(",", ":"), + ), + ) + ) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": response_id, + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) ) - exc = exc_info.value - assert exc.status_code == 502 - assert exc.payload["error"] == { - "message": "Upstream websocket closed before response.completed", - "type": "server_error", - "code": "stream_incomplete", - } - assert service._http_bridge_previous_response_index.get((previous_response_id, None)) is None - assert service._http_bridge_sessions[stale_key] is stale_session +class _FailingSendThenCloseUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + await self._messages.put(_FakeUpstreamMessage("close", close_code=1011)) + raise RuntimeError("socket closed during send") -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_previous_response_alias_rejects_service_tier_provenance_mismatch( - async_client, - app_instance, - monkeypatch, -): - _install_bridge_settings_with_limits(monkeypatch, enabled=True) - service = get_proxy_service_for_app(app_instance) - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() - service._http_bridge_previous_response_index.clear() - account_id = await _import_account( - async_client, - "acc_previous_response_tier_owner", - "previous-response-tier-owner@example.com", - plan_type="pro", - ) - account = await _get_account(account_id) - previous_response_id = "resp_previous_response_tier_owner" - owner_key = proxy_module._HTTPBridgeSessionKey("prompt_cache", "bridge-old-prompt", None) - owner_session = _make_dummy_bridge_session(owner_key) - owner_session.request_model = "gpt-5.3-codex-spark" - owner_session.request_service_tier = None - owner_session.account = account - owner_session.unanchored_reservation_id = None - owner_upstream = _FakeBridgeUpstreamWebSocket() - owner_session.upstream = cast(proxy_module.UpstreamWebSocket, owner_upstream) - owner_session.catalog_omission_quota_admission = CatalogOmissionQuotaAdmission( - normalized_model="gpt-5.3-codex-spark", - canonical_quota_key="codex_spark", - normalized_effective_service_tier=None, +def _make_dummy_bridge_session(session_key: proxy_module._HTTPBridgeSessionKey) -> proxy_module._HTTPBridgeSession: + async def _close() -> None: + return None + + return proxy_module._HTTPBridgeSession( + key=session_key, + headers={}, + affinity=proxy_module._AffinityPolicy(), + request_model="gpt-5.4", + account=cast(Account, SimpleNamespace(id=None, status=AccountStatus.ACTIVE, plan_type="plus")), + upstream=cast(proxy_module.UpstreamWebSocket, SimpleNamespace(close=_close)), + upstream_control=proxy_module._WebSocketUpstreamControl(), + pending_lock=anyio.Lock(), + pending_requests=deque(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=time.monotonic(), + idle_ttl_seconds=120.0, ) - owner_session.previous_response_ids.add(previous_response_id) - service._http_bridge_sessions[owner_key] = owner_session - service._http_bridge_previous_response_index[(previous_response_id, None)] = owner_key - turn_state = "http_turn_previous_response_tier_owner" - turn_state_alias_key = proxy_module._http_bridge_turn_state_alias_key(turn_state, None) - owner_session.downstream_turn_state_aliases.add(turn_state) - service._http_bridge_turn_state_index[turn_state_alias_key] = owner_key - previous_response_index_before = dict(service._http_bridge_previous_response_index) - turn_state_index_before = dict(service._http_bridge_turn_state_index) - class Registry: - def account_ids_for_model(self, model: str) -> set[str]: - assert model == "gpt-5.3-codex-spark" - return set() - def plan_types_for_model(self, model: str) -> set[str]: - assert model == "gpt-5.3-codex-spark" - return {"pro"} +class _PrewarmingBridgeUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + payload = json.loads(text) + response_id = f"resp_prewarm_{len(self.sent_text)}" + output = [] + usage = { + "input_tokens": 12, + "output_tokens": 0, + "total_tokens": 12, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + } + if payload.get("generate") is not False: + response_id = f"resp_actual_{len(self.sent_text)}" + output = [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ] + usage = { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + } + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": {"id": response_id, "object": "response", "status": "in_progress"}, + }, + separators=(",", ":"), + ), + ) + ) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": response_id, + "object": "response", + "status": "completed", + "output": output, + "usage": usage, + }, + }, + separators=(",", ":"), + ), + ) + ) - def account_ids_for_model_service_tier(self, model: str, service_tier: str) -> set[str]: - assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") - return set() - def plan_types_for_model_service_tier(self, model: str, service_tier: str) -> set[str]: - assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") - return {"pro"} +class _TurnStateBridgeUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + def __init__(self, turn_state: str) -> None: + super().__init__() + self._turn_state = turn_state - def get_snapshot(self): - return SimpleNamespace(account_plans={account_id: "pro"}) + def response_header(self, name: str) -> str | None: + if name.lower() == "x-codex-turn-state": + return self._turn_state + return None - monkeypatch.setattr(proxy_support, "get_model_registry", lambda: Registry()) - fallback_session_id = "priority-compatible-session" - fallback_key = proxy_module._HTTPBridgeSessionKey("session_header", fallback_session_id, None) - fallback_upstream = _FakeBridgeUpstreamWebSocket() - fallback_session = proxy_module._HTTPBridgeSession( - key=fallback_key, - headers={"x-codex-session-id": fallback_session_id}, - affinity=proxy_module._AffinityPolicy( - key=fallback_session_id, - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ), - request_model="gpt-5.3-codex-spark", - account=account, - upstream=cast(proxy_module.UpstreamWebSocket, fallback_upstream), - upstream_control=proxy_module._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=time.monotonic(), - idle_ttl_seconds=120.0, - request_service_tier="priority", - catalog_omission_quota_admission=CatalogOmissionQuotaAdmission( - normalized_model="gpt-5.3-codex-spark", - canonical_quota_key="codex_spark", - normalized_effective_service_tier="priority", +def _make_api_key_data( + *, + key_id: str, + assigned_account_ids: list[str], + account_assignment_scope_enabled: bool | None = None, +) -> proxy_module.ApiKeyData: + return proxy_module.ApiKeyData( + id=key_id, + name="bridge-key", + key_prefix="sk-bridge", + allowed_models=None, + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=datetime(2025, 1, 1, tzinfo=timezone.utc), + last_used_at=None, + account_assignment_scope_enabled=( + bool(assigned_account_ids) if account_assignment_scope_enabled is None else account_assignment_scope_enabled ), + assigned_account_ids=assigned_account_ids, ) - fallback_session.upstream_reader = asyncio.create_task( - service._relay_http_bridge_upstream_messages(fallback_session) - ) - service._http_bridge_sessions[fallback_key] = fallback_session - create_http_bridge_session = AsyncMock( - side_effect=AssertionError("anchored mismatch must fail before bridge creation") - ) - monkeypatch.setattr(service, "_create_http_bridge_session", create_http_bridge_session) - scheduled_sessions: list[proxy_module._HTTPBridgeSession] = [] - schedule_session_closes = service._schedule_http_bridge_session_closes - def capture_scheduled_sessions(sessions, *, reason): - scheduled_sessions.extend(sessions) - schedule_session_closes(sessions, reason=reason) - monkeypatch.setattr(service, "_schedule_http_bridge_session_closes", capture_scheduled_sessions) - owner_state_before = ( - owner_session.request_model, - owner_session.request_service_tier, - owner_session.catalog_omission_quota_admission, - owner_session.upstream, - set(owner_session.previous_response_ids), +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_fails_over_confirmed_proxy_connect_before_dispatch( + async_client, + monkeypatch, +): + _install_bridge_settings(monkeypatch, enabled=True) + first_account_id = await _import_account( + async_client, + "acc_http_bridge_proxy_connect_a", + "http-bridge-proxy-connect-a@example.com", ) - fallback_state_before = ( - fallback_session.request_model, - fallback_session.request_service_tier, - fallback_session.catalog_omission_quota_admission, - fallback_session.upstream, - set(fallback_session.previous_response_ids), + second_account_id = await _import_account( + async_client, + "acc_http_bridge_proxy_connect_b", + "http-bridge-proxy-connect-b@example.com", ) - owner_request_count = len(owner_upstream.sent_text) - fallback_request_count = len(fallback_upstream.sent_text) + first_account = await _get_account(first_account_id) + second_account = await _get_account(second_account_id) + upstream = _FakeBridgeUpstreamWebSocket() + connect_calls: list[str | None] = [] + selection_exclusions: list[set[str]] = [] + backed_off_accounts: list[str] = [] + handle_stream_error = AsyncMock() - rejected = await async_client.post( + async def fake_select_account_with_budget(self, deadline, *, exclude_account_ids=None, **kwargs): + del self, deadline, kwargs + excluded = set(exclude_account_ids or set()) + selection_exclusions.append(excluded) + account = second_account if first_account.id in excluded else first_account + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + **kwargs, + ): + del headers, access_token, kwargs + connect_calls.append(account_id_header) + if len(connect_calls) == 1: + raise proxy_module.ProxyResponseError( + 502, + proxy_module.openai_error("upstream_unavailable", "sanitized bridge proxy failure"), + failure_phase="connect", + retryable_same_contract=True, + failure_detail="proxy_connect_pre_dispatch", + failure_exception_type="ClientProxyConnectionError", + ) + return upstream + + async def fake_record_error_backoff(self, account): + del self + backed_off_accounts.append(account.id) + + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_handle_stream_error", handle_stream_error) + monkeypatch.setattr(proxy_module.LoadBalancer, "record_error_backoff", fake_record_error_backoff) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + events = await _collect_sse_events( + async_client, "/v1/responses", - headers={ - "x-codex-turn-state": "http_turn_previous_response_tier_fallback", - "x-codex-session-id": fallback_session_id, - }, - json={ - "model": "gpt-5.3-codex-spark", + json_body={ + "model": "gpt-5.4", "instructions": "Return exactly OK.", - "input": "must fail before session fallback", - "previous_response_id": previous_response_id, - "service_tier": "priority", + "input": "hello", + "prompt_cache_key": "http-bridge-proxy-connect-failover-key", + "stream": True, }, ) - assert len(fallback_upstream.sent_text) == fallback_request_count - assert rejected.status_code == 502, rejected.text - assert rejected.json()["error"] == { - "message": "Upstream websocket closed before response.completed", - "type": "server_error", - "code": "stream_incomplete", - } - create_http_bridge_session.assert_not_awaited() - assert len(owner_upstream.sent_text) == owner_request_count - assert owner_session.closed is False - assert fallback_session.closed is False - assert service._http_bridge_sessions[owner_key] is owner_session - assert service._http_bridge_sessions[fallback_key] is fallback_session - assert owner_session not in scheduled_sessions - assert fallback_session not in scheduled_sessions - assert service._http_bridge_previous_response_index == previous_response_index_before - assert service._http_bridge_turn_state_index == turn_state_index_before - assert ( - owner_session.request_model, - owner_session.request_service_tier, - owner_session.catalog_omission_quota_admission, - owner_session.upstream, - set(owner_session.previous_response_ids), - ) == owner_state_before - assert ( - fallback_session.request_model, - fallback_session.request_service_tier, - fallback_session.catalog_omission_quota_admission, - fallback_session.upstream, - set(fallback_session.previous_response_ids), - ) == fallback_state_before + _assert_created_text_delta_completed(events) + assert len(connect_calls) == 2 + assert selection_exclusions == [set(), {first_account.id}] + assert backed_off_accounts == [first_account.id] + assert len(upstream.sent_text) == 1 + handle_stream_error.assert_not_awaited() - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._get_or_create_http_bridge_session( - proxy_module._HTTPBridgeSessionKey("prompt_cache", "bridge-new-prompt", None), - headers={}, - affinity=proxy_module._AffinityPolicy( - key="bridge-new-prompt", - kind=proxy_module.StickySessionKind.PROMPT_CACHE, - ), - api_key=None, - request_model="gpt-5.3-codex-spark", - request_service_tier="priority", - idle_ttl_seconds=120.0, - max_sessions=128, - previous_response_id=previous_response_id, + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_codex_session_uses_extended_idle_ttl(async_client, app_instance, monkeypatch): + _install_bridge_settings_with_limits(monkeypatch, enabled=True, codex_idle_ttl_seconds=600.0) + account_id = await _import_account(async_client, "acc_http_bridge_codex_ttl", "http-bridge-codex-ttl@example.com") + account = await _get_account(account_id) + service = get_proxy_service_for_app(app_instance) + fake_upstream = _FakeBridgeUpstreamWebSocket() + + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, ) + return AccountSelection(account=account, error_message=None, error_code=None) - assert exc_info.value.status_code == 502 - assert owner_session.request_service_tier is None - assert service._http_bridge_sessions[owner_key] is owner_session + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target - assert service._http_bridge_previous_response_index == previous_response_index_before - assert service._http_bridge_turn_state_index == turn_state_index_before + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + return fake_upstream - with pytest.raises(proxy_module.ProxyResponseError) as turn_state_exc_info: - await service._get_or_create_http_bridge_session( - proxy_module._HTTPBridgeSessionKey("request", "bridge-turn-state-tier-mismatch", None), - headers={"x-codex-turn-state": turn_state}, - affinity=proxy_module._AffinityPolicy( - key=turn_state, - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ), - api_key=None, - request_model="gpt-5.3-codex-spark", - request_service_tier="priority", - idle_ttl_seconds=120.0, - max_sessions=128, - ) + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - assert turn_state_exc_info.value.status_code == 502 - assert owner_session.request_model == "gpt-5.3-codex-spark" - assert owner_session.request_service_tier is None - assert service._http_bridge_sessions[owner_key] is owner_session - assert service._http_bridge_previous_response_index == previous_response_index_before - assert service._http_bridge_turn_state_index == turn_state_index_before + payload = proxy_module.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} + ) + affinity = proxy_module._sticky_key_for_responses_request( + payload, + {"x-codex-turn-state": "turn_state_1"}, + codex_session_affinity=True, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + key = proxy_module._make_http_bridge_session_key( + payload, + headers={"x-codex-turn-state": "turn_state_1"}, + affinity=affinity, + api_key=None, + request_id="req_1", + ) - reused = await service._get_or_create_http_bridge_session( - proxy_module._HTTPBridgeSessionKey("prompt_cache", "bridge-correct-prompt", None), - headers={}, - affinity=proxy_module._AffinityPolicy( - key="bridge-correct-prompt", - kind=proxy_module.StickySessionKind.PROMPT_CACHE, - ), + session = await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-turn-state": "turn_state_1"}, + affinity=affinity, api_key=None, - request_model="gpt-5.3-codex-spark", - request_service_tier=None, - idle_ttl_seconds=120.0, - max_sessions=128, - previous_response_id=previous_response_id, + request_model=payload.model, + idle_ttl_seconds=proxy_module._effective_http_bridge_idle_ttl_seconds( + affinity=affinity, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=600.0, + ), + max_sessions=8, ) - assert reused is owner_session - assert service._http_bridge_previous_response_index == previous_response_index_before - assert service._http_bridge_turn_state_index == turn_state_index_before + session.last_used_at = time.monotonic() - 300.0 + async with service._http_bridge_lock: + stale_sessions = service._prune_http_bridge_sessions_locked() + assert key in service._http_bridge_sessions + assert stale_sessions == [] + + session.last_used_at = time.monotonic() - 601.0 + async with service._http_bridge_lock: + stale_sessions = service._prune_http_bridge_sessions_locked() + assert key not in service._http_bridge_sessions + for stale_session in stale_sessions: + await service._close_http_bridge_session(stale_session) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_replayed_turn_state_alias_preserves_owner_without_rekeying_session( - async_client, - app_instance, - monkeypatch, -): - _install_bridge_settings_with_limits( - monkeypatch, - enabled=True, - codex_idle_ttl_seconds=600.0, - instance_id="instance-a", - instance_ring=["instance-a", "instance-b"], - ) +async def test_v1_responses_http_bridge_creation_honors_prefer_earlier_reset(async_client, app_instance, monkeypatch): + _install_bridge_settings_with_limits(monkeypatch, enabled=True, prefer_earlier_reset_accounts=True) account_id = await _import_account( async_client, - "acc_http_bridge_alias_owner", - "http-bridge-alias-owner@example.com", + "acc_http_bridge_prefer_earlier_reset", + "http-bridge-prefer-earlier-reset@example.com", ) account = await _get_account(account_id) service = get_proxy_service_for_app(app_instance) - upstreams = [_FakeBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] - connect_headers_seen: list[dict[str, str]] = [] + fake_upstream = _FakeBridgeUpstreamWebSocket() + select_calls: list[tuple[bool, str | None]] = [] async def fake_select_account_with_budget( self, @@ -5630,6 +1908,7 @@ async def fake_select_account_with_budget( reallocate_sticky, sticky_max_age_seconds, prefer_earlier_reset_accounts, + service_tier=None, routing_strategy, model, exclude_account_ids=None, @@ -5648,186 +1927,82 @@ async def fake_select_account_with_budget( sticky_kind, reallocate_sticky, sticky_max_age_seconds, - prefer_earlier_reset_accounts, routing_strategy, model, exclude_account_ids, additional_limit_name, ) + select_calls.append((prefer_earlier_reset_accounts, service_tier)) return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds return target - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del access_token, account_id_header, base_url, session - connect_headers_seen.append(dict(headers)) - return upstreams.pop(0) + async def fake_open_upstream_websocket_with_budget(self, target, headers, *, timeout_seconds): + del self, target, headers, timeout_seconds + return fake_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - candidate_suffix = 0 - while True: - payload = proxy_module.ResponsesRequest( - model="gpt-5.1", - instructions="Return exactly OK.", - input="hello", - prompt_cache_key=f"owner-alias-thread-{candidate_suffix}", - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={}, - affinity=affinity, - api_key=None, - request_id="req_owner_alias", - ) - if await proxy_module._http_bridge_owner_instance(key, proxy_module.get_settings()) == "instance-a": - break - candidate_suffix += 1 + monkeypatch.setattr( + proxy_module.ProxyService, + "_open_upstream_websocket_with_budget", + fake_open_upstream_websocket_with_budget, + ) - session = await service._get_or_create_http_bridge_session( - key, + payload = proxy_module.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "", + "input": "hello", + "prompt_cache_key": "bridge_prefer_earlier_reset", + "service_tier": "priority", + } + ) + affinity = proxy_module._sticky_key_for_responses_request( + payload, + {}, + codex_session_affinity=False, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + key = proxy_module._make_http_bridge_session_key( + payload, headers={}, affinity=affinity, api_key=None, - request_model=payload.model, - idle_ttl_seconds=120.0, - max_sessions=128, - ) - - replay_turn_state = None - for candidate in ("turn_owner_alias_b", "turn_owner_alias_c", "turn_owner_alias_d", "turn_owner_alias_e"): - if ( - await proxy_module._http_bridge_owner_instance( - proxy_module._HTTPBridgeSessionKey("turn_state_header", candidate, None), - proxy_module.get_settings(), - ) - == "instance-b" - ): - replay_turn_state = candidate - break - assert replay_turn_state is not None - await service._register_http_bridge_turn_state(session, replay_turn_state) - replay_key = proxy_module._HTTPBridgeSessionKey("turn_state_header", replay_turn_state, None) - assert ( - service._http_bridge_turn_state_index[ - proxy_module._http_bridge_turn_state_alias_key(replay_turn_state, session.key.api_key_id) - ] - == key + request_id="req_bridge_prefer_earlier_reset", ) - replayed = await service._get_or_create_http_bridge_session( - replay_key, - headers={"x-codex-turn-state": replay_turn_state}, - affinity=proxy_module._AffinityPolicy(key=replay_turn_state, kind=proxy_module.StickySessionKind.CODEX_SESSION), + session = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=affinity, api_key=None, request_model=payload.model, + request_service_tier=payload.service_tier, idle_ttl_seconds=120.0, - max_sessions=128, + max_sessions=8, + gateway_safe_mode=True, ) - assert replayed is session - assert replayed.key == key - assert key in service._http_bridge_sessions - assert replay_key not in service._http_bridge_sessions - assert ( - service._http_bridge_turn_state_index[ - proxy_module._http_bridge_turn_state_alias_key(replay_turn_state, session.key.api_key_id) - ] - == key - ) - assert replayed.codex_session is True - assert replayed.affinity.kind == proxy_module.StickySessionKind.CODEX_SESSION - assert replayed.affinity.key == replay_turn_state - assert replayed.idle_ttl_seconds >= 600.0 - replayed.upstream_turn_state = "upstream_turn_state_stale" - request_state = proxy_module._WebSocketRequestState( - request_id="req_owner_alias_reconnect", - model=payload.model, - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - ) - await service._reconnect_http_bridge_session(replayed, request_state=request_state) - assert connect_headers_seen[-1]["x-codex-turn-state"] == replay_turn_state + assert select_calls == [(True, "priority")] await service._close_http_bridge_session(session) - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_waits_for_inflight_recreation_on_missing_turn_state_alias(app_instance): - service = get_proxy_service_for_app(app_instance) - service._http_bridge_sessions.clear() - service._http_bridge_turn_state_index.clear() - service._http_bridge_inflight_sessions.clear() - - replay_turn_state = "http_turn_inflight_replay" - replay_key = proxy_module._HTTPBridgeSessionKey("turn_state_header", replay_turn_state, None) - expected_session = _make_dummy_bridge_session(replay_key) - inflight_future: asyncio.Future = asyncio.get_running_loop().create_future() - service._http_bridge_inflight_sessions[replay_key] = inflight_future - - request_key = proxy_module._HTTPBridgeSessionKey("request", "derived-key", None) - try: - waiter = asyncio.create_task( - service._get_or_create_http_bridge_session( - request_key, - headers={"x-codex-turn-state": replay_turn_state}, - affinity=proxy_module._AffinityPolicy(key="derived-key"), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - ) - ) - await asyncio.sleep(0) - assert not waiter.done() - inflight_future.set_result(expected_session) - returned = await waiter - finally: - service._http_bridge_inflight_sessions.clear() - - assert returned is expected_session - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_generated_turn_state_fails_closed_without_local_alias( - async_client, - app_instance, - monkeypatch, -): + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_codex_session_prewarms_first_request(async_client, monkeypatch): _install_bridge_settings_with_limits( monkeypatch, enabled=True, - instance_id="instance-a", - instance_ring=["instance-a", "instance-b"], - ) - account_id = await _import_account( - async_client, - "acc_http_bridge_missing_alias", - "http-bridge-missing-alias@example.com", + codex_idle_ttl_seconds=600.0, + codex_prewarm_enabled=True, ) + account_id = await _import_account(async_client, "acc_http_bridge_prewarm", "http-bridge-prewarm@example.com") account = await _get_account(account_id) - service = get_proxy_service_for_app(app_instance) + fake_upstream = _PrewarmingBridgeUpstreamWebSocket() async def fake_select_account_with_budget( self, @@ -5867,42 +2042,48 @@ async def fake_select_account_with_budget( ) return AccountSelection(account=account, error_message=None, error_code=None) + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + return fake_upstream + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._get_or_create_http_bridge_session( - proxy_module._HTTPBridgeSessionKey("turn_state_header", "http_turn_missing_alias", None), - headers={"x-codex-turn-state": "http_turn_missing_alias"}, - affinity=proxy_module._AffinityPolicy( - key="http_turn_missing_alias", - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ), - api_key=None, - request_model="gpt-5.1", - idle_ttl_seconds=120.0, - max_sessions=128, - ) + response = await async_client.post( + "/v1/responses", + headers={"x-codex-turn-state": "turn_state_prewarm"}, + json={ + "model": "gpt-5.4", + "instructions": "hi", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}], + }, + ) - exc = exc_info.value - assert exc.status_code == 409 - assert exc.payload["error"].get("code") == "bridge_instance_mismatch" + assert response.status_code == 200 + assert response.json()["id"] == "resp_actual_2" + assert len(fake_upstream.sent_text) == 2 + assert json.loads(fake_upstream.sent_text[0])["generate"] is False + assert "generate" not in json.loads(fake_upstream.sent_text[1]) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_turn_state_alias_respects_api_key_isolation( - async_client, - app_instance, - monkeypatch, -): - _install_bridge_settings_with_limits(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_api_key_alias", - "http-bridge-api-key-alias@example.com", - ) +async def test_v1_responses_http_bridge_codex_session_does_not_prewarm_by_default(async_client, monkeypatch): + _install_bridge_settings_with_limits(monkeypatch, enabled=True, codex_idle_ttl_seconds=600.0) + account_id = await _import_account(async_client, "acc_http_bridge_no_prewarm", "http-bridge-no-prewarm@example.com") account = await _get_account(account_id) - service = get_proxy_service_for_app(app_instance) - fake_upstream = _FakeBridgeUpstreamWebSocket() + fake_upstream = _PrewarmingBridgeUpstreamWebSocket() async def fake_select_account_with_budget( self, @@ -5961,138 +2142,170 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - payload = proxy_module.ResponsesRequest( - model="gpt-5.1", - instructions="Return exactly OK.", - input="hello", - prompt_cache_key="api-key-alias-thread", - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - api_key_a = cast(proxy_module.ApiKeyData, SimpleNamespace(id="api-key-a")) - session = await service._get_or_create_http_bridge_session( - proxy_module._make_http_bridge_session_key( - payload, - headers={}, - affinity=affinity, - api_key=api_key_a, - request_id="req_api_key_alias", - ), - headers={}, - affinity=affinity, - api_key=api_key_a, - request_model=payload.model, - idle_ttl_seconds=120.0, - max_sessions=128, + response = await async_client.post( + "/v1/responses", + headers={"x-codex-turn-state": "turn_state_no_prewarm"}, + json={ + "model": "gpt-5.4", + "instructions": "hi", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}], + }, ) - await service._register_http_bridge_turn_state(session, "http_turn_api_key_alias") - - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._get_or_create_http_bridge_session( - proxy_module._HTTPBridgeSessionKey("turn_state_header", "http_turn_api_key_alias", "api-key-b"), - headers={"x-codex-turn-state": "http_turn_api_key_alias"}, - affinity=proxy_module._AffinityPolicy( - key="http_turn_api_key_alias", - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ), - api_key=cast(proxy_module.ApiKeyData, SimpleNamespace(id="api-key-b")), - request_model=payload.model, - idle_ttl_seconds=120.0, - max_sessions=128, - ) - assert isinstance(exc_info.value, proxy_module.ProxyResponseError) - exc = exc_info.value - assert exc.status_code == 409 - assert exc.payload["error"].get("code") == "bridge_instance_mismatch" - await service._close_http_bridge_session(session) + assert response.status_code == 200 + assert response.json()["id"] == "resp_actual_1" + assert len(fake_upstream.sent_text) == 1 + assert "generate" not in json.loads(fake_upstream.sent_text[0]) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_closes_disallowed_session_before_owner_mismatch_retry( - app_instance, monkeypatch +async def test_v1_responses_http_bridge_non_owner_instance_falls_back_to_local_session( + async_client, + app_instance, + monkeypatch, ): _install_bridge_settings_with_limits( monkeypatch, enabled=True, - instance_id="instance-a", + gateway_safe_mode=True, + instance_id="instance-b", instance_ring=["instance-a", "instance-b"], ) + account_id = await _import_account(async_client, "acc_http_bridge_owner", "http-bridge-owner@example.com") + account = await _get_account(account_id) service = get_proxy_service_for_app(app_instance) - key = proxy_module._HTTPBridgeSessionKey("session_header", "shared-session", "key-assignments") - stale_api_key = _make_api_key_data(key_id="key-assignments", assigned_account_ids=["acc-stale"]) - refreshed_api_key = _make_api_key_data(key_id="key-assignments", assigned_account_ids=["acc-fresh"]) - upstream = _FakeBridgeUpstreamWebSocket() - stale_session = _make_dummy_bridge_session(key) - alias_key = proxy_module._http_bridge_turn_state_alias_key("http_turn_owner_retry", key.api_key_id) + service._ring_membership = cast( + proxy_module.RingMembershipService, + SimpleNamespace(list_active=AsyncMock(return_value=["instance-a", "instance-b"])), + ) - cast(Any, stale_session).account = SimpleNamespace(id="acc-stale", status=AccountStatus.ACTIVE, plan_type="plus") - cast(Any, stale_session).api_key = stale_api_key - cast(Any, stale_session).upstream = upstream - stale_session.downstream_turn_state_aliases.add("http_turn_owner_retry") - service._http_bridge_sessions[key] = stale_session - service._http_bridge_turn_state_index[alias_key] = key + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + ) + return AccountSelection(account=account, error_message=None, error_code=None) - async def fake_http_bridge_owner_instance(session_key, settings, ring_membership=None): - del settings, ring_membership - assert session_key == key - return "instance-b" + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target - async def fake_active_http_bridge_instance_ring(settings, ring_membership): - del settings, ring_membership - return "instance-a", ("instance-a", "instance-b") + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "_http_bridge_owner_instance", fake_http_bridge_owner_instance) - monkeypatch.setattr(proxy_module, "_active_http_bridge_instance_ring", fake_active_http_bridge_instance_ring) + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + return _FakeBridgeUpstreamWebSocket() - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._get_or_create_http_bridge_session( - key, - headers={"session_id": "shared-session"}, - affinity=proxy_module._AffinityPolicy( - key="shared-session", - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ), - api_key=refreshed_api_key, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + candidate_suffix = 0 + while True: + payload = proxy_module.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": [{"role": "user", "content": "hi"}], + "prompt_cache_key": f"owner-check-{candidate_suffix}", + } + ) + affinity = proxy_module._sticky_key_for_responses_request( + payload, + {}, + codex_session_affinity=False, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + key = proxy_module._make_http_bridge_session_key( + payload, + headers={}, + affinity=affinity, + api_key=None, + request_id="req_owner", ) + owner = await proxy_module._http_bridge_owner_instance(key, proxy_module.get_settings()) + if owner != "instance-b": + break + candidate_suffix += 1 - exc = exc_info.value - if exc.status_code == 409: - assert exc.payload["error"].get("code") == "bridge_instance_mismatch" - else: - assert exc.status_code == 503 - assert key not in service._http_bridge_inflight_sessions - assert key not in service._http_bridge_sessions - assert alias_key not in service._http_bridge_turn_state_index - assert stale_session.closed is True - assert upstream.closed is True + session = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=affinity, + api_key=None, + request_model=payload.model, + idle_ttl_seconds=120.0, + max_sessions=8, + gateway_safe_mode=True, + ) + + assert session is not None @pytest.mark.asyncio -async def test_v1_responses_http_bridge_preserves_prior_turn_state_aliases( +async def test_v1_responses_http_bridge_non_owner_prompt_cache_rebinds_locally_when_gateway_safe_mode_disabled( async_client, app_instance, monkeypatch, ): - _install_bridge_settings_with_limits(monkeypatch, enabled=True) + _install_bridge_settings_with_limits( + monkeypatch, + enabled=True, + gateway_safe_mode=False, + instance_id="instance-b", + instance_ring=["instance-a", "instance-b"], + ) account_id = await _import_account( async_client, - "acc_http_bridge_alias_preserve", - "http-bridge-alias-preserve@example.com", + "acc_http_bridge_owner_strict", + "http-bridge-owner-strict@example.com", ) account = await _get_account(account_id) service = get_proxy_service_for_app(app_instance) - fake_upstream = _FakeBridgeUpstreamWebSocket() + service._ring_membership = cast( + proxy_module.RingMembershipService, + SimpleNamespace(list_active=AsyncMock(return_value=["instance-a", "instance-b"])), + ) async def fake_select_account_with_budget( self, @@ -6108,10 +2321,10 @@ async def fake_select_account_with_budget( prefer_earlier_reset_accounts, routing_strategy, model, - exclude_account_ids=None, - additional_limit_name=None, api_key=None, preferred_account_id=None, + exclude_account_ids=None, + additional_limit_name=None, ): del preferred_account_id del ( @@ -6127,6 +2340,7 @@ async def fake_select_account_with_budget( prefer_earlier_reset_accounts, routing_strategy, model, + api_key, exclude_account_ids, additional_limit_name, ) @@ -6136,182 +2350,385 @@ async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_se del self, force, timeout_seconds return target - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - return fake_upstream + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr( + proxy_module, + "connect_responses_websocket", + AsyncMock(return_value=_FakeBridgeUpstreamWebSocket()), + ) + + candidate_suffix = 0 + while True: + payload = proxy_module.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": [{"role": "user", "content": "hi"}], + "prompt_cache_key": f"owner-check-strict-{candidate_suffix}", + } + ) + affinity = proxy_module._sticky_key_for_responses_request( + payload, + {}, + codex_session_affinity=False, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + key = proxy_module._make_http_bridge_session_key( + payload, + headers={}, + affinity=affinity, + api_key=None, + request_id="req_owner_strict", + ) + owner = await proxy_module._http_bridge_owner_instance(key, proxy_module.get_settings()) + if owner != "instance-b": + break + candidate_suffix += 1 + + session = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=affinity, + api_key=None, + request_model=payload.model, + idle_ttl_seconds=120.0, + max_sessions=8, + gateway_safe_mode=False, + ) + + assert session.account.id == account.id + assert session.key == key + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_missing_turn_state_alias_with_previous_response_id_fails_closed( + app_instance, + monkeypatch, +): + _install_bridge_settings_with_limits(monkeypatch, enabled=True) + service = get_proxy_service_for_app(app_instance) + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + proxy_module._HTTPBridgeSessionKey("turn_state_header", "http_turn_missing_alias", None), + headers={"x-codex-turn-state": "http_turn_missing_alias"}, + affinity=proxy_module._AffinityPolicy( + key="http_turn_missing_alias", + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ), + api_key=None, + request_model="gpt-5.1", + idle_ttl_seconds=120.0, + max_sessions=128, + previous_response_id="resp_missing_alias", + ) + + exc = exc_info.value + assert exc.status_code == 502 + assert exc.payload["error"] == { + "message": "Upstream websocket closed before response.completed", + "type": "server_error", + "code": "stream_incomplete", + } + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_stale_previous_response_alias_same_model_fails_closed( + app_instance, + monkeypatch, +): + _install_bridge_settings_with_limits(monkeypatch, enabled=True) + service = get_proxy_service_for_app(app_instance) + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() + service._http_bridge_previous_response_index.clear() + + previous_response_id = "resp_stale_same_model_alias" + stale_key = proxy_module._HTTPBridgeSessionKey("prompt_cache", "bridge-stale-prev-owner", None) + stale_session = _make_dummy_bridge_session(stale_key) + stale_session.request_model = "gpt-5.1" + stale_session.account = cast(Account, SimpleNamespace(id="acc-stale-prev-owner", status=AccountStatus.PAUSED)) + stale_session.previous_response_ids.add(previous_response_id) + service._http_bridge_sessions[stale_key] = stale_session + service._http_bridge_previous_response_index[(previous_response_id, None)] = stale_key + + async def fail_create_http_bridge_session(self, *args, **kwargs): + del self, args, kwargs + raise AssertionError("stale same-model previous_response_id must fail closed before replacement creation") + + monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fail_create_http_bridge_session) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + proxy_module._HTTPBridgeSessionKey("request", "bridge-stale-prev-request", None), + headers={}, + affinity=proxy_module._AffinityPolicy(), + api_key=None, + request_model="gpt-5.1", + idle_ttl_seconds=120.0, + max_sessions=128, + previous_response_id=previous_response_id, + ) + + exc = exc_info.value + assert exc.status_code == 502 + assert exc.payload["error"] == { + "message": "Upstream websocket closed before response.completed", + "type": "server_error", + "code": "stream_incomplete", + } + assert service._http_bridge_previous_response_index.get((previous_response_id, None)) is None + assert service._http_bridge_sessions[stale_key] is stale_session + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_previous_response_alias_rejects_service_tier_provenance_mismatch( + async_client, + app_instance, + monkeypatch, +): + _install_bridge_settings_with_limits(monkeypatch, enabled=True) + service = get_proxy_service_for_app(app_instance) + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() + service._http_bridge_previous_response_index.clear() + + account_id = await _import_account( + async_client, + "acc_previous_response_tier_owner", + "previous-response-tier-owner@example.com", + plan_type="pro", + ) + account = await _get_account(account_id) + previous_response_id = "resp_previous_response_tier_owner" + owner_key = proxy_module._HTTPBridgeSessionKey("prompt_cache", "bridge-old-prompt", None) + owner_session = _make_dummy_bridge_session(owner_key) + owner_session.request_model = "gpt-5.3-codex-spark" + owner_session.request_service_tier = None + owner_session.account = account + owner_session.unanchored_reservation_id = None + owner_upstream = _FakeBridgeUpstreamWebSocket() + owner_session.upstream = cast(proxy_module.UpstreamWebSocket, owner_upstream) + owner_session.catalog_omission_quota_admission = CatalogOmissionQuotaAdmission( + normalized_model="gpt-5.3-codex-spark", + canonical_quota_key="codex_spark", + normalized_effective_service_tier=None, + ) + owner_session.previous_response_ids.add(previous_response_id) + service._http_bridge_sessions[owner_key] = owner_session + service._http_bridge_previous_response_index[(previous_response_id, None)] = owner_key + turn_state = "http_turn_previous_response_tier_owner" + turn_state_alias_key = proxy_module._http_bridge_turn_state_alias_key(turn_state, None) + owner_session.downstream_turn_state_aliases.add(turn_state) + service._http_bridge_turn_state_index[turn_state_alias_key] = owner_key + previous_response_index_before = dict(service._http_bridge_previous_response_index) + turn_state_index_before = dict(service._http_bridge_turn_state_index) + + class Registry: + def account_ids_for_model(self, model: str) -> set[str]: + assert model == "gpt-5.3-codex-spark" + return set() - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + def plan_types_for_model(self, model: str) -> set[str]: + assert model == "gpt-5.3-codex-spark" + return {"pro"} - payload = proxy_module.ResponsesRequest( - model="gpt-5.1", - instructions="Return exactly OK.", - input="hello", - prompt_cache_key="alias-preserve-thread", - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - session = await service._get_or_create_http_bridge_session( - proxy_module._make_http_bridge_session_key( - payload, - headers={}, - affinity=affinity, - api_key=None, - request_id="req_alias_preserve", - ), - headers={}, - affinity=affinity, - api_key=None, - request_model=payload.model, - idle_ttl_seconds=120.0, - max_sessions=128, - ) + def account_ids_for_model_service_tier(self, model: str, service_tier: str) -> set[str]: + assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") + return set() - await service._register_http_bridge_turn_state(session, "http_turn_alias_a") - await service._register_http_bridge_turn_state(session, "http_turn_alias_b") + def plan_types_for_model_service_tier(self, model: str, service_tier: str) -> set[str]: + assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") + return {"pro"} - replayed = await service._get_or_create_http_bridge_session( - proxy_module._HTTPBridgeSessionKey("turn_state_header", "http_turn_alias_a", None), - headers={"x-codex-turn-state": "http_turn_alias_a"}, + def get_snapshot(self): + return SimpleNamespace(account_plans={account_id: "pro"}) + + monkeypatch.setattr(proxy_support, "get_model_registry", lambda: Registry()) + + fallback_session_id = "priority-compatible-session" + fallback_key = proxy_module._HTTPBridgeSessionKey("session_header", fallback_session_id, None) + fallback_upstream = _FakeBridgeUpstreamWebSocket() + fallback_session = proxy_module._HTTPBridgeSession( + key=fallback_key, + headers={"x-codex-session-id": fallback_session_id}, affinity=proxy_module._AffinityPolicy( - key="http_turn_alias_a", + key=fallback_session_id, kind=proxy_module.StickySessionKind.CODEX_SESSION, ), - api_key=None, - request_model=payload.model, + request_model="gpt-5.3-codex-spark", + account=account, + upstream=cast(proxy_module.UpstreamWebSocket, fallback_upstream), + upstream_control=proxy_module._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=time.monotonic(), idle_ttl_seconds=120.0, - max_sessions=128, + request_service_tier="priority", + catalog_omission_quota_admission=CatalogOmissionQuotaAdmission( + normalized_model="gpt-5.3-codex-spark", + canonical_quota_key="codex_spark", + normalized_effective_service_tier="priority", + ), + ) + fallback_session.upstream_reader = asyncio.create_task( + service._relay_http_bridge_upstream_messages(fallback_session) + ) + service._http_bridge_sessions[fallback_key] = fallback_session + create_http_bridge_session = AsyncMock( + side_effect=AssertionError("anchored mismatch must fail before bridge creation") ) + monkeypatch.setattr(service, "_create_http_bridge_session", create_http_bridge_session) + scheduled_sessions: list[proxy_module._HTTPBridgeSession] = [] + schedule_session_closes = service._schedule_http_bridge_session_closes - assert replayed is session - assert "http_turn_alias_a" in replayed.downstream_turn_state_aliases - assert "http_turn_alias_b" in replayed.downstream_turn_state_aliases - await service._close_http_bridge_session(session) + def capture_scheduled_sessions(sessions, *, reason): + scheduled_sessions.extend(sessions) + schedule_session_closes(sessions, reason=reason) + monkeypatch.setattr(service, "_schedule_http_bridge_session_closes", capture_scheduled_sessions) + owner_state_before = ( + owner_session.request_model, + owner_session.request_service_tier, + owner_session.catalog_omission_quota_admission, + owner_session.upstream, + set(owner_session.previous_response_ids), + ) + fallback_state_before = ( + fallback_session.request_model, + fallback_session.request_service_tier, + fallback_session.catalog_omission_quota_admission, + fallback_session.upstream, + set(fallback_session.previous_response_ids), + ) + owner_request_count = len(owner_upstream.sent_text) + fallback_request_count = len(fallback_upstream.sent_text) -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_close_waits_for_turn_state_index_lock( - async_client, - app_instance, - monkeypatch, -): - _install_bridge_settings_with_limits(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_close_lock", - "http-bridge-close-lock@example.com", + rejected = await async_client.post( + "/v1/responses", + headers={ + "x-codex-turn-state": "http_turn_previous_response_tier_fallback", + "x-codex-session-id": fallback_session_id, + }, + json={ + "model": "gpt-5.3-codex-spark", + "instructions": "Return exactly OK.", + "input": "must fail before session fallback", + "previous_response_id": previous_response_id, + "service_tier": "priority", + }, ) - account = await _get_account(account_id) - service = get_proxy_service_for_app(app_instance) - fake_upstream = _FakeBridgeUpstreamWebSocket() - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, + assert len(fallback_upstream.sent_text) == fallback_request_count + assert rejected.status_code == 502, rejected.text + assert rejected.json()["error"] == { + "message": "Upstream websocket closed before response.completed", + "type": "server_error", + "code": "stream_incomplete", + } + create_http_bridge_session.assert_not_awaited() + assert len(owner_upstream.sent_text) == owner_request_count + assert owner_session.closed is False + assert fallback_session.closed is False + assert service._http_bridge_sessions[owner_key] is owner_session + assert service._http_bridge_sessions[fallback_key] is fallback_session + assert owner_session not in scheduled_sessions + assert fallback_session not in scheduled_sessions + assert service._http_bridge_previous_response_index == previous_response_index_before + assert service._http_bridge_turn_state_index == turn_state_index_before + assert ( + owner_session.request_model, + owner_session.request_service_tier, + owner_session.catalog_omission_quota_admission, + owner_session.upstream, + set(owner_session.previous_response_ids), + ) == owner_state_before + assert ( + fallback_session.request_model, + fallback_session.request_service_tier, + fallback_session.catalog_omission_quota_admission, + fallback_session.upstream, + set(fallback_session.previous_response_ids), + ) == fallback_state_before + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + proxy_module._HTTPBridgeSessionKey("prompt_cache", "bridge-new-prompt", None), + headers={}, + affinity=proxy_module._AffinityPolicy( + key="bridge-new-prompt", + kind=proxy_module.StickySessionKind.PROMPT_CACHE, + ), + api_key=None, + request_model="gpt-5.3-codex-spark", + request_service_tier="priority", + idle_ttl_seconds=120.0, + max_sessions=128, + previous_response_id=previous_response_id, ) - return AccountSelection(account=account, error_message=None, error_code=None) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target + assert exc_info.value.status_code == 502 + assert owner_session.request_service_tier is None + assert service._http_bridge_sessions[owner_key] is owner_session - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - return fake_upstream + assert service._http_bridge_previous_response_index == previous_response_index_before + assert service._http_bridge_turn_state_index == turn_state_index_before - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + with pytest.raises(proxy_module.ProxyResponseError) as turn_state_exc_info: + await service._get_or_create_http_bridge_session( + proxy_module._HTTPBridgeSessionKey("request", "bridge-turn-state-tier-mismatch", None), + headers={"x-codex-turn-state": turn_state}, + affinity=proxy_module._AffinityPolicy( + key=turn_state, + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ), + api_key=None, + request_model="gpt-5.3-codex-spark", + request_service_tier="priority", + idle_ttl_seconds=120.0, + max_sessions=128, + ) - payload = proxy_module.ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) - affinity = proxy_module._AffinityPolicy(key="turn-close-lock", kind=proxy_module.StickySessionKind.CODEX_SESSION) + assert turn_state_exc_info.value.status_code == 502 + assert owner_session.request_model == "gpt-5.3-codex-spark" + assert owner_session.request_service_tier is None + assert service._http_bridge_sessions[owner_key] is owner_session + assert service._http_bridge_previous_response_index == previous_response_index_before + assert service._http_bridge_turn_state_index == turn_state_index_before - session = await service._get_or_create_http_bridge_session( - proxy_module._make_http_bridge_session_key( - payload, - headers={}, - affinity=affinity, - api_key=None, - request_id="req_close_lock", - ), + reused = await service._get_or_create_http_bridge_session( + proxy_module._HTTPBridgeSessionKey("prompt_cache", "bridge-correct-prompt", None), headers={}, - affinity=affinity, + affinity=proxy_module._AffinityPolicy( + key="bridge-correct-prompt", + kind=proxy_module.StickySessionKind.PROMPT_CACHE, + ), api_key=None, - request_model=payload.model, + request_model="gpt-5.3-codex-spark", + request_service_tier=None, idle_ttl_seconds=120.0, max_sessions=128, + previous_response_id=previous_response_id, ) - await service._register_http_bridge_turn_state(session, "http_turn_close_lock") - alias_key = proxy_module._http_bridge_turn_state_alias_key("http_turn_close_lock", session.key.api_key_id) - - async with service._http_bridge_lock: - close_task = asyncio.create_task(service._close_http_bridge_session(session)) - await asyncio.sleep(0) - assert not close_task.done() - assert service._http_bridge_turn_state_index[alias_key] == session.key - - await close_task - - assert alias_key not in service._http_bridge_turn_state_index + assert reused is owner_session + assert service._http_bridge_previous_response_index == previous_response_index_before + assert service._http_bridge_turn_state_index == turn_state_index_before @pytest.mark.asyncio -async def test_v1_responses_http_bridge_allows_unstable_request_key_even_on_non_owner_instance( +async def test_v1_responses_http_bridge_replayed_turn_state_alias_preserves_owner_without_rekeying_session( async_client, app_instance, monkeypatch, @@ -6319,13 +2736,19 @@ async def test_v1_responses_http_bridge_allows_unstable_request_key_even_on_non_ _install_bridge_settings_with_limits( monkeypatch, enabled=True, - instance_id="instance-b", + codex_idle_ttl_seconds=600.0, + instance_id="instance-a", instance_ring=["instance-a", "instance-b"], ) - account_id = await _import_account(async_client, "acc_http_bridge_unstable", "http-bridge-unstable@example.com") + account_id = await _import_account( + async_client, + "acc_http_bridge_alias_owner", + "http-bridge-alias-owner@example.com", + ) account = await _get_account(account_id) service = get_proxy_service_for_app(app_instance) - fake_upstream = _FakeBridgeUpstreamWebSocket() + upstreams = [_FakeBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] + connect_headers_seen: list[dict[str, str]] = [] async def fake_select_account_with_budget( self, @@ -6377,32 +2800,41 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, account_id_header, base_url, session - return fake_upstream + del access_token, account_id_header, base_url, session + connect_headers_seen.append(dict(headers)) + return upstreams.pop(0) monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - payload = proxy_module.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=False, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={}, - affinity=affinity, - api_key=None, - request_id="req_owner_unstable", - ) + candidate_suffix = 0 + while True: + payload = proxy_module.ResponsesRequest( + model="gpt-5.1", + instructions="Return exactly OK.", + input="hello", + prompt_cache_key=f"owner-alias-thread-{candidate_suffix}", + ) + affinity = proxy_module._sticky_key_for_responses_request( + payload, + {}, + codex_session_affinity=False, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + key = proxy_module._make_http_bridge_session_key( + payload, + headers={}, + affinity=affinity, + api_key=None, + request_id="req_owner_alias", + ) + if await proxy_module._http_bridge_owner_instance(key, proxy_module.get_settings()) == "instance-a": + break + candidate_suffix += 1 session = await service._get_or_create_http_bridge_session( key, @@ -6411,32 +2843,123 @@ async def fake_connect_responses_websocket( api_key=None, request_model=payload.model, idle_ttl_seconds=120.0, - max_sessions=8, + max_sessions=128, ) - assert session.key.affinity_kind == "request" + replay_turn_state = None + for candidate in ("turn_owner_alias_b", "turn_owner_alias_c", "turn_owner_alias_d", "turn_owner_alias_e"): + if ( + await proxy_module._http_bridge_owner_instance( + proxy_module._HTTPBridgeSessionKey("turn_state_header", candidate, None), + proxy_module.get_settings(), + ) + == "instance-b" + ): + replay_turn_state = candidate + break + assert replay_turn_state is not None + await service._register_http_bridge_turn_state(session, replay_turn_state) + replay_key = proxy_module._HTTPBridgeSessionKey("turn_state_header", replay_turn_state, None) + assert ( + service._http_bridge_turn_state_index[ + proxy_module._http_bridge_turn_state_alias_key(replay_turn_state, session.key.api_key_id) + ] + == key + ) + + replayed = await service._get_or_create_http_bridge_session( + replay_key, + headers={"x-codex-turn-state": replay_turn_state}, + affinity=proxy_module._AffinityPolicy(key=replay_turn_state, kind=proxy_module.StickySessionKind.CODEX_SESSION), + api_key=None, + request_model=payload.model, + idle_ttl_seconds=120.0, + max_sessions=128, + ) + + assert replayed is session + assert replayed.key == key + assert key in service._http_bridge_sessions + assert replay_key not in service._http_bridge_sessions + assert ( + service._http_bridge_turn_state_index[ + proxy_module._http_bridge_turn_state_alias_key(replay_turn_state, session.key.api_key_id) + ] + == key + ) + assert replayed.codex_session is True + assert replayed.affinity.kind == proxy_module.StickySessionKind.CODEX_SESSION + assert replayed.affinity.key == replay_turn_state + assert replayed.idle_ttl_seconds >= 600.0 + replayed.upstream_turn_state = "upstream_turn_state_stale" + request_state = proxy_module._WebSocketRequestState( + request_id="req_owner_alias_reconnect", + model=payload.model, + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + ) + await service._reconnect_http_bridge_session(replayed, request_state=request_state) + assert connect_headers_seen[-1]["x-codex-turn-state"] == replay_turn_state + await service._close_http_bridge_session(session) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_reconnect_uses_last_upstream_turn_state( +async def test_v1_responses_http_bridge_waits_for_inflight_recreation_on_missing_turn_state_alias(app_instance): + service = get_proxy_service_for_app(app_instance) + service._http_bridge_sessions.clear() + service._http_bridge_turn_state_index.clear() + service._http_bridge_inflight_sessions.clear() + + replay_turn_state = "http_turn_inflight_replay" + replay_key = proxy_module._HTTPBridgeSessionKey("turn_state_header", replay_turn_state, None) + expected_session = _make_dummy_bridge_session(replay_key) + inflight_future: asyncio.Future = asyncio.get_running_loop().create_future() + service._http_bridge_inflight_sessions[replay_key] = inflight_future + + request_key = proxy_module._HTTPBridgeSessionKey("request", "derived-key", None) + try: + waiter = asyncio.create_task( + service._get_or_create_http_bridge_session( + request_key, + headers={"x-codex-turn-state": replay_turn_state}, + affinity=proxy_module._AffinityPolicy(key="derived-key"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + ) + await asyncio.sleep(0) + assert not waiter.done() + inflight_future.set_result(expected_session) + returned = await waiter + finally: + service._http_bridge_inflight_sessions.clear() + + assert returned is expected_session + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_generated_turn_state_fails_closed_without_local_alias( async_client, app_instance, monkeypatch, ): - _install_bridge_settings_with_limits(monkeypatch, enabled=True) + _install_bridge_settings_with_limits( + monkeypatch, + enabled=True, + instance_id="instance-a", + instance_ring=["instance-a", "instance-b"], + ) account_id = await _import_account( async_client, - "acc_http_bridge_upstream_turn", - "http-bridge-upstream-turn@example.com", + "acc_http_bridge_missing_alias", + "http-bridge-missing-alias@example.com", ) account = await _get_account(account_id) service = get_proxy_service_for_app(app_instance) - connect_headers_seen: list[dict[str, str]] = [] - upstreams = [ - _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_1"), - _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_2"), - _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_3"), - ] async def fake_select_account_with_budget( self, @@ -6461,90 +2984,44 @@ async def fake_select_account_with_budget( del ( self, deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - ) - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del access_token, account_id_header, base_url, session - connect_headers_seen.append(dict(headers)) - return upstreams.pop(0) + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + ) + return AccountSelection(account=account, error_message=None, error_code=None) monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - payload = proxy_module.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {"x-codex-turn-state": "local_turn_state"}, - codex_session_affinity=True, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={"x-codex-turn-state": "local_turn_state"}, - affinity=affinity, - api_key=None, - request_id="req_turn_state", - ) - bridge_session = await service._get_or_create_http_bridge_session( - key, - headers={"x-codex-turn-state": "local_turn_state"}, - affinity=affinity, - api_key=None, - request_model=payload.model, - idle_ttl_seconds=120.0, - max_sessions=8, - ) - request_state = proxy_module._WebSocketRequestState( - request_id="req-turn-state-reconnect", - model=payload.model, - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, - response_create_gate_acquired=True, - request_text=json.dumps({"type": "response.create", "model": "gpt-5.4", "input": []}), - ) - await service._reconnect_http_bridge_session(bridge_session, request_state=request_state) + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + proxy_module._HTTPBridgeSessionKey("turn_state_header", "http_turn_missing_alias", None), + headers={"x-codex-turn-state": "http_turn_missing_alias"}, + affinity=proxy_module._AffinityPolicy( + key="http_turn_missing_alias", + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ), + api_key=None, + request_model="gpt-5.1", + idle_ttl_seconds=120.0, + max_sessions=128, + ) - assert connect_headers_seen[0]["x-codex-turn-state"] == "local_turn_state" - assert connect_headers_seen[1]["x-codex-turn-state"] == "upstream_turn_state_1" - assert bridge_session.upstream_turn_state == "upstream_turn_state_2" + exc = exc_info.value + assert exc.status_code == 409 + assert exc.payload["error"].get("code") == "bridge_instance_mismatch" @pytest.mark.asyncio -async def test_v1_responses_http_bridge_session_id_reconnect_keeps_upstream_turn_state( +async def test_v1_responses_http_bridge_turn_state_alias_respects_api_key_isolation( async_client, app_instance, monkeypatch, @@ -6552,17 +3029,12 @@ async def test_v1_responses_http_bridge_session_id_reconnect_keeps_upstream_turn _install_bridge_settings_with_limits(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_session_reconnect", - "http-bridge-session-reconnect@example.com", + "acc_http_bridge_api_key_alias", + "http-bridge-api-key-alias@example.com", ) account = await _get_account(account_id) service = get_proxy_service_for_app(app_instance) - connect_headers_seen: list[dict[str, str]] = [] - upstreams = [ - _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_1"), - _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_2"), - _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_3"), - ] + fake_upstream = _FakeBridgeUpstreamWebSocket() async def fake_select_account_with_budget( self, @@ -6614,67 +3086,132 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del access_token, account_id_header, base_url, session - connect_headers_seen.append(dict(headers)) - return upstreams.pop(0) + del headers, access_token, account_id_header, base_url, session + return fake_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - payload = proxy_module.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} + payload = proxy_module.ResponsesRequest( + model="gpt-5.1", + instructions="Return exactly OK.", + input="hello", + prompt_cache_key="api-key-alias-thread", ) - headers = {"session_id": "session_http_bridge_1"} affinity = proxy_module._sticky_key_for_responses_request( payload, - headers, - codex_session_affinity=True, + {}, + codex_session_affinity=False, openai_cache_affinity=True, openai_cache_affinity_max_age_seconds=300, sticky_threads_enabled=False, api_key=None, ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers=headers, - affinity=affinity, - api_key=None, - request_id="req_session_turn_state", - ) - bridge_session = await service._get_or_create_http_bridge_session( - key, - headers=headers, + api_key_a = cast(proxy_module.ApiKeyData, SimpleNamespace(id="api-key-a")) + session = await service._get_or_create_http_bridge_session( + proxy_module._make_http_bridge_session_key( + payload, + headers={}, + affinity=affinity, + api_key=api_key_a, + request_id="req_api_key_alias", + ), + headers={}, affinity=affinity, - api_key=None, + api_key=api_key_a, request_model=payload.model, idle_ttl_seconds=120.0, - max_sessions=8, + max_sessions=128, ) - await service._register_http_bridge_turn_state(bridge_session, "http_turn_alias_session") + await service._register_http_bridge_turn_state(session, "http_turn_api_key_alias") - request_state = proxy_module._WebSocketRequestState( - request_id="req-session-turn-state-reconnect", - model=payload.model, - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, - response_create_gate_acquired=True, - request_text=json.dumps({"type": "response.create", "model": "gpt-5.4", "input": []}), + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + proxy_module._HTTPBridgeSessionKey("turn_state_header", "http_turn_api_key_alias", "api-key-b"), + headers={"x-codex-turn-state": "http_turn_api_key_alias"}, + affinity=proxy_module._AffinityPolicy( + key="http_turn_api_key_alias", + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ), + api_key=cast(proxy_module.ApiKeyData, SimpleNamespace(id="api-key-b")), + request_model=payload.model, + idle_ttl_seconds=120.0, + max_sessions=128, + ) + + assert isinstance(exc_info.value, proxy_module.ProxyResponseError) + exc = exc_info.value + assert exc.status_code == 409 + assert exc.payload["error"].get("code") == "bridge_instance_mismatch" + await service._close_http_bridge_session(session) + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_closes_disallowed_session_before_owner_mismatch_retry( + app_instance, monkeypatch +): + _install_bridge_settings_with_limits( + monkeypatch, + enabled=True, + instance_id="instance-a", + instance_ring=["instance-a", "instance-b"], ) - await service._reconnect_http_bridge_session(bridge_session, request_state=request_state) + service = get_proxy_service_for_app(app_instance) + key = proxy_module._HTTPBridgeSessionKey("session_header", "shared-session", "key-assignments") + stale_api_key = _make_api_key_data(key_id="key-assignments", assigned_account_ids=["acc-stale"]) + refreshed_api_key = _make_api_key_data(key_id="key-assignments", assigned_account_ids=["acc-fresh"]) + upstream = _FakeBridgeUpstreamWebSocket() + stale_session = _make_dummy_bridge_session(key) + alias_key = proxy_module._http_bridge_turn_state_alias_key("http_turn_owner_retry", key.api_key_id) - assert connect_headers_seen[0]["session_id"] == "session_http_bridge_1" - assert "x-codex-turn-state" not in connect_headers_seen[0] - assert connect_headers_seen[1]["x-codex-turn-state"] == "upstream_turn_state_1" - assert bridge_session.downstream_turn_state == "http_turn_alias_session" - assert bridge_session.upstream_turn_state == "upstream_turn_state_2" + cast(Any, stale_session).account = SimpleNamespace(id="acc-stale", status=AccountStatus.ACTIVE, plan_type="plus") + cast(Any, stale_session).api_key = stale_api_key + cast(Any, stale_session).upstream = upstream + stale_session.downstream_turn_state_aliases.add("http_turn_owner_retry") + service._http_bridge_sessions[key] = stale_session + service._http_bridge_turn_state_index[alias_key] = key + + async def fake_http_bridge_owner_instance(session_key, settings, ring_membership=None): + del settings, ring_membership + assert session_key == key + return "instance-b" + + async def fake_active_http_bridge_instance_ring(settings, ring_membership): + del settings, ring_membership + return "instance-a", ("instance-a", "instance-b") + + monkeypatch.setattr(proxy_module, "_http_bridge_owner_instance", fake_http_bridge_owner_instance) + monkeypatch.setattr(proxy_module, "_active_http_bridge_instance_ring", fake_active_http_bridge_instance_ring) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + key, + headers={"session_id": "shared-session"}, + affinity=proxy_module._AffinityPolicy( + key="shared-session", + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ), + api_key=refreshed_api_key, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + + exc = exc_info.value + if exc.status_code == 409: + assert exc.payload["error"].get("code") == "bridge_instance_mismatch" + else: + assert exc.status_code == 503 + assert key not in service._http_bridge_inflight_sessions + assert key not in service._http_bridge_sessions + assert alias_key not in service._http_bridge_turn_state_index + assert stale_session.closed is True + assert upstream.closed is True @pytest.mark.asyncio -async def test_v1_responses_http_bridge_reconnect_uses_refreshed_api_key_assignments_for_reused_session( +async def test_v1_responses_http_bridge_preserves_prior_turn_state_aliases( async_client, app_instance, monkeypatch, @@ -6682,17 +3219,12 @@ async def test_v1_responses_http_bridge_reconnect_uses_refreshed_api_key_assignm _install_bridge_settings_with_limits(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_assignment_refresh", - "http-bridge-assignment-refresh@example.com", + "acc_http_bridge_alias_preserve", + "http-bridge-alias-preserve@example.com", ) account = await _get_account(account_id) service = get_proxy_service_for_app(app_instance) - selection_assigned_account_ids: list[list[str]] = [] - upstreams = [ - _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_1"), - _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_2"), - _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_3"), - ] + fake_upstream = _FakeBridgeUpstreamWebSocket() async def fake_select_account_with_budget( self, @@ -6730,7 +3262,6 @@ async def fake_select_account_with_budget( exclude_account_ids, additional_limit_name, ) - selection_assigned_account_ids.append(list(api_key.assigned_account_ids if api_key is not None else [])) return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): @@ -6746,77 +3277,67 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - return upstreams.pop(0) + return fake_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - stale_api_key = _make_api_key_data(key_id="key_http_bridge_assignments", assigned_account_ids=["acc-stale"]) - refreshed_api_key = _make_api_key_data( - key_id="key_http_bridge_assignments", - assigned_account_ids=["acc-refreshed"], - ) - payload = proxy_module.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} + payload = proxy_module.ResponsesRequest( + model="gpt-5.1", + instructions="Return exactly OK.", + input="hello", + prompt_cache_key="alias-preserve-thread", ) affinity = proxy_module._sticky_key_for_responses_request( payload, - headers={"session_id": "session_http_bridge_assignment_refresh"}, - codex_session_affinity=True, + {}, + codex_session_affinity=False, openai_cache_affinity=True, openai_cache_affinity_max_age_seconds=300, sticky_threads_enabled=False, - api_key=stale_api_key, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={"session_id": "session_http_bridge_assignment_refresh"}, - affinity=affinity, - api_key=stale_api_key, - request_id="req_assignment_refresh", + api_key=None, ) - bridge_session = await service._get_or_create_http_bridge_session( - key, - headers={"session_id": "session_http_bridge_assignment_refresh"}, + session = await service._get_or_create_http_bridge_session( + proxy_module._make_http_bridge_session_key( + payload, + headers={}, + affinity=affinity, + api_key=None, + request_id="req_alias_preserve", + ), + headers={}, affinity=affinity, - api_key=stale_api_key, + api_key=None, request_model=payload.model, idle_ttl_seconds=120.0, - max_sessions=8, + max_sessions=128, ) - reused_session = await service._get_or_create_http_bridge_session( - key, - headers={"session_id": "session_http_bridge_assignment_refresh"}, - affinity=affinity, - api_key=refreshed_api_key, + await service._register_http_bridge_turn_state(session, "http_turn_alias_a") + await service._register_http_bridge_turn_state(session, "http_turn_alias_b") + + replayed = await service._get_or_create_http_bridge_session( + proxy_module._HTTPBridgeSessionKey("turn_state_header", "http_turn_alias_a", None), + headers={"x-codex-turn-state": "http_turn_alias_a"}, + affinity=proxy_module._AffinityPolicy( + key="http_turn_alias_a", + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ), + api_key=None, request_model=payload.model, idle_ttl_seconds=120.0, - max_sessions=8, - ) - assert reused_session is not bridge_session - assert bridge_session.closed is True - assert reused_session.api_key == refreshed_api_key - - request_state = proxy_module._WebSocketRequestState( - request_id="req-assignment-refresh-reconnect", - model=payload.model, - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, - api_key=refreshed_api_key, - request_text=json.dumps({"type": "response.create", "model": "gpt-5.4", "input": []}), + max_sessions=128, ) - await service._reconnect_http_bridge_session(reused_session, request_state=request_state) - assert selection_assigned_account_ids == [["acc-stale"], ["acc-refreshed"], ["acc-refreshed"]] + assert replayed is session + assert "http_turn_alias_a" in replayed.downstream_turn_state_aliases + assert "http_turn_alias_b" in replayed.downstream_turn_state_aliases + await service._close_http_bridge_session(session) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_reconnect_fails_when_reader_cancel_times_out( +async def test_v1_responses_http_bridge_close_waits_for_turn_state_index_lock( async_client, app_instance, monkeypatch, @@ -6824,12 +3345,12 @@ async def test_v1_responses_http_bridge_reconnect_fails_when_reader_cancel_times _install_bridge_settings_with_limits(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_reconnect_cancel_timeout", - "http-bridge-reconnect-cancel-timeout@example.com", + "acc_http_bridge_close_lock", + "http-bridge-close-lock@example.com", ) account = await _get_account(account_id) service = get_proxy_service_for_app(app_instance) - upstreams = [_FakeBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] + fake_upstream = _FakeBridgeUpstreamWebSocket() async def fake_select_account_with_budget( self, @@ -6882,102 +3403,61 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - return upstreams.pop(0) + return fake_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - payload = proxy_module.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {"x-codex-turn-state": "timeout_turn_state"}, - codex_session_affinity=True, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={"x-codex-turn-state": "timeout_turn_state"}, - affinity=affinity, - api_key=None, - request_id="req_timeout_turn_state", - ) - bridge_session = await service._get_or_create_http_bridge_session( - key, - headers={"x-codex-turn-state": "timeout_turn_state"}, + payload = proxy_module.ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + affinity = proxy_module._AffinityPolicy(key="turn-close-lock", kind=proxy_module.StickySessionKind.CODEX_SESSION) + + session = await service._get_or_create_http_bridge_session( + proxy_module._make_http_bridge_session_key( + payload, + headers={}, + affinity=affinity, + api_key=None, + request_id="req_close_lock", + ), + headers={}, affinity=affinity, api_key=None, request_model=payload.model, idle_ttl_seconds=120.0, - max_sessions=8, + max_sessions=128, ) - original_upstream = bridge_session.upstream - - blocker = asyncio.Event() - - async def blocking_reader_task() -> None: - await _wait_for_event(blocker) - - original_reader = bridge_session.upstream_reader - assert original_reader is not None - original_reader.cancel() - with contextlib.suppress(asyncio.CancelledError): - await original_reader - blocking_reader = asyncio.create_task(blocking_reader_task()) - bridge_session.upstream_reader = blocking_reader - - async def fake_await_cancelled_task(task, *, timeout_seconds=1.0, label, cleanup_tasks=None): - del task, timeout_seconds, label, cleanup_tasks - return False + await service._register_http_bridge_turn_state(session, "http_turn_close_lock") - monkeypatch.setattr(proxy_module, "_await_cancelled_task", fake_await_cancelled_task) + alias_key = proxy_module._http_bridge_turn_state_alias_key("http_turn_close_lock", session.key.api_key_id) - request_state = proxy_module._WebSocketRequestState( - request_id="req-timeout-reconnect", - model=payload.model, - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, - response_create_gate_acquired=True, - request_text=json.dumps({"type": "response.create", "model": "gpt-5.4", "input": []}), - ) + async with service._http_bridge_lock: + close_task = asyncio.create_task(service._close_http_bridge_session(session)) + await asyncio.sleep(0) + assert not close_task.done() + assert service._http_bridge_turn_state_index[alias_key] == session.key - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._reconnect_http_bridge_session( - bridge_session, - request_state=request_state, - restart_reader=True, - ) + await close_task - error_payload = exc_info.value.payload["error"] - assert exc_info.value.status_code == 502 - assert error_payload.get("code") == "upstream_unavailable" - assert "reader did not shut down cleanly" in (error_payload.get("message") or "") - assert bridge_session.closed is True - assert bridge_session.upstream is original_upstream - blocking_reader.cancel() - with contextlib.suppress(asyncio.CancelledError): - await blocking_reader + assert alias_key not in service._http_bridge_turn_state_index @pytest.mark.asyncio -async def test_v1_responses_http_bridge_prefers_evicting_prompt_cache_session_before_codex_session( +async def test_v1_responses_http_bridge_allows_unstable_request_key_even_on_non_owner_instance( async_client, app_instance, monkeypatch, ): - _install_bridge_settings_with_limits(monkeypatch, enabled=True, max_sessions=2, codex_idle_ttl_seconds=600.0) - account_id = await _import_account(async_client, "acc_http_bridge_evict_pref", "http-bridge-evict-pref@example.com") + _install_bridge_settings_with_limits( + monkeypatch, + enabled=True, + instance_id="instance-b", + instance_ring=["instance-a", "instance-b"], + ) + account_id = await _import_account(async_client, "acc_http_bridge_unstable", "http-bridge-unstable@example.com") account = await _get_account(account_id) service = get_proxy_service_for_app(app_instance) - upstreams = [_FakeBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] + fake_upstream = _FakeBridgeUpstreamWebSocket() async def fake_select_account_with_budget( self, @@ -7030,7 +3510,7 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - return upstreams.pop(0) + return fake_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) @@ -7039,162 +3519,56 @@ async def fake_connect_responses_websocket( payload = proxy_module.ResponsesRequest.model_validate( {"model": "gpt-5.4", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} ) - - codex_affinity = proxy_module._sticky_key_for_responses_request( - payload, - {"x-codex-turn-state": "turn_state_1"}, - codex_session_affinity=True, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - codex_key = proxy_module._make_http_bridge_session_key( - payload, - headers={"x-codex-turn-state": "turn_state_1"}, - affinity=codex_affinity, - api_key=None, - request_id="req_codex", - ) - codex_session = await service._get_or_create_http_bridge_session( - codex_key, - headers={"x-codex-turn-state": "turn_state_1"}, - affinity=codex_affinity, - api_key=None, - request_model=payload.model, - idle_ttl_seconds=120.0, - max_sessions=2, - ) - codex_session.last_used_at = time.monotonic() - 50.0 - - prompt_payload = proxy_module.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "hi", - "input": [{"role": "user", "content": "hi"}], - "prompt_cache_key": "prompt_cache_1", - } - ) - prompt_affinity = proxy_module._sticky_key_for_responses_request( - prompt_payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - prompt_key = proxy_module._make_http_bridge_session_key( - prompt_payload, - headers={}, - affinity=prompt_affinity, - api_key=None, - request_id="req_prompt", - ) - prompt_session = await service._get_or_create_http_bridge_session( - prompt_key, - headers={}, - affinity=prompt_affinity, - api_key=None, - request_model=prompt_payload.model, - idle_ttl_seconds=120.0, - max_sessions=2, - ) - prompt_session.last_used_at = time.monotonic() - 5.0 - - next_payload = proxy_module.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "next", - "input": [{"role": "user", "content": "next"}], - "prompt_cache_key": "prompt_cache_2", - } - ) - next_affinity = proxy_module._sticky_key_for_responses_request( - next_payload, + affinity = proxy_module._sticky_key_for_responses_request( + payload, {}, codex_session_affinity=False, - openai_cache_affinity=True, + openai_cache_affinity=False, openai_cache_affinity_max_age_seconds=300, sticky_threads_enabled=False, api_key=None, ) - next_key = proxy_module._make_http_bridge_session_key( - next_payload, + key = proxy_module._make_http_bridge_session_key( + payload, headers={}, - affinity=next_affinity, + affinity=affinity, api_key=None, - request_id="req_prompt_2", + request_id="req_owner_unstable", ) - created = await service._get_or_create_http_bridge_session( - next_key, + session = await service._get_or_create_http_bridge_session( + key, headers={}, - affinity=next_affinity, + affinity=affinity, api_key=None, - request_model=next_payload.model, + request_model=payload.model, idle_ttl_seconds=120.0, - max_sessions=2, + max_sessions=8, ) - async with service._http_bridge_lock: - assert codex_key in service._http_bridge_sessions - assert prompt_key not in service._http_bridge_sessions - assert next_key in service._http_bridge_sessions - assert created.key == next_key + assert session.key.affinity_kind == "request" @pytest.mark.asyncio -async def test_get_or_create_http_bridge_session_honors_passed_prompt_cache_idle_ttl( +async def test_v1_responses_http_bridge_reconnect_uses_last_upstream_turn_state( async_client, app_instance, monkeypatch, ): - _install_bridge_settings_with_limits( - monkeypatch, - enabled=True, - prompt_cache_idle_ttl_seconds=1800.0, + _install_bridge_settings_with_limits(monkeypatch, enabled=True) + account_id = await _import_account( + async_client, + "acc_http_bridge_upstream_turn", + "http-bridge-upstream-turn@example.com", ) - account_id = await _import_account(async_client, "acc_prompt_ttl", "prompt-ttl@example.com") account = await _get_account(account_id) service = get_proxy_service_for_app(app_instance) - fake_upstream = _FakeBridgeUpstreamWebSocket() - payload = proxy_module.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "hi", - "input": [{"role": "user", "content": "hi"}], - "prompt_cache_key": "prompt-cache-ttl-test", - } - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={}, - affinity=affinity, - api_key=None, - request_id="req_prompt_ttl", - ) - cached_settings = await proxy_module.get_settings_cache().get() - monkeypatch.setattr( - proxy_module, - "get_settings_cache", - lambda: _SettingsCache( - _make_dashboard_settings( - prefer_earlier_reset_accounts=cached_settings.prefer_earlier_reset_accounts, - gateway_safe_mode=cached_settings.http_responses_session_bridge_gateway_safe_mode, - prompt_cache_idle_ttl_seconds=3600, - ) - ), - ) + connect_headers_seen: list[dict[str, str]] = [] + upstreams = [ + _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_1"), + _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_2"), + _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_3"), + ] async def fake_select_account_with_budget( self, @@ -7238,47 +3612,89 @@ async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_se del self, force, timeout_seconds return target - async def fake_open_upstream_websocket_with_budget(self, account, headers, *, timeout_seconds): - del self, account, headers, timeout_seconds - return fake_upstream + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del access_token, account_id_header, base_url, session + connect_headers_seen.append(dict(headers)) + return upstreams.pop(0) monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr( - proxy_module.ProxyService, - "_open_upstream_websocket_with_budget", - fake_open_upstream_websocket_with_budget, - ) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - session = await service._get_or_create_http_bridge_session( + payload = proxy_module.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} + ) + affinity = proxy_module._sticky_key_for_responses_request( + payload, + {"x-codex-turn-state": "local_turn_state"}, + codex_session_affinity=True, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + key = proxy_module._make_http_bridge_session_key( + payload, + headers={"x-codex-turn-state": "local_turn_state"}, + affinity=affinity, + api_key=None, + request_id="req_turn_state", + ) + bridge_session = await service._get_or_create_http_bridge_session( key, - headers={}, + headers={"x-codex-turn-state": "local_turn_state"}, affinity=affinity, api_key=None, request_model=payload.model, - idle_ttl_seconds=proxy_module._effective_http_bridge_idle_ttl_seconds( - affinity=affinity, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=900.0, - prompt_cache_idle_ttl_seconds=1800.0, - ), - max_sessions=32, + idle_ttl_seconds=120.0, + max_sessions=8, ) - assert session.idle_ttl_seconds == 1800.0 - await service._close_http_bridge_session(session) + request_state = proxy_module._WebSocketRequestState( + request_id="req-turn-state-reconnect", + model=payload.model, + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + response_create_gate_acquired=True, + request_text=json.dumps({"type": "response.create", "model": "gpt-5.4", "input": []}), + ) + await service._reconnect_http_bridge_session(bridge_session, request_state=request_state) + + assert connect_headers_seen[0]["x-codex-turn-state"] == "local_turn_state" + assert connect_headers_seen[1]["x-codex-turn-state"] == "upstream_turn_state_1" + assert bridge_session.upstream_turn_state == "upstream_turn_state_2" @pytest.mark.asyncio -async def test_v1_responses_http_bridge_reuses_upstream_websocket_and_preserves_previous_response_id( +async def test_v1_responses_http_bridge_session_id_reconnect_keeps_upstream_turn_state( async_client, + app_instance, monkeypatch, ): - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account(async_client, "acc_http_bridge_reuse", "http-bridge-reuse@example.com") + _install_bridge_settings_with_limits(monkeypatch, enabled=True) + account_id = await _import_account( + async_client, + "acc_http_bridge_session_reconnect", + "http-bridge-session-reconnect@example.com", + ) account = await _get_account(account_id) - fake_upstream = _FakeBridgeUpstreamWebSocket() - connect_calls: list[tuple[str | None, str | None]] = [] + service = get_proxy_service_for_app(app_instance) + connect_headers_seen: list[dict[str, str]] = [] + upstreams = [ + _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_1"), + _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_2"), + _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_3"), + ] async def fake_select_account_with_budget( self, @@ -7330,111 +3746,124 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, base_url, session - connect_calls.append((account_id, account_id_header)) - return fake_upstream - - async def fail_legacy_stream(*args, **kwargs): - raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + del access_token, account_id_header, base_url, session + connect_headers_seen.append(dict(headers)) + return upstreams.pop(0) monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - payload = { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-thread-1", - "client_metadata": { - "keep": "yes", - "x-codex-installation-id": "client-spoofed-installation-id", - }, - } - first = await async_client.post( - "/v1/responses", - json=payload, - headers={"x-codex-window-id": "parent-thread:0"}, + payload = proxy_module.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} ) - assert first.status_code == 200 - first_body = first.json() - - second = await async_client.post( - "/v1/responses", - json={**payload, "previous_response_id": first_body["id"]}, - headers={ - "x-openai-subagent": "collab_spawn", - "x-codex-parent-thread-id": "parent-thread", - "x-codex-window-id": "child-thread:0", - }, + headers = {"session_id": "session_http_bridge_1"} + affinity = proxy_module._sticky_key_for_responses_request( + payload, + headers, + codex_session_affinity=True, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, ) - assert second.status_code == 200 - second_body = second.json() - - assert first_body["id"] == "resp_bridge_1" - assert second_body["id"] == "resp_bridge_2" - assert connect_calls == [(account_id, account.chatgpt_account_id)] - assert len(fake_upstream.sent_text) == 2 - first_upstream_payload = json.loads(fake_upstream.sent_text[0]) - assert "tools" not in first_upstream_payload - assert first_upstream_payload["client_metadata"]["keep"] == "yes" - assert first_upstream_payload["client_metadata"]["x-codex-installation-id"] == account.codex_installation_id - assert first_upstream_payload["client_metadata"]["x-codex-installation-id"] != "client-spoofed-installation-id" - assert first_upstream_payload["client_metadata"]["x-codex-window-id"] == "parent-thread:0" - assert "x-openai-subagent" not in first_upstream_payload["client_metadata"] - assert "x-codex-parent-thread-id" not in first_upstream_payload["client_metadata"] - second_upstream_payload = json.loads(fake_upstream.sent_text[1]) - assert second_upstream_payload["previous_response_id"] == "resp_bridge_1" - assert second_upstream_payload["client_metadata"]["x-openai-subagent"] == "collab_spawn" - assert second_upstream_payload["client_metadata"]["x-codex-parent-thread-id"] == "parent-thread" - assert second_upstream_payload["client_metadata"]["x-codex-window-id"] == "child-thread:0" + key = proxy_module._make_http_bridge_session_key( + payload, + headers=headers, + affinity=affinity, + api_key=None, + request_id="req_session_turn_state", + ) + bridge_session = await service._get_or_create_http_bridge_session( + key, + headers=headers, + affinity=affinity, + api_key=None, + request_model=payload.model, + idle_ttl_seconds=120.0, + max_sessions=8, + ) + await service._register_http_bridge_turn_state(bridge_session, "http_turn_alias_session") + + request_state = proxy_module._WebSocketRequestState( + request_id="req-session-turn-state-reconnect", + model=payload.model, + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + response_create_gate_acquired=True, + request_text=json.dumps({"type": "response.create", "model": "gpt-5.4", "input": []}), + ) + await service._reconnect_http_bridge_session(bridge_session, request_state=request_state) + + assert connect_headers_seen[0]["session_id"] == "session_http_bridge_1" + assert "x-codex-turn-state" not in connect_headers_seen[0] + assert connect_headers_seen[1]["x-codex-turn-state"] == "upstream_turn_state_1" + assert bridge_session.downstream_turn_state == "http_turn_alias_session" + assert bridge_session.upstream_turn_state == "upstream_turn_state_2" @pytest.mark.asyncio -async def test_v1_responses_http_bridge_reuses_quota_admitted_spark_then_rejects_current_plan_change( +async def test_v1_responses_http_bridge_reconnect_uses_refreshed_api_key_assignments_for_reused_session( async_client, app_instance, monkeypatch, ): - _install_bridge_settings(monkeypatch, enabled=True) - raw_account_id = "acc_http_bridge_spark_catalog_omission" + _install_bridge_settings_with_limits(monkeypatch, enabled=True) account_id = await _import_account( async_client, - raw_account_id, - "http-bridge-spark-catalog-omission@example.com", - plan_type="pro", + "acc_http_bridge_assignment_refresh", + "http-bridge-assignment-refresh@example.com", ) account = await _get_account(account_id) - async with SessionLocal() as session: - additional_usage = AdditionalUsageRepository(session) - await additional_usage.add_entry( - account_id=account_id, - limit_name="GPT-5.3-Codex-Spark", - metered_feature="codex_bengalfox", - window="primary", - used_percent=0.0, - reset_at=None, - window_minutes=300, - recorded_at=utcnow(), - ) + service = get_proxy_service_for_app(app_instance) + selection_assigned_account_ids: list[list[str]] = [] + upstreams = [ + _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_1"), + _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_2"), + _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_3"), + ] - registry = ModelRegistry(ttl_seconds=60.0) - spark_model = replace( - registry.get_models_with_fallback()["gpt-5.3-codex-spark"], - raw={ - "service_tiers": [{"slug": "priority"}], - "additional_speed_tiers": ["fast"], - "default_service_tier": "priority", - }, - ) - await registry.update( - {"pro": [spark_model]}, - per_account_results={account_id: ("pro", [])}, - active_account_plans={account_id: "pro"}, - ) - fake_upstream = _FakeBridgeUpstreamWebSocket() - connect_calls: list[tuple[str | None, str | None]] = [] + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + ) + selection_assigned_account_ids.append(list(api_key.assigned_account_ids if api_key is not None else [])) + return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds @@ -7448,157 +3877,134 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, base_url, session - connect_calls.append((account_id, account_id_header)) - return fake_upstream - - async def fail_legacy_stream(*args, **kwargs): - raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + del headers, access_token, account_id_header, base_url, session + return upstreams.pop(0) - monkeypatch.setattr("app.modules.proxy.load_balancer.get_model_registry", lambda: registry) - monkeypatch.setattr("app.modules.proxy._service.support.get_model_registry", lambda: registry) + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - - payload = { - "model": "gpt-5.3-codex-spark", - "service_tier": " Priority ", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-spark-catalog-omission", - } - first = await async_client.post("/v1/responses", json=payload) - assert first.status_code == 200 - first_body = first.json() - second = await async_client.post( - "/v1/responses", - json={**payload, "previous_response_id": first_body["id"]}, + stale_api_key = _make_api_key_data(key_id="key_http_bridge_assignments", assigned_account_ids=["acc-stale"]) + refreshed_api_key = _make_api_key_data( + key_id="key_http_bridge_assignments", + assigned_account_ids=["acc-refreshed"], ) - assert second.status_code == 200 - second_body = second.json() - - await registry.update( - {"pro": [spark_model]}, - per_account_results={account_id: ("plus", [])}, - active_account_plans={account_id: "plus"}, + payload = proxy_module.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} ) - snapshot = registry.get_snapshot() - assert snapshot is not None - assert snapshot.account_plans[account_id] == "plus" - - rejected = await async_client.post( - "/v1/responses", - json={**payload, "previous_response_id": second_body["id"]}, + affinity = proxy_module._sticky_key_for_responses_request( + payload, + headers={"session_id": "session_http_bridge_assignment_refresh"}, + codex_session_affinity=True, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=stale_api_key, ) - assert rejected.status_code == 502, rejected.text - assert rejected.json()["error"] == { - "message": "Upstream websocket closed before response.completed", - "type": "server_error", - "code": "stream_incomplete", - } - - assert connect_calls == [(account_id, account.chatgpt_account_id)] - assert len(fake_upstream.sent_text) == 2 - service = get_proxy_service_for_app(app_instance) - bridge_key = proxy_module._HTTPBridgeSessionKey( - "prompt_cache", - "http-bridge-spark-catalog-omission", - None, + key = proxy_module._make_http_bridge_session_key( + payload, + headers={"session_id": "session_http_bridge_assignment_refresh"}, + affinity=affinity, + api_key=stale_api_key, + request_id="req_assignment_refresh", ) - bridge_session = service._http_bridge_sessions[bridge_key] - assert bridge_session.catalog_omission_quota_admission == CatalogOmissionQuotaAdmission( - normalized_model="gpt-5.3-codex-spark", - canonical_quota_key="codex_spark", - normalized_effective_service_tier="priority", + bridge_session = await service._get_or_create_http_bridge_session( + key, + headers={"session_id": "session_http_bridge_assignment_refresh"}, + affinity=affinity, + api_key=stale_api_key, + request_model=payload.model, + idle_ttl_seconds=120.0, + max_sessions=8, ) - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_forks_incompatible_prompt_cache_waiter_without_retiring_creator( - async_client, - app_instance, - monkeypatch, -): - _install_bridge_settings_with_limits( - monkeypatch, - enabled=True, - admission_wait_timeout_seconds=1.0, - ) - account_id = await _import_account( - async_client, - "acc_http_bridge_incompatible_prompt_cache_waiter", - "http-bridge-incompatible-prompt-cache-waiter@example.com", - plan_type="pro", + reused_session = await service._get_or_create_http_bridge_session( + key, + headers={"session_id": "session_http_bridge_assignment_refresh"}, + affinity=affinity, + api_key=refreshed_api_key, + request_model=payload.model, + idle_ttl_seconds=120.0, + max_sessions=8, ) - account = await _get_account(account_id) - service = get_proxy_service_for_app(app_instance) - - class Registry: - def get_snapshot(self): - return SimpleNamespace(account_plans={account_id: "pro"}) - - def account_ids_for_model(self, model: str) -> frozenset[str]: - assert model == "gpt-5.3-codex-spark" - return frozenset() - - def plan_types_for_model(self, model: str) -> frozenset[str]: - assert model == "gpt-5.3-codex-spark" - return frozenset({"pro"}) - - def account_ids_for_model_service_tier(self, model: str, service_tier: str) -> frozenset[str]: - assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") - return frozenset() + assert reused_session is not bridge_session + assert bridge_session.closed is True + assert reused_session.api_key == refreshed_api_key - def plan_types_for_model_service_tier(self, model: str, service_tier: str) -> frozenset[str]: - assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") - return frozenset({"pro"}) + request_state = proxy_module._WebSocketRequestState( + request_id="req-assignment-refresh-reconnect", + model=payload.model, + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + api_key=refreshed_api_key, + request_text=json.dumps({"type": "response.create", "model": "gpt-5.4", "input": []}), + ) + await service._reconnect_http_bridge_session(reused_session, request_state=request_state) - class DelayedUpstream(_FakeBridgeUpstreamWebSocket): - def __init__(self) -> None: - super().__init__() - self.request_started = asyncio.Event() - self.release_response = asyncio.Event() + assert selection_assigned_account_ids == [["acc-stale"], ["acc-refreshed"], ["acc-refreshed"]] - async def send_text(self, text: str) -> None: - self.request_started.set() - await _wait_for_event(self.release_response) - await super().send_text(text) + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_reconnect_fails_when_reader_cancel_times_out( + async_client, + app_instance, + monkeypatch, +): + _install_bridge_settings_with_limits(monkeypatch, enabled=True) + account_id = await _import_account( + async_client, + "acc_http_bridge_reconnect_cancel_timeout", + "http-bridge-reconnect-cancel-timeout@example.com", + ) + account = await _get_account(account_id) + service = get_proxy_service_for_app(app_instance) + upstreams = [_FakeBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] async def fake_select_account_with_budget( self, deadline, *, - service_tier=None, - **kwargs, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, ): - del self, deadline, kwargs - normalized_service_tier = ( - None - if service_tier is None or service_tier.strip().lower() in {"auto", "default"} - else service_tier.strip().lower() - ) - return AccountSelection( - account=account, - error_message=None, - error_code=None, - catalog_omission_quota_admission=CatalogOmissionQuotaAdmission( - normalized_model="gpt-5.3-codex-spark", - canonical_quota_key="codex_spark", - normalized_effective_service_tier=normalized_service_tier, - ), + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, ) + return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds return target - first_connect_started = asyncio.Event() - release_first_connect = asyncio.Event() - second_connect_started = asyncio.Event() - upstreams: list[DelayedUpstream] = [] - async def fake_connect_responses_websocket( headers, access_token, @@ -7608,185 +4014,145 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - upstream = DelayedUpstream() - upstreams.append(upstream) - if len(upstreams) == 1: - first_connect_started.set() - await _wait_for_event(release_first_connect) - else: - second_connect_started.set() - return upstream - - created_sessions: list[proxy_module._HTTPBridgeSession] = [] - second_session_created = asyncio.Event() - create_session = service._create_http_bridge_session - - async def capture_created_session(key, **kwargs): - created_session = await create_session(key, **kwargs) - created_sessions.append(created_session) - if len(created_sessions) == 2: - second_session_created.set() - return created_session - - scheduled_sessions: list[proxy_module._HTTPBridgeSession] = [] - schedule_session_closes = service._schedule_http_bridge_session_closes - - def capture_scheduled_sessions(sessions, *, reason): - scheduled_sessions.extend(sessions) - schedule_session_closes(sessions, reason=reason) - - async def fail_legacy_stream(*args, **kwargs): - raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + return upstreams.pop(0) - monkeypatch.setattr(proxy_support, "get_model_registry", lambda: Registry()) monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - monkeypatch.setattr(service, "_create_http_bridge_session", capture_created_session) - monkeypatch.setattr(service, "_schedule_http_bridge_session_closes", capture_scheduled_sessions) - - payload = { - "model": "gpt-5.3-codex-spark", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-incompatible-prompt-cache-waiter", - } - first_task = asyncio.create_task(async_client.post("/v1/responses", json=payload)) - second_task = None - try: - await _wait_for_event(first_connect_started) - second_task = asyncio.create_task( - async_client.post( - "/v1/responses", - json={**payload, "input": "hello priority", "service_tier": "priority"}, - ) - ) - for _ in range(10): - await asyncio.sleep(0) - release_first_connect.set() - await _wait_for_event(second_connect_started) - await _wait_for_event(second_session_created) - assert len(created_sessions) == 2 - creator_session, waiter_session = created_sessions + payload = proxy_module.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} + ) + affinity = proxy_module._sticky_key_for_responses_request( + payload, + {"x-codex-turn-state": "timeout_turn_state"}, + codex_session_affinity=True, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + key = proxy_module._make_http_bridge_session_key( + payload, + headers={"x-codex-turn-state": "timeout_turn_state"}, + affinity=affinity, + api_key=None, + request_id="req_timeout_turn_state", + ) + bridge_session = await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-turn-state": "timeout_turn_state"}, + affinity=affinity, + api_key=None, + request_model=payload.model, + idle_ttl_seconds=120.0, + max_sessions=8, + ) + original_upstream = bridge_session.upstream - assert len(upstreams) == 2 - assert creator_session is not waiter_session - assert creator_session.upstream is upstreams[0] - assert waiter_session.upstream is upstreams[1] - assert creator_session.closed is False - assert service._http_bridge_sessions.get(creator_session.key) is creator_session - assert creator_session not in scheduled_sessions - assert waiter_session.key.affinity_kind == "internal_request_parallel" + blocker = asyncio.Event() - await _wait_for_event(upstreams[0].request_started) - await _wait_for_event(upstreams[1].request_started) - for upstream in upstreams: - upstream.release_response.set() - first_response, second_response = await asyncio.wait_for( - asyncio.gather(first_task, second_task), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, - ) - finally: - release_first_connect.set() - for upstream in upstreams: - upstream.release_response.set() - pending_tasks = [task for task in (first_task, second_task) if task is not None] - await asyncio.gather(*pending_tasks, return_exceptions=True) + async def blocking_reader_task() -> None: + await _wait_for_event(blocker) - assert first_response.status_code == 200 - assert second_response.status_code == 200 - assert first_response.json()["output"][0]["content"][0]["text"] == "OK" - assert second_response.json()["output"][0]["content"][0]["text"] == "OK" - assert creator_session.closed is False + original_reader = bridge_session.upstream_reader + assert original_reader is not None + original_reader.cancel() + with contextlib.suppress(asyncio.CancelledError): + await original_reader + blocking_reader = asyncio.create_task(blocking_reader_task()) + bridge_session.upstream_reader = blocking_reader + async def fake_await_cancelled_task(task, *, timeout_seconds=1.0, label, cleanup_tasks=None): + del task, timeout_seconds, label, cleanup_tasks + return False -@pytest.mark.asyncio -async def test_forwarded_priority_prompt_cache_mismatch_forks_on_canonical_owner( - async_client, - app_instance, - monkeypatch, -): - from app.core.middleware import request_id as request_id_middleware_module - from app.modules.proxy import api as proxy_api_module - from app.modules.proxy.http_bridge_forwarding import HTTPBridgeForwardContext, build_owner_forward_headers + monkeypatch.setattr(proxy_module, "_await_cancelled_task", fake_await_cancelled_task) - owner_settings = _make_app_settings( - enabled=True, - instance_id="instance-a", - instance_ring=["instance-a", "instance-b"], - ) - origin_settings = _make_app_settings( - enabled=True, - instance_id="instance-b", - instance_ring=["instance-a", "instance-b"], - ) - _install_proxy_settings( - monkeypatch, - app_settings=owner_settings, - dashboard_settings=_make_dashboard_settings(), - ) - monkeypatch.setattr(proxy_api_module, "get_settings", lambda: owner_settings) - account_id = await _import_account( - async_client, - "acc_http_bridge_forwarded_prompt_mismatch", - "http-bridge-forwarded-prompt-mismatch@example.com", - plan_type="pro", + request_state = proxy_module._WebSocketRequestState( + request_id="req-timeout-reconnect", + model=payload.model, + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + response_create_gate_acquired=True, + request_text=json.dumps({"type": "response.create", "model": "gpt-5.4", "input": []}), ) - account = await _get_account(account_id) - service = get_proxy_service_for_app(app_instance) - - class Registry: - def get_snapshot(self): - return SimpleNamespace(account_plans={account_id: "pro"}) - def account_ids_for_model(self, model: str) -> frozenset[str]: - assert model == "gpt-5.3-codex-spark" - return frozenset() + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session( + bridge_session, + request_state=request_state, + restart_reader=True, + ) - def plan_types_for_model(self, model: str) -> frozenset[str]: - assert model == "gpt-5.3-codex-spark" - return frozenset({"pro"}) + error_payload = exc_info.value.payload["error"] + assert exc_info.value.status_code == 502 + assert error_payload.get("code") == "upstream_unavailable" + assert "reader did not shut down cleanly" in (error_payload.get("message") or "") + assert bridge_session.closed is True + assert bridge_session.upstream is original_upstream + blocking_reader.cancel() + with contextlib.suppress(asyncio.CancelledError): + await blocking_reader - def account_ids_for_model_service_tier(self, model: str, service_tier: str) -> frozenset[str]: - assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") - return frozenset() - def plan_types_for_model_service_tier(self, model: str, service_tier: str) -> frozenset[str]: - assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") - return frozenset({"pro"}) +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_prefers_evicting_prompt_cache_session_before_codex_session( + async_client, + app_instance, + monkeypatch, +): + _install_bridge_settings_with_limits(monkeypatch, enabled=True, max_sessions=2, codex_idle_ttl_seconds=600.0) + account_id = await _import_account(async_client, "acc_http_bridge_evict_pref", "http-bridge-evict-pref@example.com") + account = await _get_account(account_id) + service = get_proxy_service_for_app(app_instance) + upstreams = [_FakeBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] async def fake_select_account_with_budget( self, deadline, *, - service_tier=None, - **kwargs, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, ): - del self, deadline, kwargs - normalized_service_tier = ( - None - if service_tier is None or service_tier.strip().lower() in {"auto", "default"} - else service_tier.strip().lower() - ) - return AccountSelection( - account=account, - error_message=None, - error_code=None, - catalog_omission_quota_admission=CatalogOmissionQuotaAdmission( - normalized_model="gpt-5.3-codex-spark", - canonical_quota_key="codex_spark", - normalized_effective_service_tier=normalized_service_tier, - ), + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, ) + return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds return target - upstreams: list[_FakeBridgeUpstreamWebSocket] = [] - async def fake_connect_responses_websocket( headers, access_token, @@ -7796,302 +4162,255 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - upstream = _FakeBridgeUpstreamWebSocket() - upstreams.append(upstream) - return upstream - - async def fail_legacy_stream(*args, **kwargs): - del args, kwargs - raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") - - class Ring: - async def list_active(self, *, require_endpoint: bool = False) -> list[str]: - assert require_endpoint is True - return ["instance-a", "instance-b"] - - async def resolve_endpoint(self, instance_id: str) -> str: - return f"http://{instance_id}" + return upstreams.pop(0) - monkeypatch.setattr(proxy_support, "get_model_registry", lambda: Registry()) monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - monkeypatch.setattr(request_id_middleware_module, "uuid4", lambda: "forwarded-request-scope") - ring = cast(Any, Ring()) - original_ring = service._ring_membership - service._ring_membership = ring - canonical_key = proxy_module._HTTPBridgeSessionKey("prompt_cache", "forwarded-prompt-1", None) - fork_key = proxy_module._HTTPBridgeSessionKey( - "internal_request_parallel", - "95427abf10b750a60b5a5d3528343e28c89e8c3a3e428ae51df95534cbf803b3", - None, + payload = proxy_module.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} ) - assert await proxy_module._http_bridge_owner_instance(canonical_key, owner_settings, ring) == "instance-a" - assert await proxy_module._http_bridge_owner_instance(canonical_key, origin_settings, ring) == "instance-a" - assert await proxy_module._http_bridge_owner_instance(fork_key, owner_settings, ring) == "instance-b" - - scheduled_sessions: list[proxy_module._HTTPBridgeSession] = [] - schedule_session_closes = service._schedule_http_bridge_session_closes - def capture_scheduled_sessions(sessions, *, reason): - scheduled_sessions.extend(sessions) - schedule_session_closes(sessions, reason=reason) + codex_affinity = proxy_module._sticky_key_for_responses_request( + payload, + {"x-codex-turn-state": "turn_state_1"}, + codex_session_affinity=True, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + codex_key = proxy_module._make_http_bridge_session_key( + payload, + headers={"x-codex-turn-state": "turn_state_1"}, + affinity=codex_affinity, + api_key=None, + request_id="req_codex", + ) + codex_session = await service._get_or_create_http_bridge_session( + codex_key, + headers={"x-codex-turn-state": "turn_state_1"}, + affinity=codex_affinity, + api_key=None, + request_model=payload.model, + idle_ttl_seconds=120.0, + max_sessions=2, + ) + codex_session.last_used_at = time.monotonic() - 50.0 - monkeypatch.setattr(service, "_schedule_http_bridge_session_closes", capture_scheduled_sessions) - creator_payload = { - "model": "gpt-5.3-codex-spark", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": canonical_key.affinity_key, - } - priority_payload = proxy_module.ResponsesRequest.model_validate( - {**creator_payload, "input": "hello priority", "service_tier": "priority"} + prompt_payload = proxy_module.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": [{"role": "user", "content": "hi"}], + "prompt_cache_key": "prompt_cache_1", + } ) - forward_context = HTTPBridgeForwardContext( - origin_instance="instance-b", - target_instance="instance-a", + prompt_affinity = proxy_module._sticky_key_for_responses_request( + prompt_payload, + {}, codex_session_affinity=False, - downstream_turn_state=None, - original_request_unanchored=False, - original_affinity_kind=canonical_key.affinity_kind, - original_affinity_key=canonical_key.affinity_key, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, ) - forward_headers = build_owner_forward_headers( - headers={"x-request-id": "forwarded-priority-request"}, - payload=priority_payload, - context=forward_context, + prompt_key = proxy_module._make_http_bridge_session_key( + prompt_payload, + headers={}, + affinity=prompt_affinity, + api_key=None, + request_id="req_prompt", ) + prompt_session = await service._get_or_create_http_bridge_session( + prompt_key, + headers={}, + affinity=prompt_affinity, + api_key=None, + request_model=prompt_payload.model, + idle_ttl_seconds=120.0, + max_sessions=2, + ) + prompt_session.last_used_at = time.monotonic() - 5.0 - try: - creator_response = await async_client.post( - "/v1/responses", - json=creator_payload, - headers={"x-request-id": "creator-request"}, - ) - assert creator_response.status_code == 200, creator_response.text - creator_session = service._http_bridge_sessions[canonical_key] - creator_response_ids = set(creator_session.previous_response_ids) - - priority_response = await async_client.post( - "/internal/bridge/responses", - json=priority_payload.model_dump_for_forwarding(), - headers=forward_headers, - ) + next_payload = proxy_module.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "next", + "input": [{"role": "user", "content": "next"}], + "prompt_cache_key": "prompt_cache_2", + } + ) + next_affinity = proxy_module._sticky_key_for_responses_request( + next_payload, + {}, + codex_session_affinity=False, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + next_key = proxy_module._make_http_bridge_session_key( + next_payload, + headers={}, + affinity=next_affinity, + api_key=None, + request_id="req_prompt_2", + ) - assert priority_response.status_code == 200, priority_response.text - assert creator_response.json()["output"][0]["content"][0]["text"] == "OK" - assert '"type":"response.completed"' in priority_response.text - assert '"text":"OK"' in priority_response.text - assert len(upstreams) == 2 - assert creator_session.upstream is upstreams[0] - assert service._http_bridge_sessions[fork_key].upstream is upstreams[1] - assert creator_session.closed is False - assert service._http_bridge_sessions[canonical_key] is creator_session - assert creator_session not in scheduled_sessions - assert creator_session.request_service_tier is None - assert service._http_bridge_sessions[fork_key].request_service_tier == "priority" - assert creator_response_ids <= creator_session.previous_response_ids - finally: - service._ring_membership = original_ring + created = await service._get_or_create_http_bridge_session( + next_key, + headers={}, + affinity=next_affinity, + api_key=None, + request_model=next_payload.model, + idle_ttl_seconds=120.0, + max_sessions=2, + ) + + async with service._http_bridge_lock: + assert codex_key in service._http_bridge_sessions + assert prompt_key not in service._http_bridge_sessions + assert next_key in service._http_bridge_sessions + assert created.key == next_key @pytest.mark.asyncio -async def test_forwarded_recovery_uses_durable_owner_and_strips_stale_affinity( +async def test_get_or_create_http_bridge_session_honors_passed_prompt_cache_idle_ttl( async_client, app_instance, monkeypatch, ): - from app.modules.proxy import api as proxy_api_module - from app.modules.proxy.continuity import make_http_bridge_account_neutral_replay_key - from app.modules.proxy.http_bridge_forwarding import HTTPBridgeForwardContext, build_owner_forward_headers - - target_settings = _make_app_settings(enabled=True, instance_id="instance-b") - _install_proxy_settings( + _install_bridge_settings_with_limits( monkeypatch, - app_settings=target_settings, - dashboard_settings=_make_dashboard_settings(), - ) - monkeypatch.setattr(proxy_api_module, "get_settings", lambda: target_settings) - alternate_account_id = await _import_account( - async_client, - "acc_http_bridge_forwarded_recovery_alternate", - "http-bridge-forwarded-recovery-alternate@example.com", - ) - account_id = await _import_account( - async_client, - "acc_http_bridge_forwarded_recovery", - "http-bridge-forwarded-recovery@example.com", + enabled=True, + prompt_cache_idle_ttl_seconds=1800.0, ) - alternate_account = await _get_account(alternate_account_id) + account_id = await _import_account(async_client, "acc_prompt_ttl", "prompt-ttl@example.com") account = await _get_account(account_id) - chatgpt_account_id = cast(str, account.chatgpt_account_id) service = get_proxy_service_for_app(app_instance) - recovery_kind, recovery_key = make_http_bridge_account_neutral_replay_key("forwarded-recovery") - recovered_turn_state = "http_turn_forwarded_recovery" - recovered_response_id = "resp_forwarded_recovery" - durable_lookup = await service._durable_bridge.claim_live_session( - session_key_kind=recovery_kind, - session_key_value=recovery_key, - api_key_id=None, - instance_id=target_settings.http_responses_session_bridge_instance_id, - owner_process_epoch="test-process", - lease_ttl_seconds=60.0, - account_id=account.id, - model="gpt-5.1", - service_tier=None, - latest_turn_state=recovered_turn_state, - latest_response_id=recovered_response_id, - allow_takeover=True, + fake_upstream = _FakeBridgeUpstreamWebSocket() + payload = proxy_module.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": [{"role": "user", "content": "hi"}], + "prompt_cache_key": "prompt-cache-ttl-test", + } ) - await service._durable_bridge.register_turn_state( - session_id=durable_lookup.session_id, - api_key_id=None, - instance_id=target_settings.http_responses_session_bridge_instance_id, - owner_epoch=durable_lookup.owner_epoch, - turn_state=recovered_turn_state, - lease_ttl_seconds=60.0, + affinity = proxy_module._sticky_key_for_responses_request( + payload, + {}, + codex_session_affinity=False, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, ) - await service._durable_bridge.register_previous_response_id( - session_id=durable_lookup.session_id, - api_key_id=None, - instance_id=target_settings.http_responses_session_bridge_instance_id, - owner_epoch=durable_lookup.owner_epoch, - response_id=recovered_response_id, - lease_ttl_seconds=60.0, + key = proxy_module._make_http_bridge_session_key( + payload, + headers={}, + affinity=affinity, + api_key=None, + request_id="req_prompt_ttl", + ) + cached_settings = await proxy_module.get_settings_cache().get() + monkeypatch.setattr( + proxy_module, + "get_settings_cache", + lambda: _SettingsCache( + _make_dashboard_settings( + prefer_earlier_reset_accounts=cached_settings.prefer_earlier_reset_accounts, + gateway_safe_mode=cached_settings.http_responses_session_bridge_gateway_safe_mode, + prompt_cache_idle_ttl_seconds=3600, + ) + ), ) - lookup_request_targets = AsyncMock(wraps=service._durable_bridge.lookup_request_targets) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", lookup_request_targets) - - selection_calls: list[dict[str, object]] = [] - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline - selection_calls.append(dict(kwargs)) - if kwargs.get("preferred_account_id") is None: - return AccountSelection(account=alternate_account, error_message=None, error_code=None) - assert kwargs.get("preferred_account_id") == account.id - assert kwargs.get("preferred_account_is_continuity_owner") is True - assert kwargs.get("fallback_on_preferred_account_unavailable") is False + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + ) return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds return target - upstream = _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_forwarded_recovery") - connect_calls: list[tuple[dict[str, str], str]] = [] - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del access_token, base_url, session - connect_calls.append((dict(headers), account_id_header)) - return upstream - - async def fail_legacy_stream(*args, **kwargs): - del args, kwargs - raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + async def fake_open_upstream_websocket_with_budget(self, account, headers, *, timeout_seconds): + del self, account, headers, timeout_seconds + return fake_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - - payload = proxy_module.ResponsesRequest( - model="gpt-5.1", - instructions="Return exactly OK.", - input="continue on the recovered account", - previous_response_id=recovered_response_id, - ) - forward_context = HTTPBridgeForwardContext( - origin_instance="instance-a", - target_instance=target_settings.http_responses_session_bridge_instance_id, - codex_session_affinity=False, - downstream_turn_state=recovered_turn_state, - original_request_unanchored=True, - original_affinity_kind=recovery_kind, - original_affinity_key=recovery_key, - ) - forward_headers = build_owner_forward_headers( - headers={ - "session_id": "stale-session", - "session-id": "stale-session-dash", - "thread-id": "stale-thread", - "x-codex-conversation-id": "stale-conversation", - "x-codex-session-id": "stale-codex-session", - "x-codex-turn-state": "http_turn_stale", - "x-request-trace": "keep-me", - }, - payload=payload, - context=forward_context, + monkeypatch.setattr( + proxy_module.ProxyService, + "_open_upstream_websocket_with_budget", + fake_open_upstream_websocket_with_budget, ) - response = await asyncio.wait_for( - async_client.post( - "/internal/bridge/responses", - json=payload.model_dump_for_forwarding(), - headers=forward_headers, + session = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=affinity, + api_key=None, + request_model=payload.model, + idle_ttl_seconds=proxy_module._effective_http_bridge_idle_ttl_seconds( + affinity=affinity, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=900.0, + prompt_cache_idle_ttl_seconds=1800.0, ), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, + max_sessions=32, ) - assert response.status_code == 200, response.text - assert '"type":"response.completed"' in response.text - lookup_request_targets.assert_awaited_once_with( - session_key_kind=recovery_kind, - session_key_value=recovery_key, - api_key_id=None, - turn_state=recovered_turn_state, - session_header=None, - previous_response_id=recovered_response_id, - ) - assert len(selection_calls) == 1 - assert selection_calls[0]["preferred_account_id"] == account.id - assert selection_calls[0]["preferred_account_is_continuity_owner"] is True - assert selection_calls[0]["fallback_on_preferred_account_unavailable"] is False - assert len(connect_calls) == 1 - connect_headers, connected_account_id = connect_calls[0] - assert connected_account_id == chatgpt_account_id - normalized_connect_headers = {key.lower(): value for key, value in connect_headers.items()} - assert normalized_connect_headers["x-request-trace"] == "keep-me" - assert ( - not { - "session_id", - "session-id", - "thread-id", - "x-codex-conversation-id", - "x-codex-session-id", - "x-codex-turn-state", - } - & normalized_connect_headers.keys() - ) - assert json.loads(upstream.sent_text[0])["previous_response_id"] == recovered_response_id - recovery_session_key = proxy_module._HTTPBridgeSessionKey(recovery_kind, recovery_key, None) - recovery_session = service._http_bridge_sessions[recovery_session_key] - assert recovery_session.account.id == account.id - assert recovered_turn_state in recovery_session.downstream_turn_state_aliases + assert session.idle_ttl_seconds == 1800.0 + await service._close_http_bridge_session(session) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_injects_interrupted_custom_tool_output_on_followup( +async def test_v1_responses_http_bridge_reuses_upstream_websocket_and_preserves_previous_response_id( async_client, monkeypatch, ): _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_custom_interrupt", - "http-bridge-custom-interrupt@example.com", - ) + account_id = await _import_account(async_client, "acc_http_bridge_reuse", "http-bridge-reuse@example.com") account = await _get_account(account_id) - fake_upstream = _InterruptedCustomToolUpstreamWebSocket() + fake_upstream = _FakeBridgeUpstreamWebSocket() + connect_calls: list[tuple[str | None, str | None]] = [] async def fake_select_account_with_budget( self, @@ -8143,126 +4462,275 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, account_id_header, base_url, session + del headers, access_token, base_url, session + connect_calls.append((account_id, account_id_header)) return fake_upstream + async def fail_legacy_stream(*args, **kwargs): + raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - interrupted_user_message = { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "\nThe user interrupted the previous turn on purpose.\n", - } - ], + payload = { + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + "prompt_cache_key": "http-bridge-thread-1", + "client_metadata": { + "keep": "yes", + "x-codex-installation-id": "client-spoofed-installation-id", + }, } first = await async_client.post( "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Use the shell tool.", - "input": [{"role": "user", "content": [{"type": "input_text", "text": "run the shell tool"}]}], - "prompt_cache_key": "http-bridge-custom-interrupt-1", - }, + json=payload, + headers={"x-codex-window-id": "parent-thread:0"}, ) assert first.status_code == 200 first_body = first.json() - assert first_body["id"] == "resp_bridge_custom_1" second = await async_client.post( "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Use the shell tool.", - "previous_response_id": first_body["id"], - "input": [interrupted_user_message], - "prompt_cache_key": "http-bridge-custom-interrupt-1", + json={**payload, "previous_response_id": first_body["id"]}, + headers={ + "x-openai-subagent": "collab_spawn", + "x-codex-parent-thread-id": "parent-thread", + "x-codex-window-id": "child-thread:0", }, ) assert second.status_code == 200 - assert second.json()["id"] == "resp_bridge_custom_2" + second_body = second.json() + assert first_body["id"] == "resp_bridge_1" + assert second_body["id"] == "resp_bridge_2" + assert connect_calls == [(account_id, account.chatgpt_account_id)] assert len(fake_upstream.sent_text) == 2 + first_upstream_payload = json.loads(fake_upstream.sent_text[0]) + assert "tools" not in first_upstream_payload + assert first_upstream_payload["client_metadata"]["keep"] == "yes" + assert first_upstream_payload["client_metadata"]["x-codex-installation-id"] == account.codex_installation_id + assert first_upstream_payload["client_metadata"]["x-codex-installation-id"] != "client-spoofed-installation-id" + assert first_upstream_payload["client_metadata"]["x-codex-window-id"] == "parent-thread:0" + assert "x-openai-subagent" not in first_upstream_payload["client_metadata"] + assert "x-codex-parent-thread-id" not in first_upstream_payload["client_metadata"] second_upstream_payload = json.loads(fake_upstream.sent_text[1]) - assert second_upstream_payload["previous_response_id"] == "resp_bridge_custom_1" - interrupted_tool_output = ( - "Tool call was not executed because the previous turn was interrupted before tool output was available." + assert second_upstream_payload["previous_response_id"] == "resp_bridge_1" + assert second_upstream_payload["client_metadata"]["x-openai-subagent"] == "collab_spawn" + assert second_upstream_payload["client_metadata"]["x-codex-parent-thread-id"] == "parent-thread" + assert second_upstream_payload["client_metadata"]["x-codex-window-id"] == "child-thread:0" + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_reuses_quota_admitted_spark_then_rejects_current_plan_change( + async_client, + app_instance, + monkeypatch, +): + _install_bridge_settings(monkeypatch, enabled=True) + raw_account_id = "acc_http_bridge_spark_catalog_omission" + account_id = await _import_account( + async_client, + raw_account_id, + "http-bridge-spark-catalog-omission@example.com", + plan_type="pro", ) - assert second_upstream_payload["input"][0] == { - "type": "custom_tool_call_output", - "call_id": "call_custom_shell", - "output": interrupted_tool_output, + account = await _get_account(account_id) + async with SessionLocal() as session: + additional_usage = AdditionalUsageRepository(session) + await additional_usage.add_entry( + account_id=account_id, + limit_name="GPT-5.3-Codex-Spark", + metered_feature="codex_bengalfox", + window="primary", + used_percent=0.0, + reset_at=None, + window_minutes=300, + recorded_at=utcnow(), + ) + + registry = ModelRegistry(ttl_seconds=60.0) + spark_model = replace( + registry.get_models_with_fallback()["gpt-5.3-codex-spark"], + raw={ + "service_tiers": [{"slug": "priority"}], + "additional_speed_tiers": ["fast"], + "default_service_tier": "priority", + }, + ) + await registry.update( + {"pro": [spark_model]}, + per_account_results={account_id: ("pro", [])}, + active_account_plans={account_id: "pro"}, + ) + fake_upstream = _FakeBridgeUpstreamWebSocket() + connect_calls: list[tuple[str | None, str | None]] = [] + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, base_url, session + connect_calls.append((account_id, account_id_header)) + return fake_upstream + + async def fail_legacy_stream(*args, **kwargs): + raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + + monkeypatch.setattr("app.modules.proxy.load_balancer.get_model_registry", lambda: registry) + monkeypatch.setattr("app.modules.proxy._service.support.get_model_registry", lambda: registry) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) + + payload = { + "model": "gpt-5.3-codex-spark", + "service_tier": " Priority ", + "instructions": "Return exactly OK.", + "input": "hello", + "prompt_cache_key": "http-bridge-spark-catalog-omission", } - assert second_upstream_payload["input"][1] == interrupted_user_message + first = await async_client.post("/v1/responses", json=payload) + assert first.status_code == 200 + first_body = first.json() + + second = await async_client.post( + "/v1/responses", + json={**payload, "previous_response_id": first_body["id"]}, + ) + assert second.status_code == 200 + second_body = second.json() + + await registry.update( + {"pro": [spark_model]}, + per_account_results={account_id: ("plus", [])}, + active_account_plans={account_id: "plus"}, + ) + snapshot = registry.get_snapshot() + assert snapshot is not None + assert snapshot.account_plans[account_id] == "plus" + + rejected = await async_client.post( + "/v1/responses", + json={**payload, "previous_response_id": second_body["id"]}, + ) + assert rejected.status_code == 502, rejected.text + assert rejected.json()["error"] == { + "message": "Upstream websocket closed before response.completed", + "type": "server_error", + "code": "stream_incomplete", + } + + assert connect_calls == [(account_id, account.chatgpt_account_id)] + assert len(fake_upstream.sent_text) == 2 + service = get_proxy_service_for_app(app_instance) + bridge_key = proxy_module._HTTPBridgeSessionKey( + "prompt_cache", + "http-bridge-spark-catalog-omission", + None, + ) + bridge_session = service._http_bridge_sessions[bridge_key] + assert bridge_session.catalog_omission_quota_admission == CatalogOmissionQuotaAdmission( + normalized_model="gpt-5.3-codex-spark", + canonical_quota_key="codex_spark", + normalized_effective_service_tier="priority", + ) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_size_guard_covers_injected_interrupted_tool_outputs( +async def test_v1_responses_http_bridge_forks_incompatible_prompt_cache_waiter_without_retiring_creator( async_client, + app_instance, monkeypatch, - tmp_path, ): - _install_bridge_settings(monkeypatch, enabled=True) - monkeypatch.setattr(proxy_module, "_UPSTREAM_RESPONSE_CREATE_MAX_BYTES", 10_000_000) - monkeypatch.setattr(proxy_module, "_UPSTREAM_RESPONSE_CREATE_WARN_BYTES", 10_000_000) - monkeypatch.setattr(proxy_module, "_OVERSIZED_RESPONSE_CREATE_DUMP_DIR", tmp_path) + _install_bridge_settings_with_limits( + monkeypatch, + enabled=True, + admission_wait_timeout_seconds=1.0, + ) account_id = await _import_account( async_client, - "acc_http_bridge_custom_interrupt_size", - "http-bridge-custom-interrupt-size@example.com", + "acc_http_bridge_incompatible_prompt_cache_waiter", + "http-bridge-incompatible-prompt-cache-waiter@example.com", + plan_type="pro", ) account = await _get_account(account_id) - # Blank installation id makes the submit-time account-installation rewrite - # a no-op, so no later serialization step would re-run the size guard; - # the injection path itself must keep the request within the limit. - account.codex_installation_id = "" - fake_upstream = _InterruptedCustomToolUpstreamWebSocket() + service = get_proxy_service_for_app(app_instance) + + class Registry: + def get_snapshot(self): + return SimpleNamespace(account_plans={account_id: "pro"}) + + def account_ids_for_model(self, model: str) -> frozenset[str]: + assert model == "gpt-5.3-codex-spark" + return frozenset() + + def plan_types_for_model(self, model: str) -> frozenset[str]: + assert model == "gpt-5.3-codex-spark" + return frozenset({"pro"}) + + def account_ids_for_model_service_tier(self, model: str, service_tier: str) -> frozenset[str]: + assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") + return frozenset() + + def plan_types_for_model_service_tier(self, model: str, service_tier: str) -> frozenset[str]: + assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") + return frozenset({"pro"}) + + class DelayedUpstream(_FakeBridgeUpstreamWebSocket): + def __init__(self) -> None: + super().__init__() + self.request_started = asyncio.Event() + self.release_response = asyncio.Event() + + async def send_text(self, text: str) -> None: + self.request_started.set() + await _wait_for_event(self.release_response) + await super().send_text(text) async def fake_select_account_with_budget( self, deadline, *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, + service_tier=None, + **kwargs, ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, + del self, deadline, kwargs + normalized_service_tier = ( + None + if service_tier is None or service_tier.strip().lower() in {"auto", "default"} + else service_tier.strip().lower() + ) + return AccountSelection( + account=account, + error_message=None, + error_code=None, + catalog_omission_quota_admission=CatalogOmissionQuotaAdmission( + normalized_model="gpt-5.3-codex-spark", + canonical_quota_key="codex_spark", + normalized_effective_service_tier=normalized_service_tier, + ), ) - return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds return target + first_connect_started = asyncio.Event() + release_first_connect = asyncio.Event() + second_connect_started = asyncio.Event() + upstreams: list[DelayedUpstream] = [] + async def fake_connect_responses_websocket( headers, access_token, @@ -8272,125 +4740,185 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - return fake_upstream + upstream = DelayedUpstream() + upstreams.append(upstream) + if len(upstreams) == 1: + first_connect_started.set() + await _wait_for_event(release_first_connect) + else: + second_connect_started.set() + return upstream + + created_sessions: list[proxy_module._HTTPBridgeSession] = [] + second_session_created = asyncio.Event() + create_session = service._create_http_bridge_session + + async def capture_created_session(key, **kwargs): + created_session = await create_session(key, **kwargs) + created_sessions.append(created_session) + if len(created_sessions) == 2: + second_session_created.set() + return created_session + + scheduled_sessions: list[proxy_module._HTTPBridgeSession] = [] + schedule_session_closes = service._schedule_http_bridge_session_closes + + def capture_scheduled_sessions(sessions, *, reason): + scheduled_sessions.extend(sessions) + schedule_session_closes(sessions, reason=reason) + async def fail_legacy_stream(*args, **kwargs): + raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + + monkeypatch.setattr(proxy_support, "get_model_registry", lambda: Registry()) monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) + monkeypatch.setattr(service, "_create_http_bridge_session", capture_created_session) + monkeypatch.setattr(service, "_schedule_http_bridge_session_closes", capture_scheduled_sessions) - original_prepare = proxy_module.ProxyService._prepare_http_bridge_request - followup_cap_armed = False - - def capping_prepare(self, payload, headers, **kwargs): - nonlocal followup_cap_armed - request_state, text_data = original_prepare(self, payload, headers, **kwargs) - if not followup_cap_armed and '"previous_response_id":"resp_bridge_custom_1"' in text_data: - # The anchored follow-up fits the limit as sent by the client; - # prepending synthetic interrupted outputs pushes it over. - followup_cap_armed = True - proxy_module._UPSTREAM_RESPONSE_CREATE_MAX_BYTES = len(text_data.encode("utf-8")) + 100 - return request_state, text_data + payload = { + "model": "gpt-5.3-codex-spark", + "instructions": "Return exactly OK.", + "input": "hello", + "prompt_cache_key": "http-bridge-incompatible-prompt-cache-waiter", + } + first_task = asyncio.create_task(async_client.post("/v1/responses", json=payload)) + second_task = None + try: + await _wait_for_event(first_connect_started) + second_task = asyncio.create_task( + async_client.post( + "/v1/responses", + json={**payload, "input": "hello priority", "service_tier": "priority"}, + ) + ) + for _ in range(10): + await asyncio.sleep(0) + release_first_connect.set() - monkeypatch.setattr(proxy_module.ProxyService, "_prepare_http_bridge_request", capping_prepare) + await _wait_for_event(second_connect_started) + await _wait_for_event(second_session_created) + assert len(created_sessions) == 2 + creator_session, waiter_session = created_sessions - interrupted_user_message = { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "\nThe user interrupted the previous turn on purpose.\n", - } - ], - } - first = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Use the shell tool.", - "input": [{"role": "user", "content": [{"type": "input_text", "text": "run the shell tool"}]}], - "prompt_cache_key": "http-bridge-custom-interrupt-size-1", - }, - ) - assert first.status_code == 200 - assert first.json()["id"] == "resp_bridge_custom_1" + assert len(upstreams) == 2 + assert creator_session is not waiter_session + assert creator_session.upstream is upstreams[0] + assert waiter_session.upstream is upstreams[1] + assert creator_session.closed is False + assert service._http_bridge_sessions.get(creator_session.key) is creator_session + assert creator_session not in scheduled_sessions + assert waiter_session.key.affinity_kind == "internal_request_parallel" - second = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Use the shell tool.", - "previous_response_id": "resp_bridge_custom_1", - "input": [interrupted_user_message], - "prompt_cache_key": "http-bridge-custom-interrupt-size-1", - }, - ) + await _wait_for_event(upstreams[0].request_started) + await _wait_for_event(upstreams[1].request_started) + for upstream in upstreams: + upstream.release_response.set() + first_response, second_response = await asyncio.wait_for( + asyncio.gather(first_task, second_task), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, + ) + finally: + release_first_connect.set() + for upstream in upstreams: + upstream.release_response.set() + pending_tasks = [task for task in (first_task, second_task) if task is not None] + await asyncio.gather(*pending_tasks, return_exceptions=True) - assert followup_cap_armed is True - assert second.status_code == 400 - error = second.json()["error"] - assert error["code"] == "payload_too_large" - assert error["type"] == "invalid_request_error" - # The over-limit injected request must never be forwarded upstream. - assert len(fake_upstream.sent_text) == 1 + assert first_response.status_code == 200 + assert second_response.status_code == 200 + assert first_response.json()["output"][0]["content"][0]["text"] == "OK" + assert second_response.json()["output"][0]["content"][0]["text"] == "OK" + assert creator_session.closed is False @pytest.mark.asyncio -async def test_v1_responses_http_bridge_injected_interrupted_outputs_update_stored_input_context( +async def test_forwarded_priority_prompt_cache_mismatch_forks_on_canonical_owner( async_client, - monkeypatch, app_instance, + monkeypatch, ): - _install_bridge_settings(monkeypatch, enabled=True) + from app.core.middleware import request_id as request_id_middleware_module + from app.modules.proxy import api as proxy_api_module + from app.modules.proxy.http_bridge_forwarding import HTTPBridgeForwardContext, build_owner_forward_headers + + owner_settings = _make_app_settings( + enabled=True, + instance_id="instance-a", + instance_ring=["instance-a", "instance-b"], + ) + origin_settings = _make_app_settings( + enabled=True, + instance_id="instance-b", + instance_ring=["instance-a", "instance-b"], + ) + _install_proxy_settings( + monkeypatch, + app_settings=owner_settings, + dashboard_settings=_make_dashboard_settings(), + ) + monkeypatch.setattr(proxy_api_module, "get_settings", lambda: owner_settings) account_id = await _import_account( async_client, - "acc_http_bridge_custom_interrupt_ctx", - "http-bridge-custom-interrupt-ctx@example.com", + "acc_http_bridge_forwarded_prompt_mismatch", + "http-bridge-forwarded-prompt-mismatch@example.com", + plan_type="pro", ) account = await _get_account(account_id) - fake_upstream = _InterruptedCustomToolUpstreamWebSocket() + service = get_proxy_service_for_app(app_instance) - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, + class Registry: + def get_snapshot(self): + return SimpleNamespace(account_plans={account_id: "pro"}) + + def account_ids_for_model(self, model: str) -> frozenset[str]: + assert model == "gpt-5.3-codex-spark" + return frozenset() + + def plan_types_for_model(self, model: str) -> frozenset[str]: + assert model == "gpt-5.3-codex-spark" + return frozenset({"pro"}) + + def account_ids_for_model_service_tier(self, model: str, service_tier: str) -> frozenset[str]: + assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") + return frozenset() + + def plan_types_for_model_service_tier(self, model: str, service_tier: str) -> frozenset[str]: + assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") + return frozenset({"pro"}) + + async def fake_select_account_with_budget( + self, + deadline, + *, + service_tier=None, + **kwargs, ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, + del self, deadline, kwargs + normalized_service_tier = ( + None + if service_tier is None or service_tier.strip().lower() in {"auto", "default"} + else service_tier.strip().lower() + ) + return AccountSelection( + account=account, + error_message=None, + error_code=None, + catalog_omission_quota_admission=CatalogOmissionQuotaAdmission( + normalized_model="gpt-5.3-codex-spark", + canonical_quota_key="codex_spark", + normalized_effective_service_tier=normalized_service_tier, + ), ) - return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds return target + upstreams: list[_FakeBridgeUpstreamWebSocket] = [] + async def fake_connect_responses_websocket( headers, access_token, @@ -8400,133 +4928,193 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - return fake_upstream + upstream = _FakeBridgeUpstreamWebSocket() + upstreams.append(upstream) + return upstream + + async def fail_legacy_stream(*args, **kwargs): + del args, kwargs + raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + + class Ring: + async def list_active(self, *, require_endpoint: bool = False) -> list[str]: + assert require_endpoint is True + return ["instance-a", "instance-b"] + + async def resolve_endpoint(self, instance_id: str) -> str: + return f"http://{instance_id}" + monkeypatch.setattr(proxy_support, "get_model_registry", lambda: Registry()) monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) + monkeypatch.setattr(request_id_middleware_module, "uuid4", lambda: "forwarded-request-scope") - interrupted_user_message = { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "\nThe user interrupted the previous turn on purpose.\n", - } - ], - } - first = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Use the shell tool.", - "input": [{"role": "user", "content": [{"type": "input_text", "text": "run the shell tool"}]}], - "prompt_cache_key": "http-bridge-custom-interrupt-ctx-1", - }, + ring = cast(Any, Ring()) + original_ring = service._ring_membership + service._ring_membership = ring + canonical_key = proxy_module._HTTPBridgeSessionKey("prompt_cache", "forwarded-prompt-1", None) + fork_key = proxy_module._HTTPBridgeSessionKey( + "internal_request_parallel", + "95427abf10b750a60b5a5d3528343e28c89e8c3a3e428ae51df95534cbf803b3", + None, ) - assert first.status_code == 200 - assert first.json()["id"] == "resp_bridge_custom_1" + assert await proxy_module._http_bridge_owner_instance(canonical_key, owner_settings, ring) == "instance-a" + assert await proxy_module._http_bridge_owner_instance(canonical_key, origin_settings, ring) == "instance-a" + assert await proxy_module._http_bridge_owner_instance(fork_key, owner_settings, ring) == "instance-b" - second = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Use the shell tool.", - "previous_response_id": "resp_bridge_custom_1", - "input": [interrupted_user_message], - "prompt_cache_key": "http-bridge-custom-interrupt-ctx-1", - }, + scheduled_sessions: list[proxy_module._HTTPBridgeSession] = [] + schedule_session_closes = service._schedule_http_bridge_session_closes + + def capture_scheduled_sessions(sessions, *, reason): + scheduled_sessions.extend(sessions) + schedule_session_closes(sessions, reason=reason) + + monkeypatch.setattr(service, "_schedule_http_bridge_session_closes", capture_scheduled_sessions) + creator_payload = { + "model": "gpt-5.3-codex-spark", + "instructions": "Return exactly OK.", + "input": "hello", + "prompt_cache_key": canonical_key.affinity_key, + } + priority_payload = proxy_module.ResponsesRequest.model_validate( + {**creator_payload, "input": "hello priority", "service_tier": "priority"} + ) + forward_context = HTTPBridgeForwardContext( + origin_instance="instance-b", + target_instance="instance-a", + codex_session_affinity=False, + downstream_turn_state=None, + original_request_unanchored=False, + original_affinity_kind=canonical_key.affinity_kind, + original_affinity_key=canonical_key.affinity_key, + ) + forward_headers = build_owner_forward_headers( + headers={"x-request-id": "forwarded-priority-request"}, + payload=priority_payload, + context=forward_context, ) - assert second.status_code == 200 - assert second.json()["id"] == "resp_bridge_custom_2" - assert len(fake_upstream.sent_text) == 2 - second_upstream_input = json.loads(fake_upstream.sent_text[1])["input"] - assert len(second_upstream_input) == 2 - assert second_upstream_input[0]["type"] == "custom_tool_call_output" - assert second_upstream_input[1]["role"] == "user" + try: + creator_response = await async_client.post( + "/v1/responses", + json=creator_payload, + headers={"x-request-id": "creator-request"}, + ) + assert creator_response.status_code == 200, creator_response.text + creator_session = service._http_bridge_sessions[canonical_key] + creator_response_ids = set(creator_session.previous_response_ids) - service = get_proxy_service_for_app(app_instance) - session = None - for _ in range(100): - session = next( - ( - candidate - for candidate in service._http_bridge_sessions.values() - if candidate.last_completed_response_id == "resp_bridge_custom_2" - ), - None, + priority_response = await async_client.post( + "/internal/bridge/responses", + json=priority_payload.model_dump_for_forwarding(), + headers=forward_headers, ) - if session is not None: - break - await asyncio.sleep(0.01) - assert session is not None - # The stored context for the completed response must describe the - # upstream-shaped input (synthetic output + follow-up message), not the - # client-only input, so later full-resend/anchor comparisons on this - # bridge session match what upstream actually stored. - assert session.last_completed_input_count == len(second_upstream_input) == 2 - assert session.last_completed_input_prefix_fingerprint == proxy_module._fingerprint_input_items( - second_upstream_input - ) + + assert priority_response.status_code == 200, priority_response.text + assert creator_response.json()["output"][0]["content"][0]["text"] == "OK" + assert '"type":"response.completed"' in priority_response.text + assert '"text":"OK"' in priority_response.text + assert len(upstreams) == 2 + assert creator_session.upstream is upstreams[0] + assert service._http_bridge_sessions[fork_key].upstream is upstreams[1] + assert creator_session.closed is False + assert service._http_bridge_sessions[canonical_key] is creator_session + assert creator_session not in scheduled_sessions + assert creator_session.request_service_tier is None + assert service._http_bridge_sessions[fork_key].request_service_tier == "priority" + assert creator_response_ids <= creator_session.previous_response_ids + finally: + service._ring_membership = original_ring @pytest.mark.asyncio -async def test_v1_responses_http_bridge_trims_replayed_apply_patch_previous_response_prefix( +async def test_forwarded_recovery_uses_durable_owner_and_strips_stale_affinity( async_client, + app_instance, monkeypatch, ): - _install_bridge_settings(monkeypatch, enabled=True) + from app.modules.proxy import api as proxy_api_module + from app.modules.proxy.continuity import make_http_bridge_account_neutral_replay_key + from app.modules.proxy.http_bridge_forwarding import HTTPBridgeForwardContext, build_owner_forward_headers + + target_settings = _make_app_settings(enabled=True, instance_id="instance-b") + _install_proxy_settings( + monkeypatch, + app_settings=target_settings, + dashboard_settings=_make_dashboard_settings(), + ) + monkeypatch.setattr(proxy_api_module, "get_settings", lambda: target_settings) + alternate_account_id = await _import_account( + async_client, + "acc_http_bridge_forwarded_recovery_alternate", + "http-bridge-forwarded-recovery-alternate@example.com", + ) account_id = await _import_account( async_client, - "acc_http_bridge_apply_patch_trim", - "http-bridge-apply-patch-trim@example.com", + "acc_http_bridge_forwarded_recovery", + "http-bridge-forwarded-recovery@example.com", ) + alternate_account = await _get_account(alternate_account_id) account = await _get_account(account_id) - fake_upstream = _FakeBridgeUpstreamWebSocket() - - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - ) + chatgpt_account_id = cast(str, account.chatgpt_account_id) + service = get_proxy_service_for_app(app_instance) + recovery_kind, recovery_key = make_http_bridge_account_neutral_replay_key("forwarded-recovery") + recovered_turn_state = "http_turn_forwarded_recovery" + recovered_response_id = "resp_forwarded_recovery" + durable_lookup = await service._durable_bridge.claim_live_session( + session_key_kind=recovery_kind, + session_key_value=recovery_key, + api_key_id=None, + instance_id=target_settings.http_responses_session_bridge_instance_id, + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id=account.id, + model="gpt-5.1", + service_tier=None, + latest_turn_state=recovered_turn_state, + latest_response_id=recovered_response_id, + allow_takeover=True, + ) + await service._durable_bridge.register_turn_state( + session_id=durable_lookup.session_id, + api_key_id=None, + instance_id=target_settings.http_responses_session_bridge_instance_id, + owner_epoch=durable_lookup.owner_epoch, + turn_state=recovered_turn_state, + lease_ttl_seconds=60.0, + ) + await service._durable_bridge.register_previous_response_id( + session_id=durable_lookup.session_id, + api_key_id=None, + instance_id=target_settings.http_responses_session_bridge_instance_id, + owner_epoch=durable_lookup.owner_epoch, + response_id=recovered_response_id, + lease_ttl_seconds=60.0, + ) + lookup_request_targets = AsyncMock(wraps=service._durable_bridge.lookup_request_targets) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", lookup_request_targets) + + selection_calls: list[dict[str, object]] = [] + + async def fake_select_account_with_budget(self, deadline, **kwargs): + del self, deadline + selection_calls.append(dict(kwargs)) + if kwargs.get("preferred_account_id") is None: + return AccountSelection(account=alternate_account, error_message=None, error_code=None) + assert kwargs.get("preferred_account_id") == account.id + assert kwargs.get("preferred_account_is_continuity_owner") is True + assert kwargs.get("fallback_on_preferred_account_unavailable") is False return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds return target + upstream = _TurnStateBridgeUpstreamWebSocket("upstream_turn_state_forwarded_recovery") + connect_calls: list[tuple[dict[str, str], str]] = [] + async def fake_connect_responses_websocket( headers, access_token, @@ -8535,79 +5123,107 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, account_id_header, base_url, session - return fake_upstream + del access_token, base_url, session + connect_calls.append((dict(headers), account_id_header)) + return upstream + + async def fail_legacy_stream(*args, **kwargs): + del args, kwargs + raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - first = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Apply the patch.", - "input": [{"role": "user", "content": [{"type": "input_text", "text": "apply the patch"}]}], - "prompt_cache_key": "http-bridge-apply-patch-trim-1", + payload = proxy_module.ResponsesRequest( + model="gpt-5.1", + instructions="Return exactly OK.", + input="continue on the recovered account", + previous_response_id=recovered_response_id, + ) + forward_context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance=target_settings.http_responses_session_bridge_instance_id, + codex_session_affinity=False, + downstream_turn_state=recovered_turn_state, + original_request_unanchored=True, + original_affinity_kind=recovery_kind, + original_affinity_key=recovery_key, + ) + forward_headers = build_owner_forward_headers( + headers={ + "session_id": "stale-session", + "session-id": "stale-session-dash", + "thread-id": "stale-thread", + "x-codex-conversation-id": "stale-conversation", + "x-codex-session-id": "stale-codex-session", + "x-codex-turn-state": "http_turn_stale", + "x-request-trace": "keep-me", }, + payload=payload, + context=forward_context, ) - assert first.status_code == 200 - first_body = first.json() - assert first_body["id"] == "resp_bridge_1" - replayed_apply_patch_call = { - "id": "apc_replay", - "type": "apply_patch_call", - "status": "completed", - "call_id": "call_patch_1", - } - replayed_apply_patch_output = { - "type": "apply_patch_call_output", - "call_id": "call_patch_1", - "status": "completed", - "output": "patched", - } - next_user_message = {"role": "user", "content": [{"type": "input_text", "text": "now run the tests"}]} - second = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Apply the patch.", - "previous_response_id": first_body["id"], - "input": [replayed_apply_patch_call, replayed_apply_patch_output, next_user_message], - "prompt_cache_key": "http-bridge-apply-patch-trim-1", - }, + response = await asyncio.wait_for( + async_client.post( + "/internal/bridge/responses", + json=payload.model_dump_for_forwarding(), + headers=forward_headers, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - assert second.status_code == 200 - assert second.json()["id"] == "resp_bridge_2" - assert len(fake_upstream.sent_text) == 2 - second_upstream_payload = json.loads(fake_upstream.sent_text[1]) - assert second_upstream_payload["previous_response_id"] == "resp_bridge_1" - # The replayed apply_patch_call prefix is already covered by the - # previous_response_id anchor and must be trimmed like the WebSocket - # route trims it; the output item and the new user turn are forwarded. - assert second_upstream_payload["input"] == [replayed_apply_patch_output, next_user_message] + assert response.status_code == 200, response.text + assert '"type":"response.completed"' in response.text + lookup_request_targets.assert_awaited_once_with( + session_key_kind=recovery_kind, + session_key_value=recovery_key, + api_key_id=None, + turn_state=recovered_turn_state, + session_header=None, + previous_response_id=recovered_response_id, + ) + assert len(selection_calls) == 1 + assert selection_calls[0]["preferred_account_id"] == account.id + assert selection_calls[0]["preferred_account_is_continuity_owner"] is True + assert selection_calls[0]["fallback_on_preferred_account_unavailable"] is False + assert len(connect_calls) == 1 + connect_headers, connected_account_id = connect_calls[0] + assert connected_account_id == chatgpt_account_id + normalized_connect_headers = {key.lower(): value for key, value in connect_headers.items()} + assert normalized_connect_headers["x-request-trace"] == "keep-me" + assert ( + not { + "session_id", + "session-id", + "thread-id", + "x-codex-conversation-id", + "x-codex-session-id", + "x-codex-turn-state", + } + & normalized_connect_headers.keys() + ) + assert json.loads(upstream.sent_text[0])["previous_response_id"] == recovered_response_id + recovery_session_key = proxy_module._HTTPBridgeSessionKey(recovery_kind, recovery_key, None) + recovery_session = service._http_bridge_sessions[recovery_session_key] + assert recovery_session.account.id == account.id + assert recovered_turn_state in recovery_session.downstream_turn_state_aliases @pytest.mark.asyncio -async def test_backend_responses_http_bridge_lite_request_omits_synthesized_tools( +async def test_v1_responses_http_bridge_injects_interrupted_custom_tool_output_on_followup( async_client, monkeypatch, ): - # Regression for issue #1184: Responses-Lite clients omit top-level - # ``tools`` entirely (the bundle rides in the ``additional_tools`` input - # item). The HTTP-bridge body must not synthesize ``"tools": []`` from the - # model default; gpt-5.6 reserved model tools reject any explicit - # ``tools`` param that cannot match the reserved schema. _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_backend_http_bridge_lite_no_tools", - "backend-http-bridge-lite-no-tools@example.com", + "acc_http_bridge_custom_interrupt", + "http-bridge-custom-interrupt@example.com", ) account = await _get_account(account_id) - fake_upstream = _FakeBridgeUpstreamWebSocket() + fake_upstream = _InterruptedCustomToolUpstreamWebSocket() async def fake_select_account_with_budget( self, @@ -8628,6 +5244,7 @@ async def fake_select_account_with_budget( api_key=None, preferred_account_id=None, ): + del preferred_account_id del ( self, deadline, @@ -8643,8 +5260,6 @@ async def fake_select_account_with_budget( model, exclude_account_ids, additional_limit_name, - api_key, - preferred_account_id, ) return AccountSelection(account=account, error_message=None, error_code=None) @@ -8663,66 +5278,80 @@ async def fake_connect_responses_websocket( del headers, access_token, account_id_header, base_url, session return fake_upstream - async def fail_legacy_stream(*args, **kwargs): - raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - payload = { - "model": "gpt-5.6", - "instructions": "", - "input": [ + interrupted_user_message = { + "role": "user", + "content": [ { - "type": "additional_tools", - "role": "developer", - "tools": [{"type": "custom", "name": "shell"}], - }, - {"type": "message", "role": "developer", "content": "use repository tools"}, - {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + "type": "input_text", + "text": "\nThe user interrupted the previous turn on purpose.\n", + } ], - "reasoning": { - "context": "last_turn", - "effort": "high", - "summary": "auto", - "vendor_hint": 7, - }, - "stream": True, } - events = await _collect_sse_events(async_client, "/backend-api/codex/responses", json_body=payload) + first = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Use the shell tool.", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "run the shell tool"}]}], + "prompt_cache_key": "http-bridge-custom-interrupt-1", + }, + ) + assert first.status_code == 200 + first_body = first.json() + assert first_body["id"] == "resp_bridge_custom_1" - _assert_created_text_delta_completed(events) - assert len(fake_upstream.sent_text) == 1 - bridge_body = json.loads(fake_upstream.sent_text[0]) - assert "tools" not in bridge_body - # The Lite input prefix must survive and keep signaling Responses Lite. - assert bridge_body["input"] == payload["input"] - client_metadata = bridge_body["client_metadata"] - assert client_metadata["ws_request_header_x_openai_internal_codex_responses_lite"] == "true" - assert bridge_body["reasoning"] == { - "context": "all_turns", - "effort": "high", - "summary": "auto", - "vendor_hint": 7, + second = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Use the shell tool.", + "previous_response_id": first_body["id"], + "input": [interrupted_user_message], + "prompt_cache_key": "http-bridge-custom-interrupt-1", + }, + ) + assert second.status_code == 200 + assert second.json()["id"] == "resp_bridge_custom_2" + + assert len(fake_upstream.sent_text) == 2 + second_upstream_payload = json.loads(fake_upstream.sent_text[1]) + assert second_upstream_payload["previous_response_id"] == "resp_bridge_custom_1" + interrupted_tool_output = ( + "Tool call was not executed because the previous turn was interrupted before tool output was available." + ) + assert second_upstream_payload["input"][0] == { + "type": "custom_tool_call_output", + "call_id": "call_custom_shell", + "output": interrupted_tool_output, } + assert second_upstream_payload["input"][1] == interrupted_user_message @pytest.mark.asyncio -async def test_backend_responses_http_bridge_reuses_upstream_websocket_and_preserves_previous_response_id( +async def test_v1_responses_http_bridge_size_guard_covers_injected_interrupted_tool_outputs( async_client, monkeypatch, + tmp_path, ): _install_bridge_settings(monkeypatch, enabled=True) + monkeypatch.setattr(proxy_module, "_UPSTREAM_RESPONSE_CREATE_MAX_BYTES", 10_000_000) + monkeypatch.setattr(proxy_module, "_UPSTREAM_RESPONSE_CREATE_WARN_BYTES", 10_000_000) + monkeypatch.setattr(proxy_module, "_OVERSIZED_RESPONSE_CREATE_DUMP_DIR", tmp_path) account_id = await _import_account( async_client, - "acc_backend_http_bridge_reuse", - "backend-http-bridge-reuse@example.com", + "acc_http_bridge_custom_interrupt_size", + "http-bridge-custom-interrupt-size@example.com", ) account = await _get_account(account_id) - fake_upstream = _FakeBridgeUpstreamWebSocket() - connect_calls: list[tuple[str | None, str | None]] = [] + # Blank installation id makes the submit-time account-installation rewrite + # a no-op, so no later serialization step would re-run the size guard; + # the injection path itself must keep the request within the limit. + account.codex_installation_id = "" + fake_upstream = _InterruptedCustomToolUpstreamWebSocket() async def fake_select_account_with_budget( self, @@ -8774,58 +5403,83 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, base_url, session - connect_calls.append((account_id, account_id_header)) + del headers, access_token, account_id_header, base_url, session return fake_upstream - async def fail_legacy_stream(*args, **kwargs): - raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - payload = { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "backend-http-bridge-thread-1", - "stream": True, + original_prepare = proxy_module.ProxyService._prepare_http_bridge_request + followup_cap_armed = False + + def capping_prepare(self, payload, headers, **kwargs): + nonlocal followup_cap_armed + request_state, text_data = original_prepare(self, payload, headers, **kwargs) + if not followup_cap_armed and '"previous_response_id":"resp_bridge_custom_1"' in text_data: + # The anchored follow-up fits the limit as sent by the client; + # prepending synthetic interrupted outputs pushes it over. + followup_cap_armed = True + proxy_module._UPSTREAM_RESPONSE_CREATE_MAX_BYTES = len(text_data.encode("utf-8")) + 100 + return request_state, text_data + + monkeypatch.setattr(proxy_module.ProxyService, "_prepare_http_bridge_request", capping_prepare) + + interrupted_user_message = { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "\nThe user interrupted the previous turn on purpose.\n", + } + ], } - first_events = await _collect_sse_events(async_client, "/backend-api/codex/responses", json_body=payload) - first_response = first_events[-1]["response"] + first = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Use the shell tool.", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "run the shell tool"}]}], + "prompt_cache_key": "http-bridge-custom-interrupt-size-1", + }, + ) + assert first.status_code == 200 + assert first.json()["id"] == "resp_bridge_custom_1" - second_events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={**payload, "previous_response_id": first_response["id"]}, + second = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Use the shell tool.", + "previous_response_id": "resp_bridge_custom_1", + "input": [interrupted_user_message], + "prompt_cache_key": "http-bridge-custom-interrupt-size-1", + }, ) - second_response = second_events[-1]["response"] - _assert_created_text_delta_completed(first_events) - _assert_created_text_delta_completed(second_events) - assert first_response["id"] == "resp_bridge_1" - assert second_response["id"] == "resp_bridge_2" - assert connect_calls == [(account_id, account.chatgpt_account_id)] - assert len(fake_upstream.sent_text) == 2 - assert json.loads(fake_upstream.sent_text[1])["previous_response_id"] == "resp_bridge_1" + assert followup_cap_armed is True + assert second.status_code == 400 + error = second.json()["error"] + assert error["code"] == "payload_too_large" + assert error["type"] == "invalid_request_error" + # The over-limit injected request must never be forwarded upstream. + assert len(fake_upstream.sent_text) == 1 @pytest.mark.asyncio -async def test_backend_responses_http_bridge_prefers_codex_session_header_over_prompt_cache_key( +async def test_v1_responses_http_bridge_injected_interrupted_outputs_update_stored_input_context( async_client, monkeypatch, + app_instance, ): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_backend_http_bridge_session_header", - "backend-http-bridge-session-header@example.com", + "acc_http_bridge_custom_interrupt_ctx", + "http-bridge-custom-interrupt-ctx@example.com", ) account = await _get_account(account_id) - fake_upstream = _FakeBridgeUpstreamWebSocket() - connect_calls: list[tuple[str | None, proxy_module.StickySessionKind | None]] = [] + fake_upstream = _InterruptedCustomToolUpstreamWebSocket() async def fake_select_account_with_budget( self, @@ -8852,6 +5506,9 @@ async def fake_select_account_with_budget( deadline, request_id, kind, + request_stage, + sticky_key, + sticky_kind, reallocate_sticky, sticky_max_age_seconds, prefer_earlier_reset_accounts, @@ -8860,7 +5517,6 @@ async def fake_select_account_with_budget( exclude_account_ids, additional_limit_name, ) - connect_calls.append((sticky_key, sticky_kind)) return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): @@ -8882,139 +5538,84 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - headers = {"session_id": "backend-http-session-1"} - first_events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ + interrupted_user_message = { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "\nThe user interrupted the previous turn on purpose.\n", + } + ], + } + first = await async_client.post( + "/v1/responses", + json={ "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "backend-http-prompt-a", - "stream": True, + "instructions": "Use the shell tool.", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "run the shell tool"}]}], + "prompt_cache_key": "http-bridge-custom-interrupt-ctx-1", }, - headers=headers, ) - first_response = first_events[-1]["response"] + assert first.status_code == 200 + assert first.json()["id"] == "resp_bridge_custom_1" - second_events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ + second = await async_client.post( + "/v1/responses", + json={ "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello-again", - "prompt_cache_key": "backend-http-prompt-b", - "previous_response_id": first_response["id"], - "stream": True, + "instructions": "Use the shell tool.", + "previous_response_id": "resp_bridge_custom_1", + "input": [interrupted_user_message], + "prompt_cache_key": "http-bridge-custom-interrupt-ctx-1", }, - headers=headers, ) + assert second.status_code == 200 + assert second.json()["id"] == "resp_bridge_custom_2" - _assert_created_text_delta_completed(first_events) - _assert_created_text_delta_completed(second_events) - assert len(connect_calls) == 1 - assert connect_calls[0] == ( - _codex_session_selection_key("backend-http-session-1"), - proxy_module.StickySessionKind.CODEX_SESSION, - ) assert len(fake_upstream.sent_text) == 2 - assert json.loads(fake_upstream.sent_text[1])["prompt_cache_key"] == "backend-http-prompt-b" - - -@pytest.mark.asyncio -async def test_backend_responses_http_bridge_file_owner_overrides_soft_locality( - async_client, - monkeypatch, -): - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_backend_http_bridge_file_owner", - "backend-http-bridge-file-owner@example.com", - ) - account = await _get_account(account_id) - service = get_proxy_service_for_app(async_client._transport.app) - await service._pin_file_account("file_bridge_owner", account.id) - fake_upstream = _FakeBridgeUpstreamWebSocket() - selection_calls: list[dict[str, object]] = [] - - async def fake_select_account(**kwargs: object) -> AccountSelection: - selection_calls.append(dict(kwargs)) - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - return fake_upstream - - monkeypatch.setattr(service._load_balancer, "select_account", fake_select_account) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - headers={"session_id": "bridge-soft-session"}, - json_body={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "Read the file."}, - {"type": "input_file", "file_id": "file_bridge_owner"}, - ], - } - ], - "prompt_cache_key": "bridge-soft-cache", - "stream": True, - }, - ) + second_upstream_input = json.loads(fake_upstream.sent_text[1])["input"] + assert len(second_upstream_input) == 2 + assert second_upstream_input[0]["type"] == "custom_tool_call_output" + assert second_upstream_input[1]["role"] == "user" - _assert_created_text_delta_completed(events) - assert len(selection_calls) == 1 - assert selection_calls[0]["account_ids"] is None - assert selection_calls[0]["required_account_id"] == account.id - assert selection_calls[0]["sticky_key"] is None - assert len(fake_upstream.sent_text) == 1 + service = get_proxy_service_for_app(app_instance) + session = None + for _ in range(100): + session = next( + ( + candidate + for candidate in service._http_bridge_sessions.values() + if candidate.last_completed_response_id == "resp_bridge_custom_2" + ), + None, + ) + if session is not None: + break + await asyncio.sleep(0.01) + assert session is not None + # The stored context for the completed response must describe the + # upstream-shaped input (synthetic output + follow-up message), not the + # client-only input, so later full-resend/anchor comparisons on this + # bridge session match what upstream actually stored. + assert session.last_completed_input_count == len(second_upstream_input) == 2 + assert session.last_completed_input_prefix_fingerprint == proxy_module._fingerprint_input_items( + second_upstream_input + ) @pytest.mark.asyncio -@pytest.mark.parametrize( - ("second_model", "expected_connection_count"), - [ - pytest.param("gpt-5.1", 1, id="same-model-reuse"), - pytest.param("gpt-5.4", 2, id="model-transition-fork"), - ], -) -async def test_backend_responses_http_emits_turn_state_header_and_reuses_when_compatible( +async def test_v1_responses_http_bridge_trims_replayed_apply_patch_previous_response_prefix( async_client, monkeypatch, - second_model: str, - expected_connection_count: int, ): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_backend_http_bridge_turn_state", - "backend-http-bridge-turn-state@example.com", + "acc_http_bridge_apply_patch_trim", + "http-bridge-apply-patch-trim@example.com", ) account = await _get_account(account_id) - available_upstreams = deque(_FakeBridgeUpstreamWebSocket() for _ in range(3)) - connected_upstreams: list[_FakeBridgeUpstreamWebSocket] = [] - connect_calls: list[tuple[str | None, proxy_module.StickySessionKind | None]] = [] + fake_upstream = _FakeBridgeUpstreamWebSocket() async def fake_select_account_with_budget( self, @@ -9041,6 +5642,9 @@ async def fake_select_account_with_budget( deadline, request_id, kind, + request_stage, + sticky_key, + sticky_kind, reallocate_sticky, sticky_max_age_seconds, prefer_earlier_reset_accounts, @@ -9049,7 +5653,6 @@ async def fake_select_account_with_budget( exclude_account_ids, additional_limit_name, ) - connect_calls.append((sticky_key, sticky_kind)) return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): @@ -9065,82 +5668,78 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - upstream = available_upstreams.popleft() - connected_upstreams.append(upstream) - return upstream + return fake_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - first_events, first_headers = await _collect_sse_events_with_headers( - async_client, - "/backend-api/codex/responses", - json_body={ + first = await async_client.post( + "/v1/responses", + json={ "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "backend-http-turn-state-a", - "stream": True, + "instructions": "Apply the patch.", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "apply the patch"}]}], + "prompt_cache_key": "http-bridge-apply-patch-trim-1", }, ) - turn_state = first_headers["x-codex-turn-state"] - first_response = first_events[-1]["response"] + assert first.status_code == 200 + first_body = first.json() + assert first_body["id"] == "resp_bridge_1" - second_events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ - "model": second_model, - "instructions": "Return exactly OK.", - "input": "hello-again", - "prompt_cache_key": "backend-http-turn-state-b", - "previous_response_id": first_response["id"], - "stream": True, + replayed_apply_patch_call = { + "id": "apc_replay", + "type": "apply_patch_call", + "status": "completed", + "call_id": "call_patch_1", + } + replayed_apply_patch_output = { + "type": "apply_patch_call_output", + "call_id": "call_patch_1", + "status": "completed", + "output": "patched", + } + next_user_message = {"role": "user", "content": [{"type": "input_text", "text": "now run the tests"}]} + second = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Apply the patch.", + "previous_response_id": first_body["id"], + "input": [replayed_apply_patch_call, replayed_apply_patch_output, next_user_message], + "prompt_cache_key": "http-bridge-apply-patch-trim-1", }, - headers={"x-codex-turn-state": turn_state}, ) - third_events: list[dict] | None = None - if second_model != "gpt-5.1": - third_events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello-on-original-model", - "prompt_cache_key": "backend-http-turn-state-c", - "stream": True, - }, - headers={"x-codex-turn-state": turn_state}, - ) + assert second.status_code == 200 + assert second.json()["id"] == "resp_bridge_2" - _assert_created_text_delta_completed(first_events) - _assert_created_text_delta_completed(second_events) - if third_events is not None: - _assert_created_text_delta_completed(third_events) - assert turn_state.startswith("http_turn_") - assert connect_calls[0] == ("backend-http-turn-state-a", proxy_module.StickySessionKind.PROMPT_CACHE) - assert len(connect_calls) == expected_connection_count - expected_request_counts = [2] if expected_connection_count == 1 else [2, 1] - assert [len(upstream.sent_text) for upstream in connected_upstreams] == expected_request_counts - assert connected_upstreams[0].closed is False + assert len(fake_upstream.sent_text) == 2 + second_upstream_payload = json.loads(fake_upstream.sent_text[1]) + assert second_upstream_payload["previous_response_id"] == "resp_bridge_1" + # The replayed apply_patch_call prefix is already covered by the + # previous_response_id anchor and must be trimmed like the WebSocket + # route trims it; the output item and the new user turn are forwarded. + assert second_upstream_payload["input"] == [replayed_apply_patch_output, next_user_message] @pytest.mark.asyncio -async def test_v1_responses_http_bridge_reuses_session_across_model_change_for_previous_response_id( +async def test_backend_responses_http_bridge_lite_request_omits_synthesized_tools( async_client, monkeypatch, ): + # Regression for issue #1184: Responses-Lite clients omit top-level + # ``tools`` entirely (the bundle rides in the ``additional_tools`` input + # item). The HTTP-bridge body must not synthesize ``"tools": []`` from the + # model default; gpt-5.6 reserved model tools reject any explicit + # ``tools`` param that cannot match the reserved schema. _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_model_change", - "http-bridge-model-change@example.com", + "acc_backend_http_bridge_lite_no_tools", + "backend-http-bridge-lite-no-tools@example.com", ) account = await _get_account(account_id) fake_upstream = _FakeBridgeUpstreamWebSocket() - connect_calls: list[tuple[str | None, str | None]] = [] async def fake_select_account_with_budget( self, @@ -9161,7 +5760,6 @@ async def fake_select_account_with_budget( api_key=None, preferred_account_id=None, ): - del preferred_account_id del ( self, deadline, @@ -9177,6 +5775,8 @@ async def fake_select_account_with_budget( model, exclude_account_ids, additional_limit_name, + api_key, + preferred_account_id, ) return AccountSelection(account=account, error_message=None, error_code=None) @@ -9192,56 +5792,69 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, base_url, session - connect_calls.append((account_id, account_id_header)) + del headers, access_token, account_id_header, base_url, session return fake_upstream + async def fail_legacy_stream(*args, **kwargs): + raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - first = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-model-thread", - }, - ) - assert first.status_code == 200 - first_body = first.json() - - second = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.4", - "instructions": "Return exactly OK.", - "input": "hello again", - "prompt_cache_key": "http-bridge-model-thread", - "previous_response_id": first_body["id"], + payload = { + "model": "gpt-5.6", + "instructions": "", + "input": [ + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "custom", "name": "shell"}], + }, + {"type": "message", "role": "developer", "content": "use repository tools"}, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + ], + "reasoning": { + "context": "last_turn", + "effort": "high", + "summary": "auto", + "vendor_hint": 7, }, - ) - assert second.status_code == 200 + "stream": True, + } + events = await _collect_sse_events(async_client, "/backend-api/codex/responses", json_body=payload) - assert connect_calls == [(account_id, account.chatgpt_account_id)] - assert len(fake_upstream.sent_text) == 2 - second_payload = json.loads(fake_upstream.sent_text[1]) - assert second_payload["model"] == "gpt-5.4" - assert second_payload["previous_response_id"] == first_body["id"] + _assert_created_text_delta_completed(events) + assert len(fake_upstream.sent_text) == 1 + bridge_body = json.loads(fake_upstream.sent_text[0]) + assert "tools" not in bridge_body + # The Lite input prefix must survive and keep signaling Responses Lite. + assert bridge_body["input"] == payload["input"] + client_metadata = bridge_body["client_metadata"] + assert client_metadata["ws_request_header_x_openai_internal_codex_responses_lite"] == "true" + assert bridge_body["reasoning"] == { + "context": "all_turns", + "effort": "high", + "summary": "auto", + "vendor_hint": 7, + } @pytest.mark.asyncio -async def test_v1_responses_http_bridge_recovers_previous_response_id_across_key_drift(async_client, monkeypatch): +async def test_backend_responses_http_bridge_reuses_upstream_websocket_and_preserves_previous_response_id( + async_client, + monkeypatch, +): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_live_session_required", - "http-bridge-live-session-required@example.com", + "acc_backend_http_bridge_reuse", + "backend-http-bridge-reuse@example.com", ) account = await _get_account(account_id) fake_upstream = _FakeBridgeUpstreamWebSocket() - connect_count = 0 + connect_calls: list[tuple[str | None, str | None]] = [] async def fake_select_account_with_budget( self, @@ -9293,50 +5906,54 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, account_id_header, base_url, session - nonlocal connect_count - connect_count += 1 + del headers, access_token, base_url, session + connect_calls.append((account_id, account_id_header)) return fake_upstream + async def fail_legacy_stream(*args, **kwargs): + raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - first = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-live-session-a", - }, - ) - assert first.status_code == 200 - first_body = first.json() + payload = { + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + "prompt_cache_key": "backend-http-bridge-thread-1", + "stream": True, + } + first_events = await _collect_sse_events(async_client, "/backend-api/codex/responses", json_body=payload) + first_response = first_events[-1]["response"] - second = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello-again", - "prompt_cache_key": "http-bridge-live-session-b", - "previous_response_id": first_body["id"], - }, + second_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={**payload, "previous_response_id": first_response["id"]}, ) + second_response = second_events[-1]["response"] - assert second.status_code == 200 - assert second.json()["output"][0]["content"][0]["text"] == "OK" - assert connect_count == 1 + _assert_created_text_delta_completed(first_events) + _assert_created_text_delta_completed(second_events) + assert first_response["id"] == "resp_bridge_1" + assert second_response["id"] == "resp_bridge_2" + assert connect_calls == [(account_id, account.chatgpt_account_id)] + assert len(fake_upstream.sent_text) == 2 + assert json.loads(fake_upstream.sent_text[1])["previous_response_id"] == "resp_bridge_1" @pytest.mark.asyncio -async def test_v1_responses_http_emits_turn_state_header_and_reuses_when_replayed(async_client, monkeypatch): +async def test_backend_responses_http_bridge_prefers_codex_session_header_over_prompt_cache_key( + async_client, + monkeypatch, +): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_v1_http_bridge_turn_state", - "v1-http-bridge-turn-state@example.com", + "acc_backend_http_bridge_session_header", + "backend-http-bridge-session-header@example.com", ) account = await _get_account(account_id) fake_upstream = _FakeBridgeUpstreamWebSocket() @@ -9397,81 +6014,89 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - first = await async_client.post( - "/v1/responses", - json={ + headers = {"session_id": "backend-http-session-1"} + first_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ "model": "gpt-5.1", "instructions": "Return exactly OK.", "input": "hello", - "prompt_cache_key": "v1-http-turn-state-a", + "prompt_cache_key": "backend-http-prompt-a", + "stream": True, }, + headers=headers, ) - assert first.status_code == 200 - turn_state = first.headers["x-codex-turn-state"] - first_body = first.json() + first_response = first_events[-1]["response"] - second = await async_client.post( - "/v1/responses", - headers={"x-codex-turn-state": turn_state}, - json={ + second_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ "model": "gpt-5.1", "instructions": "Return exactly OK.", "input": "hello-again", - "prompt_cache_key": "v1-http-turn-state-b", - "previous_response_id": first_body["id"], + "prompt_cache_key": "backend-http-prompt-b", + "previous_response_id": first_response["id"], + "stream": True, }, + headers=headers, ) - assert second.status_code == 200 - assert turn_state.startswith("http_turn_") - assert connect_calls == [("v1-http-turn-state-a", proxy_module.StickySessionKind.PROMPT_CACHE)] + _assert_created_text_delta_completed(first_events) + _assert_created_text_delta_completed(second_events) + assert len(connect_calls) == 1 + assert connect_calls[0] == ( + _codex_session_selection_key("backend-http-session-1"), + proxy_module.StickySessionKind.CODEX_SESSION, + ) + assert len(fake_upstream.sent_text) == 2 + assert json.loads(fake_upstream.sent_text[1])["prompt_cache_key"] == "backend-http-prompt-b" @pytest.mark.asyncio -async def test_v1_responses_http_bridge_streaming_path_uses_persistent_upstream_websocket(async_client, monkeypatch): +async def test_backend_responses_goal_restart_bypasses_live_bridge_and_retires_unavailable_legacy_owner( + async_client, + app_instance, + monkeypatch, +): _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account(async_client, "acc_http_bridge_sse", "http-bridge-sse@example.com") - account = await _get_account(account_id) - fake_upstream = _FakeBridgeUpstreamWebSocket() - connect_count = 0 - - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, + owner_id = await _import_account( + async_client, + "acc_backend_bridge_goal_restart_owner", + "backend-bridge-goal-restart-owner@example.com", + ) + replacement_id = await _import_account( + async_client, + "acc_backend_bridge_goal_restart_replacement", + "backend-bridge-goal-restart-replacement@example.com", + ) + owner = await _get_account(owner_id) + replacement = await _get_account(replacement_id) + owner_chatgpt_account_id = cast(str, owner.chatgpt_account_id) + replacement_chatgpt_account_id = cast(str, replacement.chatgpt_account_id) + raw_session = "backend-bridge-goal-restart-session" + selection_key = _codex_session_selection_key(raw_session) + async with SessionLocal() as session: + await StickySessionsRepository(session).upsert( + raw_session, + owner.id, + kind=proxy_module.StickySessionKind.CODEX_SESSION, ) - return AccountSelection(account=account, error_message=None, error_code=None) + + owner_send_started = asyncio.Event() + owner_send_release = asyncio.Event() + + class _BlockingOwnerUpstream(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + if not self.sent_text: + owner_send_started.set() + await owner_send_release.wait() + await super().send_text(text) + + owner_upstream = _BlockingOwnerUpstream("resp_bridge_goal_owner") + replacement_upstream = _FakeBridgeUpstreamWebSocket("resp_bridge_goal_replacement") + connected_account_ids: list[str] = [] async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds @@ -9485,263 +6110,430 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, account_id_header, base_url, session - nonlocal connect_count - connect_count += 1 - return fake_upstream - - async def fail_legacy_stream(*args, **kwargs): - raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + del headers, access_token, base_url, session + connected_account_ids.append(account_id_header) + if account_id_header == owner_chatgpt_account_id: + return owner_upstream + assert account_id_header == replacement_chatgpt_account_id + return replacement_upstream - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - - payload = { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-sse-thread-1", - "stream": True, - } - async with async_client.stream("POST", "/v1/responses", json=payload) as response: - assert response.status_code == 200 - lines = [line async for line in response.aiter_lines() if line.startswith("data: ")] - - events = [json.loads(line[6:]) for line in lines if line[6:] != "[DONE]"] - _assert_created_text_delta_completed(events) - assert connect_count == 1 + headers = {"session_id": raw_session} + first_response_task = asyncio.create_task( + _collect_sse_events( + async_client, + "/backend-api/codex/responses", + headers=headers, + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "prime the live bridge", + "stream": True, + }, + ) + ) + await _wait_for_event(owner_send_started) -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_kill_switch_falls_back_to_legacy_path(async_client, monkeypatch): - _install_bridge_settings(monkeypatch, enabled=False) - await _import_account(async_client, "acc_http_bridge_fallback", "http-bridge-fallback@example.com") - seen = {"legacy": 0} + service = get_proxy_service_for_app(app_instance) + bridge_key = proxy_module._HTTPBridgeSessionKey("session_header", raw_session, None) + async with service._http_bridge_lock: + old_bridge = service._http_bridge_sessions[bridge_key] + assert old_bridge.account.id == owner.id + # Change only the persisted account row. The live bridge deliberately + # retains its earlier detached ACTIVE Account object, reproducing the + # reuse path that used to bypass authoritative restart selection. + async with SessionLocal() as session: + await session.execute(update(Account).where(Account.id == owner.id).values(status=AccountStatus.QUOTA_EXCEEDED)) + await session.commit() - async def fake_legacy_stream( - payload, - headers, - access_token, - account_id, - base_url=None, - raise_for_status=False, - **_kw, - ): - del headers, access_token, account_id, base_url, raise_for_status, _kw - seen["legacy"] += 1 - yield ( - 'data: {"type":"response.completed","response":{"id":"resp_legacy",' - '"object":"response","status":"completed",' - '"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2,"input_tokens_details":{"cached_tokens":0},' - '"output_tokens_details":{"reasoning_tokens":0}}}}\n\n' + try: + restart_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + headers=headers, + json_body={ + "model": "gpt-5.1", + "instructions": "Continue the existing task.", + "input": [ + { + "role": "developer", + "content": ( + '\nContinue working toward the active thread goal.' + ), + }, + {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ], + "stream": True, + }, ) + async with service._http_bridge_lock: + replacement_bridge = service._http_bridge_sessions[bridge_key] + assert replacement_bridge is not old_bridge + assert replacement_bridge.account.id == replacement.id + assert replacement_bridge.key == bridge_key + assert old_bridge.upstream_control.retire_after_drain is True + finally: + owner_send_release.set() + first_events = await asyncio.wait_for(first_response_task, timeout=_TEST_SYNC_TIMEOUT_SECONDS) - async def fail_connect(*args, **kwargs): - raise AssertionError("bridge websocket path must not be used when the kill switch disables it") + assert first_events[-1]["response"]["id"] == "resp_bridge_goal_owner_1" + assert restart_events[-1]["response"]["id"] == "resp_bridge_goal_replacement_1" + assert connected_account_ids == [owner_chatgpt_account_id, replacement_chatgpt_account_id] + assert len(owner_upstream.sent_text) == 1 + assert len(replacement_upstream.sent_text) == 1 + assert old_bridge.closed is True - monkeypatch.setattr(proxy_module, "core_stream_responses", fake_legacy_stream) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fail_connect) + follow_up_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + headers=headers, + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "continue on the replacement bridge", + "stream": True, + }, + ) + assert follow_up_events[-1]["response"]["id"] == "resp_bridge_goal_replacement_2" + assert len(owner_upstream.sent_text) == 1 + assert len(replacement_upstream.sent_text) == 2 - response = await async_client.post("/v1/responses", json={"model": "gpt-5.1", "input": "hi"}) - assert response.status_code == 200 - assert response.json()["id"] == "resp_legacy" - assert "x-codex-turn-state" not in response.headers - assert seen["legacy"] == 1 + async with SessionLocal() as session: + rows = { + row.key: row + for row in ( + await session.execute( + select(StickySession).where( + StickySession.key.in_((raw_session, selection_key)), + StickySession.kind == proxy_module.StickySessionKind.CODEX_SESSION, + ) + ) + ).scalars() + } + assert rows[raw_session].account_id == owner.id + assert rows[raw_session].continuity_abandoned_at is None + assert rows[raw_session].continuity_abandonment_scope == "session_header" + assert rows[selection_key].account_id == replacement.id + assert rows[selection_key].continuity_abandoned_at is None @pytest.mark.asyncio -async def test_backend_responses_http_bridge_kill_switch_falls_back_to_legacy_path(async_client, monkeypatch): - _install_bridge_settings(monkeypatch, enabled=False) - await _import_account(async_client, "acc_backend_http_bridge_fallback", "backend-http-bridge-fallback@example.com") - seen = {"legacy": 0} - - async def fake_legacy_stream( - payload, - headers, - access_token, - account_id, - base_url=None, - raise_for_status=False, - **_kw, - ): - del payload, headers, access_token, account_id, base_url, raise_for_status, _kw - seen["legacy"] += 1 - yield ( - 'data: {"type":"response.completed","response":{"id":"resp_backend_legacy",' - '"object":"response","status":"completed",' - '"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2,' - '"input_tokens_details":{"cached_tokens":0},"output_tokens_details":{"reasoning_tokens":0}}}}\n\n' +async def test_backend_responses_goal_restart_keeps_authority_for_same_request_reconnect( + async_client, + monkeypatch, +): + _install_bridge_settings(monkeypatch, enabled=True) + owner_id = await _import_account( + async_client, + "acc_backend_bridge_reconnect_restart_owner", + "backend-bridge-reconnect-restart-owner@example.com", + ) + replacement_id = await _import_account( + async_client, + "acc_backend_bridge_reconnect_restart_replacement", + "backend-bridge-reconnect-restart-replacement@example.com", + ) + owner = await _get_account(owner_id) + replacement = await _get_account(replacement_id) + owner_chatgpt_account_id = cast(str, owner.chatgpt_account_id) + replacement_chatgpt_account_id = cast(str, replacement.chatgpt_account_id) + raw_session = "backend-bridge-reconnect-restart-session" + selection_key = _codex_session_selection_key(raw_session) + async with SessionLocal() as session: + await StickySessionsRepository(session).upsert( + raw_session, + owner.id, + kind=proxy_module.StickySessionKind.CODEX_SESSION, ) - async def fail_connect(*args, **kwargs): - raise AssertionError("bridge websocket path must not be used when the kill switch disables it") - - monkeypatch.setattr(proxy_module, "core_stream_responses", fake_legacy_stream) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fail_connect) + class _OwnerBecomesUnavailableBeforeResponse(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + async with SessionLocal() as session: + await session.execute( + update(Account).where(Account.id == owner.id).values(status=AccountStatus.QUOTA_EXCEEDED) + ) + await session.commit() + await self._messages.put(_FakeUpstreamMessage("close", close_code=1000)) - events, response_headers = await _collect_sse_events_with_headers( - async_client, - "/backend-api/codex/responses", - json_body={"model": "gpt-5.1", "instructions": "hi", "input": "hello", "stream": True}, - ) + owner_upstream = _OwnerBecomesUnavailableBeforeResponse("resp_bridge_reconnect_restart_owner") + replacement_upstream = _FakeBridgeUpstreamWebSocket("resp_bridge_reconnect_restart_replacement") + connected_account_ids: list[str] = [] - assert [event["type"] for event in events] == ["response.completed"] - assert events[0]["response"]["id"] == "resp_backend_legacy" - assert "x-codex-turn-state" not in response_headers - assert seen["legacy"] == 1 + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, base_url, session + connected_account_ids.append(account_id_header) + if account_id_header == owner_chatgpt_account_id: + return owner_upstream + assert account_id_header == replacement_chatgpt_account_id + return replacement_upstream -@pytest.mark.asyncio -async def test_backend_responses_http_bridge_startup_error_omits_turn_state_header(async_client, monkeypatch): - _install_bridge_settings(monkeypatch, enabled=True) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - response = await async_client.post( + restart_events = await _collect_sse_events( + async_client, "/backend-api/codex/responses", - json={ + headers={"session_id": raw_session}, + json_body={ "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", + "instructions": "Continue the existing task.", + "input": [ + { + "role": "developer", + "content": ( + '\nContinue working toward the active thread goal.' + ), + }, + {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ], "stream": True, }, ) - assert response.status_code == 503 - assert response.json()["error"]["code"] == "no_accounts" - assert "x-codex-turn-state" not in response.headers + assert restart_events[-1]["response"]["id"] == "resp_bridge_reconnect_restart_replacement_1" + assert connected_account_ids == [owner_chatgpt_account_id, replacement_chatgpt_account_id] + assert len(owner_upstream.sent_text) == 1 + assert len(replacement_upstream.sent_text) == 1 + async with SessionLocal() as session: + rows = { + row.key: row + for row in ( + await session.execute( + select(StickySession).where( + StickySession.key.in_((raw_session, selection_key)), + StickySession.kind == proxy_module.StickySessionKind.CODEX_SESSION, + ) + ) + ).scalars() + } + assert rows[raw_session].account_id == owner.id + assert rows[raw_session].continuity_abandoned_at is None + assert rows[raw_session].continuity_abandonment_scope == "session_header" + assert rows[selection_key].account_id == replacement.id @pytest.mark.asyncio -async def test_backend_responses_http_bridge_pool_usage_exhaustion_returns_429(async_client, monkeypatch): +async def test_backend_responses_goal_restart_authority_does_not_leak_to_reused_bridge( + async_client, + app_instance, + monkeypatch, +): _install_bridge_settings(monkeypatch, enabled=True) - - async def fake_select_account_with_budget(*_args, **_kwargs): - return proxy_module.AccountSelection( - account=None, - error_message="Usage limit reached", - error_code="usage_limit_reached", + owner_id = await _import_account( + async_client, + "acc_backend_bridge_one_shot_restart_owner", + "backend-bridge-one-shot-restart-owner@example.com", + ) + replacement_id = await _import_account( + async_client, + "acc_backend_bridge_one_shot_restart_replacement", + "backend-bridge-one-shot-restart-replacement@example.com", + ) + owner = await _get_account(owner_id) + replacement = await _get_account(replacement_id) + owner_chatgpt_account_id = cast(str, owner.chatgpt_account_id) + replacement_chatgpt_account_id = cast(str, replacement.chatgpt_account_id) + raw_session = "backend-bridge-one-shot-restart-session" + async with SessionLocal() as session: + await StickySessionsRepository(session).upsert( + raw_session, + owner.id, + kind=proxy_module.StickySessionKind.CODEX_SESSION, ) - monkeypatch.setattr( - proxy_module.ProxyService, - "_select_account_with_budget", - fake_select_account_with_budget, - ) + owner_upstream = _CompleteThenPrecreatedCloseUpstreamWebSocket("resp_bridge_one_shot_owner") + replacement_upstream = _FakeBridgeUpstreamWebSocket("resp_bridge_one_shot_replacement") + connected_account_ids: list[str] = [] - response = await async_client.post( + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, base_url, session + connected_account_ids.append(account_id_header) + if account_id_header == owner_chatgpt_account_id: + return owner_upstream + assert account_id_header == replacement_chatgpt_account_id + return replacement_upstream + + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + headers = {"session_id": raw_session} + restart_events = await _collect_sse_events( + async_client, "/backend-api/codex/responses", - json={ + headers=headers, + json_body={ "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", + "instructions": "Continue the existing task.", + "input": [ + { + "role": "developer", + "content": ( + '\nContinue working toward the active thread goal.' + ), + }, + {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ], "stream": True, }, ) + assert restart_events[-1]["response"]["id"] == "resp_bridge_one_shot_owner_1" - assert response.status_code == 429 - assert response.json()["error"]["type"] == "usage_limit_reached" - assert response.json()["error"]["code"] == "usage_limit_reached" - assert "x-codex-turn-state" not in response.headers - + service = get_proxy_service_for_app(app_instance) + bridge_key = proxy_module._HTTPBridgeSessionKey("session_header", raw_session, None) + async with service._http_bridge_lock: + bridge = service._http_bridge_sessions[bridge_key] + assert bridge.affinity.abandon_unavailable_legacy_owner is False -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_startup_error_omits_turn_state_header(async_client, monkeypatch): - _install_bridge_settings(monkeypatch, enabled=True) + async with SessionLocal() as session: + await session.execute(update(Account).where(Account.id == owner.id).values(status=AccountStatus.QUOTA_EXCEEDED)) + await session.commit() - response = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - }, + ordinary_response = await asyncio.wait_for( + async_client.post( + "/backend-api/codex/responses", + headers=headers, + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "ordinary follow-up without restart authority", + "stream": True, + }, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - assert response.status_code == 503 - assert response.json()["error"]["code"] == "no_accounts" - assert "x-codex-turn-state" not in response.headers + assert ordinary_response.status_code == 502 + assert ordinary_response.json()["error"]["code"] == "upstream_unavailable" + assert connected_account_ids == [owner_chatgpt_account_id] + assert len(owner_upstream.sent_text) == 2 + assert replacement_upstream.sent_text == [] + async with SessionLocal() as session: + raw_mapping = await StickySessionsRepository(session).get_account_id_and_abandonment( + raw_session, + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ) + assert raw_mapping.account_id == owner.id + assert raw_mapping.continuity_abandoned is False @pytest.mark.asyncio -async def test_backend_responses_http_bridge_refresh_failure_returns_proxy_error(async_client, monkeypatch): +async def test_backend_responses_http_bridge_file_owner_overrides_soft_locality( + async_client, + monkeypatch, +): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_backend_http_bridge_refresh_failure", - "backend-http-bridge-refresh-failure@example.com", + "acc_backend_http_bridge_file_owner", + "backend-http-bridge-file-owner@example.com", ) account = await _get_account(account_id) + service = get_proxy_service_for_app(async_client._transport.app) + await service._pin_file_account("file_bridge_owner", account.id) + fake_upstream = _FakeBridgeUpstreamWebSocket() + selection_calls: list[dict[str, object]] = [] - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - ) + async def fake_select_account(**kwargs: object) -> AccountSelection: + selection_calls.append(dict(kwargs)) return AccountSelection(account=account, error_message=None, error_code=None) - async def fail_refresh(self, target, *, force=False, timeout_seconds): - del self, target, force, timeout_seconds - raise proxy_module.RefreshError("refresh_token_expired", "token expired", True) + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fail_refresh) + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + return fake_upstream - response = await async_client.post( + monkeypatch.setattr(service._load_balancer, "select_account", fake_select_account) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + events = await _collect_sse_events( + async_client, "/backend-api/codex/responses", - json={ + headers={"session_id": "bridge-soft-session"}, + json_body={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": "hello", + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Read the file."}, + {"type": "input_file", "file_id": "file_bridge_owner"}, + ], + } + ], + "prompt_cache_key": "bridge-soft-cache", "stream": True, }, ) - assert response.status_code == 401 - assert response.json()["error"]["code"] == "invalid_api_key" - assert "x-codex-turn-state" not in response.headers + _assert_created_text_delta_completed(events) + assert len(selection_calls) == 1 + assert selection_calls[0]["account_ids"] is None + assert selection_calls[0]["required_account_id"] == account.id + assert selection_calls[0]["sticky_key"] is None + assert len(fake_upstream.sent_text) == 1 @pytest.mark.asyncio -async def test_v1_responses_http_bridge_refresh_failure_returns_proxy_error(async_client, monkeypatch): +@pytest.mark.parametrize( + ("second_model", "expected_connection_count"), + [ + pytest.param("gpt-5.1", 1, id="same-model-reuse"), + pytest.param("gpt-5.4", 2, id="model-transition-fork"), + ], +) +async def test_backend_responses_http_emits_turn_state_header_and_reuses_when_compatible( + async_client, + monkeypatch, + second_model: str, + expected_connection_count: int, +): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_v1_http_bridge_refresh_failure", - "v1-http-bridge-refresh-failure@example.com", + "acc_backend_http_bridge_turn_state", + "backend-http-bridge-turn-state@example.com", ) account = await _get_account(account_id) + available_upstreams = deque(_FakeBridgeUpstreamWebSocket() for _ in range(3)) + connected_upstreams: list[_FakeBridgeUpstreamWebSocket] = [] + connect_calls: list[tuple[str | None, proxy_module.StickySessionKind | None]] = [] async def fake_select_account_with_budget( self, @@ -9768,9 +6560,6 @@ async def fake_select_account_with_budget( deadline, request_id, kind, - request_stage, - sticky_key, - sticky_kind, reallocate_sticky, sticky_max_age_seconds, prefer_earlier_reset_accounts, @@ -9779,38 +6568,98 @@ async def fake_select_account_with_budget( exclude_account_ids, additional_limit_name, ) + connect_calls.append((sticky_key, sticky_kind)) return AccountSelection(account=account, error_message=None, error_code=None) - async def fail_refresh(self, target, *, force=False, timeout_seconds): - del self, target, force, timeout_seconds - raise proxy_module.RefreshError("refresh_token_expired", "token expired", True) + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + upstream = available_upstreams.popleft() + connected_upstreams.append(upstream) + return upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fail_refresh) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - response = await async_client.post( - "/v1/responses", - json={ + first_events, first_headers = await _collect_sse_events_with_headers( + async_client, + "/backend-api/codex/responses", + json_body={ "model": "gpt-5.1", "instructions": "Return exactly OK.", "input": "hello", + "prompt_cache_key": "backend-http-turn-state-a", + "stream": True, }, ) + turn_state = first_headers["x-codex-turn-state"] + first_response = first_events[-1]["response"] - assert response.status_code == 401 - assert response.json()["error"]["code"] == "invalid_api_key" - assert "x-codex-turn-state" not in response.headers + second_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": second_model, + "instructions": "Return exactly OK.", + "input": "hello-again", + "prompt_cache_key": "backend-http-turn-state-b", + "previous_response_id": first_response["id"], + "stream": True, + }, + headers={"x-codex-turn-state": turn_state}, + ) + third_events: list[dict] | None = None + if second_model != "gpt-5.1": + third_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello-on-original-model", + "prompt_cache_key": "backend-http-turn-state-c", + "stream": True, + }, + headers={"x-codex-turn-state": turn_state}, + ) + + _assert_created_text_delta_completed(first_events) + _assert_created_text_delta_completed(second_events) + if third_events is not None: + _assert_created_text_delta_completed(third_events) + assert turn_state.startswith("http_turn_") + assert connect_calls[0] == ("backend-http-turn-state-a", proxy_module.StickySessionKind.PROMPT_CACHE) + assert len(connect_calls) == expected_connection_count + expected_request_counts = [2] if expected_connection_count == 1 else [2, 1] + assert [len(upstream.sent_text) for upstream in connected_upstreams] == expected_request_counts + assert connected_upstreams[0].closed is False @pytest.mark.asyncio -async def test_v1_responses_http_bridge_transient_refresh_failure_returns_upstream_error(async_client, monkeypatch): +async def test_v1_responses_http_bridge_reuses_session_across_model_change_for_previous_response_id( + async_client, + monkeypatch, +): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_v1_http_bridge_refresh_transient_failure", - "v1-http-bridge-refresh-transient-failure@example.com", + "acc_http_bridge_model_change", + "http-bridge-model-change@example.com", ) account = await _get_account(account_id) + fake_upstream = _FakeBridgeUpstreamWebSocket() + connect_calls: list[tuple[str | None, str | None]] = [] async def fake_select_account_with_budget( self, @@ -9850,42 +6699,68 @@ async def fake_select_account_with_budget( ) return AccountSelection(account=account, error_message=None, error_code=None) - async def fail_refresh(self, target, *, force=False, timeout_seconds): - del self, target, force, timeout_seconds - raise proxy_module.RefreshError("invalid_response", "temporary refresh failure", False) + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, base_url, session + connect_calls.append((account_id, account_id_header)) + return fake_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fail_refresh) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - response = await async_client.post( + first = await async_client.post( "/v1/responses", json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", "input": "hello", + "prompt_cache_key": "http-bridge-model-thread", }, ) + assert first.status_code == 200 + first_body = first.json() - assert response.status_code == 502 - assert response.json()["error"]["code"] == "upstream_unavailable" - assert "x-codex-turn-state" not in response.headers + second = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.4", + "instructions": "Return exactly OK.", + "input": "hello again", + "prompt_cache_key": "http-bridge-model-thread", + "previous_response_id": first_body["id"], + }, + ) + assert second.status_code == 200 + + assert connect_calls == [(account_id, account.chatgpt_account_id)] + assert len(fake_upstream.sent_text) == 2 + second_payload = json.loads(fake_upstream.sent_text[1]) + assert second_payload["model"] == "gpt-5.4" + assert second_payload["previous_response_id"] == first_body["id"] @pytest.mark.asyncio -async def test_v1_responses_http_bridge_does_not_register_turn_state_alias_before_request_admission( - async_client, - app_instance, - monkeypatch, -): +async def test_v1_responses_http_bridge_recovers_previous_response_id_across_key_drift(async_client, monkeypatch): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_alias_after_admission", - "http-bridge-alias-after-admission@example.com", + "acc_http_bridge_live_session_required", + "http-bridge-live-session-required@example.com", ) - service = get_proxy_service_for_app(app_instance) account = await _get_account(account_id) - upstream = _SilentUpstreamWebSocket() + fake_upstream = _FakeBridgeUpstreamWebSocket() + connect_count = 0 async def fake_select_account_with_budget( self, @@ -9938,79 +6813,53 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - return upstream - - async def fake_submit_http_bridge_request( - self, - session, - *, - request_state, - text_data, - queue_limit, - ): - del self, session, request_state, text_data, queue_limit - raise proxy_module.ProxyResponseError( - 429, - proxy_module.openai_error( - "rate_limit_exceeded", - "HTTP responses session bridge queue is full", - error_type="rate_limit_error", - ), - ) + nonlocal connect_count + connect_count += 1 + return fake_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(proxy_module.ProxyService, "_submit_http_bridge_request", fake_submit_http_bridge_request) - payload = proxy_module.ResponsesRequest( - model="gpt-5.1", - instructions="Return exactly OK.", - input="hello", - prompt_cache_key="bridge-alias-after-admission", - ) - stream = service.stream_http_responses( - payload, - {}, - openai_cache_affinity=True, - downstream_turn_state="http_turn_unadmitted", + first = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + "prompt_cache_key": "http-bridge-live-session-a", + }, ) + assert first.status_code == 200 + first_body = first.json() - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await stream.__anext__() + second = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello-again", + "prompt_cache_key": "http-bridge-live-session-b", + "previous_response_id": first_body["id"], + }, + ) - exc = exc_info.value - assert exc.status_code == 429 - async with service._http_bridge_lock: - sessions = list(service._http_bridge_sessions.values()) - assert len(sessions) == 1 - bridge_session = sessions[0] - assert bridge_session.downstream_turn_state is None - assert bridge_session.downstream_turn_state_aliases == set() - assert service._http_bridge_turn_state_index == {} + assert second.status_code == 200 + assert second.json()["output"][0]["content"][0]["text"] == "OK" + assert connect_count == 1 @pytest.mark.asyncio -async def test_v1_responses_http_bridge_reconnects_after_clean_upstream_close(async_client, monkeypatch): - # Keep this reconnect contract on the instance that the application - # lifespan registered in the durable ring. Using the helper's synthetic - # ``instance-a`` default here creates an unrelated race: the clean-close - # reader can remove the local session just before its durable lease is - # released, and the next request then observes that lease as a foreign - # owner. Owner-mismatch behavior has dedicated tests; this one verifies - # that a clean upstream close reconnects transparently on one replica. - runtime_instance_id = proxy_module.get_settings().http_responses_session_bridge_instance_id - _install_bridge_settings_with_limits( - monkeypatch, - enabled=True, - instance_id=runtime_instance_id, +async def test_v1_responses_http_emits_turn_state_header_and_reuses_when_replayed(async_client, monkeypatch): + _install_bridge_settings(monkeypatch, enabled=True) + account_id = await _import_account( + async_client, + "acc_v1_http_bridge_turn_state", + "v1-http-bridge-turn-state@example.com", ) - account_id = await _import_account(async_client, "acc_http_bridge_reconnect", "http-bridge-reconnect@example.com") account = await _get_account(account_id) - first_upstream = _ClosingBridgeUpstreamWebSocket() - second_upstream = _FakeBridgeUpstreamWebSocket() - upstreams = [first_upstream, second_upstream] - connect_count = 0 + fake_upstream = _FakeBridgeUpstreamWebSocket() + connect_calls: list[tuple[str | None, proxy_module.StickySessionKind | None]] = [] async def fake_select_account_with_budget( self, @@ -10037,9 +6886,6 @@ async def fake_select_account_with_budget( deadline, request_id, kind, - request_stage, - sticky_key, - sticky_kind, reallocate_sticky, sticky_max_age_seconds, prefer_earlier_reset_accounts, @@ -10048,6 +6894,7 @@ async def fake_select_account_with_budget( exclude_account_ids, additional_limit_name, ) + connect_calls.append((sticky_key, sticky_kind)) return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): @@ -10062,50 +6909,49 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, account_id_header, base_url, session - nonlocal connect_count - upstream = upstreams[connect_count] - connect_count += 1 - return upstream - - async def fail_legacy_stream(*args, **kwargs): - raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + del headers, access_token, account_id_header, base_url, session + return fake_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - payload = { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-reconnect-thread-1", - } - first = await asyncio.wait_for(async_client.post("/v1/responses", json=payload), timeout=_TEST_SYNC_TIMEOUT_SECONDS) - second = await asyncio.wait_for( - async_client.post("/v1/responses", json=payload), timeout=_TEST_SYNC_TIMEOUT_SECONDS + first = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + "prompt_cache_key": "v1-http-turn-state-a", + }, ) - assert first.status_code == 200 + turn_state = first.headers["x-codex-turn-state"] + first_body = first.json() + + second = await async_client.post( + "/v1/responses", + headers={"x-codex-turn-state": turn_state}, + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello-again", + "prompt_cache_key": "v1-http-turn-state-b", + "previous_response_id": first_body["id"], + }, + ) assert second.status_code == 200 - assert connect_count == 2 + + assert turn_state.startswith("http_turn_") + assert connect_calls == [("v1-http-turn-state-a", proxy_module.StickySessionKind.PROMPT_CACHE)] @pytest.mark.asyncio -async def test_v1_responses_http_bridge_opens_fresh_session_for_previous_response_id_recovery( - async_client, monkeypatch -): +async def test_v1_responses_http_bridge_streaming_path_uses_persistent_upstream_websocket(async_client, monkeypatch): _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_previous_response_reconnect", - "http-bridge-previous-response-reconnect@example.com", - ) + account_id = await _import_account(async_client, "acc_http_bridge_sse", "http-bridge-sse@example.com") account = await _get_account(account_id) - first_upstream = _ClosingBridgeUpstreamWebSocket() - second_upstream = _FakeBridgeUpstreamWebSocket() - upstreams = [first_upstream, second_upstream] + fake_upstream = _FakeBridgeUpstreamWebSocket() connect_count = 0 async def fake_select_account_with_budget( @@ -10160,873 +7006,443 @@ async def fake_connect_responses_websocket( ): del headers, access_token, account_id_header, base_url, session nonlocal connect_count - upstream = upstreams[connect_count] connect_count += 1 - return upstream + return fake_upstream + + async def fail_legacy_stream(*args, **kwargs): + raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - first = await asyncio.wait_for( - async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-previous-response-reconnect", - }, - ), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, - ) - assert first.status_code == 200 - first_body = first.json() - - second = await asyncio.wait_for( - async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello-again", - "prompt_cache_key": "http-bridge-previous-response-reconnect", - "previous_response_id": first_body["id"], - }, - ), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, - ) + payload = { + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + "prompt_cache_key": "http-bridge-sse-thread-1", + "stream": True, + } + async with async_client.stream("POST", "/v1/responses", json=payload) as response: + assert response.status_code == 200 + lines = [line async for line in response.aiter_lines() if line.startswith("data: ")] - assert second.status_code == 200 - assert second.json()["output"][0]["content"][0]["text"] == "OK" - assert connect_count == 2 + events = [json.loads(line[6:]) for line in lines if line[6:] != "[DONE]"] + _assert_created_text_delta_completed(events) + assert connect_count == 1 -@pytest.mark.parametrize( - ("developer_message_extra", "fresh_developer_message", "leading_input_item", "preserves_full_resend"), - [ - pytest.param({}, None, None, True, id="unowned-developer-message"), - pytest.param({"id": "msg_response_owned"}, None, None, False, id="response-owned-developer-message"), - pytest.param( - {}, - { - "type": "message", - "role": "developer", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn_fresh"}, - "content": [{"type": "input_text", "text": "fresh control"}], - }, - None, - True, - id="fresh-developer-interleave", - ), - pytest.param( - {}, - None, - { - "role": "user", - "content": [{"type": "input_text", "text": "leading question"}], - }, - False, - id="lite-bundle-not-at-prefix-start", - ), - ], -) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_classifies_responses_lite_developer_interleaved_full_resend( - async_client, - app_instance, - monkeypatch, - developer_message_extra, - fresh_developer_message, - leading_input_item, - preserves_full_resend, -): - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_preserve_fresh_reattach", - "http-bridge-preserve-fresh-reattach@example.com", - ) - account = await _get_account(account_id) - first_upstream = _ClosingInterruptedCustomToolUpstreamWebSocket("resp_preserve_source") - replay_upstream = _FakeBridgeUpstreamWebSocket("resp_preserve_replay") - upstreams = [first_upstream, replay_upstream] - connect_headers: list[dict[str, str]] = [] - - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline, kwargs - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target +async def test_v1_responses_http_bridge_kill_switch_falls_back_to_legacy_path(async_client, monkeypatch): + _install_bridge_settings(monkeypatch, enabled=False) + await _import_account(async_client, "acc_http_bridge_fallback", "http-bridge-fallback@example.com") + seen = {"legacy": 0} - async def fake_connect_responses_websocket( + async def fake_legacy_stream( + payload, headers, access_token, - account_id_header, - *, + account_id, base_url=None, - session=None, + raise_for_status=False, + **_kw, ): - del access_token, account_id_header, base_url, session - connect_headers.append(dict(headers)) - return upstreams[len(connect_headers) - 1] + del headers, access_token, account_id, base_url, raise_for_status, _kw + seen["legacy"] += 1 + yield ( + 'data: {"type":"response.completed","response":{"id":"resp_legacy",' + '"object":"response","status":"completed",' + '"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2,"input_tokens_details":{"cached_tokens":0},' + '"output_tokens_details":{"reasoning_tokens":0}}}}\n\n' + ) - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + async def fail_connect(*args, **kwargs): + raise AssertionError("bridge websocket path must not be used when the kill switch disables it") - # Hold the old session's durable release across the replacement claim. - # Without an epoch advance, the old close can clear the new owner and the - # follow-up fails nondeterministically with bridge_instance_mismatch. - service = get_proxy_service_for_app(app_instance) - release_started = asyncio.Event() - allow_old_release = asyncio.Event() - old_release_finished = asyncio.Event() - replacement_claimed = asyncio.Event() - original_release = service._durable_bridge.release_live_session - original_claim = service._claim_durable_http_bridge_session - durable_claim_count = 0 + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_legacy_stream) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fail_connect) - async def gated_release_live_session(**kwargs): - release_started.set() - await _wait_for_event(allow_old_release) - try: - return await original_release(**kwargs) - finally: - old_release_finished.set() + response = await async_client.post("/v1/responses", json={"model": "gpt-5.1", "input": "hi"}) + assert response.status_code == 200 + assert response.json()["id"] == "resp_legacy" + assert "x-codex-turn-state" not in response.headers + assert seen["legacy"] == 1 - async def observed_claim_durable_http_bridge_session(target_session, **kwargs): - nonlocal durable_claim_count - await original_claim(target_session, **kwargs) - durable_claim_count += 1 - if durable_claim_count == 2: - replacement_claimed.set() - await _wait_for_event(old_release_finished) - monkeypatch.setattr(service._durable_bridge, "release_live_session", gated_release_live_session) - monkeypatch.setattr(service, "_claim_durable_http_bridge_session", observed_claim_durable_http_bridge_session) +@pytest.mark.asyncio +async def test_backend_responses_http_bridge_kill_switch_falls_back_to_legacy_path(async_client, monkeypatch): + _install_bridge_settings(monkeypatch, enabled=False) + await _import_account(async_client, "acc_backend_http_bridge_fallback", "backend-http-bridge-fallback@example.com") + seen = {"legacy": 0} - if developer_message_extra: - scenario_id = "response-owned-developer-message" - elif fresh_developer_message is not None: - scenario_id = "fresh-developer-interleave" - elif leading_input_item is not None: - scenario_id = "lite-bundle-not-at-prefix-start" - else: - scenario_id = "unowned-developer-message" - # Durable bridge ownership intentionally survives process-local session - # teardown. Give each independently parameterized scenario its own - # logical session so a delayed release from one case cannot fence the - # next case as though it were a cross-replica retry. - session_headers = {"x-codex-session-id": f"fresh-reattach-full-resend-{scenario_id}"} - historical_input = [ - *([leading_input_item] if leading_input_item is not None else []), - { - "type": "additional_tools", - "role": "developer", - "tools": [{"type": "custom", "name": "shell"}], - }, - { - "type": "message", - "role": "developer", - "content": [{"type": "input_text", "text": "canonical Lite instructions"}], - }, - { - "role": "user", - "content": [{"type": "input_text", "text": "first question"}], - }, - { - "type": "custom_tool_call", - "call_id": "call_historical_shell", - "name": "shell", - "input": "printf historical", - }, - { - "role": "developer", - "content": [{"type": "input_text", "text": "historical control"}], - **developer_message_extra, - }, - { - "type": "custom_tool_call_output", - "call_id": "call_historical_shell", - "output": "historical", - }, - ] - first = await asyncio.wait_for( - async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": historical_input, - }, - headers=session_headers, - ), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, + async def fake_legacy_stream( + payload, + headers, + access_token, + account_id, + base_url=None, + raise_for_status=False, + **_kw, + ): + del payload, headers, access_token, account_id, base_url, raise_for_status, _kw + seen["legacy"] += 1 + yield ( + 'data: {"type":"response.completed","response":{"id":"resp_backend_legacy",' + '"object":"response","status":"completed",' + '"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2,' + '"input_tokens_details":{"cached_tokens":0},"output_tokens_details":{"reasoning_tokens":0}}}}\n\n' + ) + + async def fail_connect(*args, **kwargs): + raise AssertionError("bridge websocket path must not be used when the kill switch disables it") + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_legacy_stream) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fail_connect) + + events, response_headers = await _collect_sse_events_with_headers( + async_client, + "/backend-api/codex/responses", + json_body={"model": "gpt-5.1", "instructions": "hi", "input": "hello", "stream": True}, ) - assert first.status_code == 200, first.text - full_resend = [ - *historical_input, - { - "type": "custom_tool_call", - "call_id": "call_custom_shell", - "name": "shell", - "input": "pwd", - }, - *([fresh_developer_message] if fresh_developer_message is not None else []), - { - "type": "custom_tool_call_output", - "call_id": "call_custom_shell", - "output": "/workspace", + assert [event["type"] for event in events] == ["response.completed"] + assert events[0]["response"]["id"] == "resp_backend_legacy" + assert "x-codex-turn-state" not in response_headers + assert seen["legacy"] == 1 + + +@pytest.mark.asyncio +async def test_backend_responses_http_bridge_startup_error_omits_turn_state_header(async_client, monkeypatch): + _install_bridge_settings(monkeypatch, enabled=True) + + response = await async_client.post( + "/backend-api/codex/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + "stream": True, }, - ] - await _wait_for_event(release_started) - second_task = asyncio.create_task( - async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": full_resend, - }, - headers=session_headers, - ) ) - await _wait_for_event(replacement_claimed) - allow_old_release.set() - second = await asyncio.wait_for(second_task, timeout=_TEST_SYNC_TIMEOUT_SECONDS) - assert second.status_code == 200, second.text - assert second.json()["id"] == "resp_preserve_replay_1" - assert len(connect_headers) == 2 - replay_connect_headers = {key.lower(): value for key, value in connect_headers[1].items()} - assert replay_connect_headers["x-codex-session-id"] == session_headers["x-codex-session-id"] - assert len(first_upstream.sent_text) == 1 - assert len(replay_upstream.sent_text) == 1 - replay_payload = json.loads(replay_upstream.sent_text[0]) - if preserves_full_resend: - assert "previous_response_id" not in replay_payload - assert replay_payload["input"] == full_resend - else: - assert replay_payload["previous_response_id"] == "resp_bridge_custom_1" + assert response.status_code == 503 + assert response.json()["error"]["code"] == "no_accounts" + assert "x-codex-turn-state" not in response.headers @pytest.mark.asyncio -async def test_http_bridge_replacement_does_not_steal_replica_claim_during_upstream_connect( - async_client, - app_instance, - monkeypatch, -): - service = get_proxy_service_for_app(app_instance) - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() +async def test_backend_responses_http_bridge_pool_usage_exhaustion_returns_429(async_client, monkeypatch): + _install_bridge_settings(monkeypatch, enabled=True) - settings = _make_app_settings( - enabled=True, - max_sessions=8, - admission_wait_timeout_seconds=1.0, - codex_idle_ttl_seconds=120.0, - instance_id="instance-a", - instance_ring=[], - ) - _install_proxy_settings( - monkeypatch, - app_settings=settings, - dashboard_settings=_make_dashboard_settings(), - ) - account_id = await _import_account( - async_client, - "acc_http_bridge_cross_replica_claim", - "http-bridge-cross-replica-claim@example.com", - ) - session_header = "cross-replica-claim-during-connect" - key = proxy_module._HTTPBridgeSessionKey("session_header", session_header, None) - predecessor = await service._durable_bridge.claim_live_session( - session_key_kind=key.affinity_kind, - session_key_value=key.affinity_key, - api_key_id=None, - instance_id="instance-a", - owner_process_epoch="process-a", - lease_ttl_seconds=60.0, - account_id=account_id, - model="gpt-5.4", - service_tier=None, - latest_turn_state=None, - latest_response_id=None, - allow_takeover=False, - ) - connect_started = asyncio.Event() - allow_connect = asyncio.Event() - - async def fake_create_http_bridge_session(self, target_key, **kwargs): - del self, kwargs - connect_started.set() - await _wait_for_event(allow_connect) - session = _make_dummy_bridge_session(target_key) - cast(Any, session).account = SimpleNamespace(id=account_id, status=AccountStatus.ACTIVE) - return session + async def fake_select_account_with_budget(*_args, **_kwargs): + return proxy_module.AccountSelection( + account=None, + error_message="Usage limit reached", + error_code="usage_limit_reached", + ) monkeypatch.setattr( proxy_module.ProxyService, - "_create_http_bridge_session", - fake_create_http_bridge_session, + "_select_account_with_budget", + fake_select_account_with_budget, ) - replacement = asyncio.create_task( - service._get_or_create_http_bridge_session( - key, - headers={"session_id": session_header}, - affinity=proxy_module._AffinityPolicy( - key=session_header, - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - durable_lookup=predecessor, - ) - ) - await _wait_for_event(connect_started) - released = await service._durable_bridge.release_live_session( - session_id=predecessor.session_id, - instance_id="instance-a", - owner_epoch=predecessor.owner_epoch, - draining=False, - ) - assert released is not None - competing_owner = await service._durable_bridge.claim_live_session( - session_key_kind=key.affinity_kind, - session_key_value=key.affinity_key, - api_key_id=None, - instance_id="instance-b", - owner_process_epoch="process-b", - lease_ttl_seconds=60.0, - account_id=account_id, - model="gpt-5.4", - service_tier=None, - latest_turn_state=None, - latest_response_id=None, - allow_takeover=False, + response = await async_client.post( + "/backend-api/codex/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + "stream": True, + }, ) - allow_connect.set() - - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await replacement - assert exc_info.value.status_code == 409 - assert exc_info.value.payload["error"]["code"] == "bridge_instance_mismatch" - snapshots = await service._durable_bridge.lookup_sessions(session_ids=[predecessor.session_id]) - assert len(snapshots) == 1 - assert snapshots[0].owner_instance_id == "instance-b" - assert snapshots[0].owner_epoch == competing_owner.owner_epoch + assert response.status_code == 429 + assert response.json()["error"]["type"] == "usage_limit_reached" + assert response.json()["error"]["code"] == "usage_limit_reached" + assert "x-codex-turn-state" not in response.headers @pytest.mark.asyncio -async def test_v1_responses_http_bridge_reports_unavailable_required_owner_when_other_account_exists( - async_client, monkeypatch -): +async def test_v1_responses_http_bridge_startup_error_omits_turn_state_header(async_client, monkeypatch): _install_bridge_settings(monkeypatch, enabled=True) - owner_account_id = await _import_account( - async_client, - "acc_http_bridge_required_owner", - "http-bridge-required-owner@example.com", - ) - alternate_account_id = await _import_account( - async_client, - "acc_http_bridge_available_other", - "http-bridge-available-other@example.com", + + response = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + }, ) - owner_account = await _get_account(owner_account_id) - alternate_account = await _get_account(alternate_account_id) - first_upstream = _ClosingBridgeUpstreamWebSocket() - selection_calls: list[tuple[str, str | None, bool, bool]] = [] - connect_count = 0 - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline - request_stage = cast(str, kwargs.get("request_stage", "first_turn")) - preferred_account_id = cast(str | None, kwargs.get("preferred_account_id")) - reallocate_sticky = bool(kwargs.get("reallocate_sticky")) - fallback_enabled = bool(kwargs.get("fallback_on_preferred_account_unavailable", True)) - selection_calls.append((request_stage, preferred_account_id, reallocate_sticky, fallback_enabled)) - if preferred_account_id is None: - return AccountSelection(account=owner_account, error_message=None, error_code=None) - if fallback_enabled: - return AccountSelection(account=alternate_account, error_message=None, error_code=None) - assert kwargs.get("preferred_account_is_continuity_owner") is True - return AccountSelection( - account=None, - error_message="Required continuity owner account no longer exists", - error_code=CONTINUITY_OWNER_UNAVAILABLE, - ) + assert response.status_code == 503 + assert response.json()["error"]["code"] == "no_accounts" + assert "x-codex-turn-state" not in response.headers - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, +@pytest.mark.asyncio +async def test_backend_responses_http_bridge_refresh_failure_returns_proxy_error(async_client, monkeypatch): + _install_bridge_settings(monkeypatch, enabled=True) + account_id = await _import_account( + async_client, + "acc_backend_http_bridge_refresh_failure", + "backend-http-bridge-refresh-failure@example.com", + ) + account = await _get_account(account_id) + + async def fake_select_account_with_budget( + self, + deadline, *, - base_url=None, - session=None, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, ): - del headers, access_token, account_id_header, base_url, session - nonlocal connect_count - connect_count += 1 - return first_upstream + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + ) + return AccountSelection(account=account, error_message=None, error_code=None) - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + async def fail_refresh(self, target, *, force=False, timeout_seconds): + del self, target, force, timeout_seconds + raise proxy_module.RefreshError("refresh_token_expired", "token expired", True) - first = await asyncio.wait_for( - async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-required-owner", - }, - ), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, - ) - assert first.status_code == 200 + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fail_refresh) - second = await asyncio.wait_for( - async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "continue", - "prompt_cache_key": "http-bridge-required-owner", - "previous_response_id": first.json()["id"], - }, - ), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, + response = await async_client.post( + "/backend-api/codex/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + "stream": True, + }, ) - assert second.status_code == 502 - assert second.json()["error"] == { - "message": "Previous response owner account is unavailable; retry later.", - "type": "server_error", - "code": "previous_response_owner_unavailable", - } - assert selection_calls == [ - ("first_turn", None, False, True), - ("follow_up", owner_account.id, False, False), - ] - assert connect_count == 1 + assert response.status_code == 401 + assert response.json()["error"]["code"] == "invalid_api_key" + assert "x-codex-turn-state" not in response.headers @pytest.mark.asyncio -async def test_backend_responses_soft_prompt_cache_follow_up_uses_durable_owner_over_stale_local_lane( - async_client, - app_instance, - monkeypatch, -): +async def test_v1_responses_http_bridge_refresh_failure_returns_proxy_error(async_client, monkeypatch): _install_bridge_settings(monkeypatch, enabled=True) - owner_account_id = await _import_account( - async_client, - "acc_backend_soft_owner", - "backend-soft-owner@example.com", - ) - stale_account_id = await _import_account( + account_id = await _import_account( async_client, - "acc_backend_soft_stale", - "backend-soft-stale@example.com", - ) - owner_account = await _get_account(owner_account_id) - stale_account = await _get_account(stale_account_id) - owner_chatgpt_account_id = cast(str, owner_account.chatgpt_account_id) - service = get_proxy_service_for_app(app_instance) - prompt_cache_key = "backend-soft-owner-route" - turn_state = "http_turn_backend_soft_owner" - await service._durable_bridge.claim_live_session( - session_key_kind="prompt_cache", - session_key_value=prompt_cache_key, - api_key_id=None, - instance_id="instance-a", - owner_process_epoch="test-process", - lease_ttl_seconds=60.0, - account_id=owner_account.id, - model="gpt-5.1", - service_tier=None, - latest_turn_state=turn_state, - latest_response_id="resp_backend_soft_previous", - allow_takeover=True, - ) - - key = proxy_module._HTTPBridgeSessionKey("prompt_cache", prompt_cache_key, None) - stale_upstream = _FakeBridgeUpstreamWebSocket("resp_backend_soft_stale") - stale_session = _make_dummy_bridge_session(key) - stale_session.account = stale_account - stale_session.upstream = cast(proxy_module.UpstreamWebSocket, stale_upstream) - stale_session.request_model = "gpt-5.1" - stale_session.affinity = proxy_module._AffinityPolicy( - key=prompt_cache_key, - kind=proxy_module.StickySessionKind.PROMPT_CACHE, + "acc_v1_http_bridge_refresh_failure", + "v1-http-bridge-refresh-failure@example.com", ) - service._http_bridge_sessions[key] = stale_session - - owner_upstream = _FakeBridgeUpstreamWebSocket("resp_backend_soft_owner") - connected_account_ids: list[str] = [] - - async def fake_ensure_fresh_with_budget(self, account, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return account + account = await _get_account(account_id) - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, + async def fake_select_account_with_budget( + self, + deadline, *, - base_url=None, - session=None, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, ): - del headers, access_token, base_url, session - connected_account_ids.append(account_id_header) - assert account_id_header == owner_chatgpt_account_id - return owner_upstream + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + ) + return AccountSelection(account=account, error_message=None, error_code=None) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + async def fail_refresh(self, target, *, force=False, timeout_seconds): + del self, target, force, timeout_seconds + raise proxy_module.RefreshError("refresh_token_expired", "token expired", True) - events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fail_refresh) + + response = await async_client.post( + "/v1/responses", + json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": "continue", - "prompt_cache_key": prompt_cache_key, - "stream": True, + "input": "hello", }, - headers={"x-codex-turn-state": turn_state}, ) - assert events[-1]["response"]["id"] == "resp_backend_soft_owner_1" - assert connected_account_ids == [owner_chatgpt_account_id] - assert stale_upstream.sent_text == [] - assert len(owner_upstream.sent_text) == 1 - assert stale_session.closed is True + assert response.status_code == 401 + assert response.json()["error"]["code"] == "invalid_api_key" + assert "x-codex-turn-state" not in response.headers -@pytest.mark.parametrize( - "fresh_developer_followup", - [ - pytest.param(False, id="ordinary-user-followup"), - pytest.param(True, id="fresh-developer-followup"), - ], -) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_replays_full_resend_once_then_stays_on_new_owner( - async_client, app_instance, monkeypatch, fresh_developer_followup -): +async def test_v1_responses_http_bridge_transient_refresh_failure_returns_upstream_error(async_client, monkeypatch): _install_bridge_settings(monkeypatch, enabled=True) - owner_account_id = await _import_account( - async_client, - "acc_http_bridge_replay_owner", - "http-bridge-replay-owner@example.com", - ) - alternate_account_id = await _import_account( + account_id = await _import_account( async_client, - "acc_http_bridge_replay_alternate", - "http-bridge-replay-alternate@example.com", + "acc_v1_http_bridge_refresh_transient_failure", + "v1-http-bridge-refresh-transient-failure@example.com", ) - owner_account = await _get_account(owner_account_id) - alternate_account = await _get_account(alternate_account_id) - owner_chatgpt_account_id = cast(str, owner_account.chatgpt_account_id) - alternate_chatgpt_account_id = cast(str, alternate_account.chatgpt_account_id) - owner_upstream = _ClosingBridgeUpstreamWebSocket("resp_owner") - alternate_upstream = _FakeBridgeUpstreamWebSocket("resp_alternate") - selection_calls: list[dict[str, object]] = [] - connected_account_ids: list[str] = [] - connect_headers_by_account: dict[str, dict[str, str]] = {} - - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline - selection_calls.append(dict(kwargs)) - preferred_account_id = cast(str | None, kwargs.get("preferred_account_id")) - excluded_account_ids = cast(set[str], kwargs.get("exclude_account_ids") or set()) - fallback_enabled = bool(kwargs.get("fallback_on_preferred_account_unavailable", True)) - if preferred_account_id == owner_account.id and not fallback_enabled: - assert kwargs.get("preferred_account_is_continuity_owner") is True - return AccountSelection( - account=None, - error_message="Required continuity owner account no longer exists", - error_code=CONTINUITY_OWNER_UNAVAILABLE, - ) - if owner_account.id in excluded_account_ids or preferred_account_id == alternate_account.id: - return AccountSelection(account=alternate_account, error_message=None, error_code=None) - return AccountSelection(account=owner_account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target + account = await _get_account(account_id) - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, + async def fake_select_account_with_budget( + self, + deadline, *, - base_url=None, - session=None, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, ): - del access_token, base_url, session - connected_account_ids.append(account_id_header) - connect_headers_by_account[account_id_header] = dict(headers) - if account_id_header == owner_chatgpt_account_id: - return owner_upstream - assert account_id_header == alternate_chatgpt_account_id - return alternate_upstream + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + ) + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fail_refresh(self, target, *, force=False, timeout_seconds): + del self, target, force, timeout_seconds + raise proxy_module.RefreshError("invalid_response", "temporary refresh failure", False) monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fail_refresh) - historical_input = [ - *( - [ - { - "type": "additional_tools", - "role": "developer", - "tools": [{"type": "custom", "name": "shell"}], - } - ] - if fresh_developer_followup - else [] - ), - { - "role": "user", - "content": [{"type": "input_text", "text": "first question"}], + response = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", }, - ] - first = await asyncio.wait_for( - async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": historical_input, - "prompt_cache_key": "http-bridge-full-replay", - }, - ), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, - ) - assert first.status_code == 200, first.text - service = get_proxy_service_for_app(app_instance) - durable_lookup = await service._durable_bridge.lookup_request_targets( - session_key_kind="prompt_cache", - session_key_value="http-bridge-full-replay", - api_key_id=None, - turn_state=None, - session_header=None, - previous_response_id=first.json()["id"], - ) - assert durable_lookup is not None - assert durable_lookup.latest_input_item_count == len(historical_input) - assert durable_lookup.latest_input_full_fingerprint is not None - assert durable_lookup.latest_response_transition_manifest is not None - - if fresh_developer_followup: - retained_prior_output = { - "type": "message", - "role": "assistant", - "phase": "final_answer", - "status": "completed", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn_previous"}, - "content": [{"type": "output_text", "text": "first answer"}], - } - fresh_followup_items = [ - { - "type": "message", - "role": "user", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn_current"}, - "content": [{"type": "input_text", "text": "second question"}], - }, - { - "type": "message", - "role": "developer", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn_current"}, - "content": [{"type": "input_text", "text": "fresh control"}], - }, - ] - else: - retained_prior_output = { - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "first answer"}], - } - fresh_followup_items = [ - { - "role": "user", - "content": [{"type": "input_text", "text": "second question"}], - } - ] - full_resend = [ - *historical_input, - retained_prior_output, - *fresh_followup_items, - ] - second = await asyncio.wait_for( - async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": full_resend, - "prompt_cache_key": "http-bridge-full-replay", - "previous_response_id": first.json()["id"], - }, - headers={ - "session_id": "stale-session", - "session-id": "stale-session-dash", - "thread-id": "stale-thread", - "x-codex-conversation-id": "stale-conversation", - "x-codex-session-id": "stale-codex-session", - "x-codex-turn-state": "http_turn_stale", - "x-request-trace": "keep-me", - }, - ), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, - ) - assert second.status_code == 200, second.text - assert second.json()["id"] == "resp_alternate_1" - - third = await asyncio.wait_for( - async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "third question", - "prompt_cache_key": "http-bridge-full-replay", - "previous_response_id": second.json()["id"], - }, - ), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - assert third.status_code == 200, third.text - assert third.json()["id"] == "resp_alternate_2" - assert connected_account_ids == [ - owner_chatgpt_account_id, - alternate_chatgpt_account_id, - ] - alternate_connect_headers = { - key.lower(): value for key, value in connect_headers_by_account[alternate_chatgpt_account_id].items() - } - assert alternate_connect_headers["x-request-trace"] == "keep-me" - assert ( - not { - "session_id", - "session-id", - "thread-id", - "x-codex-conversation-id", - "x-codex-session-id", - "x-codex-turn-state", - } - & alternate_connect_headers.keys() - ) - assert len(owner_upstream.sent_text) == 1 - assert len(alternate_upstream.sent_text) == 2 - replay_payload = json.loads(alternate_upstream.sent_text[0]) - assert "previous_response_id" not in replay_payload - assert replay_payload["input"] == full_resend - follow_up_payload = json.loads(alternate_upstream.sent_text[1]) - assert follow_up_payload["previous_response_id"] == second.json()["id"] - owner_miss = next( - call - for call in selection_calls - if call.get("preferred_account_id") == owner_account.id - and call.get("fallback_on_preferred_account_unavailable") is False - ) - assert owner_miss["preferred_account_is_continuity_owner"] is True + assert response.status_code == 502 + assert response.json()["error"]["code"] == "upstream_unavailable" + assert "x-codex-turn-state" not in response.headers @pytest.mark.asyncio -async def test_backend_responses_verified_full_resend_ignores_stale_broad_owner_on_durable_account( +async def test_v1_responses_http_bridge_does_not_register_turn_state_alias_before_request_admission( async_client, app_instance, monkeypatch, ): - _install_bridge_settings(monkeypatch, enabled=True) - owner_account_id = await _import_account( - async_client, - "acc_backend_durable_full_resend_owner", - "backend-durable-full-resend-owner@example.com", - ) - stale_account_id = await _import_account( - async_client, - "acc_backend_durable_full_resend_stale", - "backend-durable-full-resend-stale@example.com", - ) - owner_account = await _get_account(owner_account_id) - stale_account = await _get_account(stale_account_id) - owner_chatgpt_account_id = cast(str, owner_account.chatgpt_account_id) - service = get_proxy_service_for_app(app_instance) - session_id = "backend-durable-full-resend-session" - historical_input: list[proxy_module.JsonValue] = [ - { - "role": "user", - "content": [{"type": "input_text", "text": "first question"}], - } - ] - claimed = await service._durable_bridge.claim_live_session( - session_key_kind="session_header", - session_key_value=session_id, - api_key_id=None, - instance_id="instance-a", - owner_process_epoch="test-process", - lease_ttl_seconds=60.0, - account_id=owner_account.id, - model="gpt-5.1", - service_tier=None, - latest_turn_state="http_turn_durable_full_resend", - latest_response_id="resp_durable_full_resend_previous", - allow_takeover=True, - ) - renewed = await service._durable_bridge.renew_live_session( - session_id=claimed.session_id, - api_key_id=None, - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - lease_ttl_seconds=60.0, - latest_turn_state="http_turn_durable_full_resend", - latest_response_id="resp_durable_full_resend_previous", - latest_input_item_count=len(historical_input), - latest_input_full_fingerprint=proxy_module._fingerprint_input_items(historical_input), - latest_pending_tool_calls={}, - ) - assert renewed is not None - released = await service._durable_bridge.release_live_session( - session_id=claimed.session_id, - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - draining=False, + _install_bridge_settings(monkeypatch, enabled=True) + account_id = await _import_account( + async_client, + "acc_http_bridge_alias_after_admission", + "http-bridge-alias-after-admission@example.com", ) - assert released is not None - assert released.account_id == owner_account.id + service = get_proxy_service_for_app(app_instance) + account = await _get_account(account_id) + upstream = _SilentUpstreamWebSocket() - async with SessionLocal() as session: - await StickySessionsRepository(session).upsert( - session_id, - stale_account.id, - kind=proxy_module.StickySessionKind.CODEX_SESSION, + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, ) - - upstream = _FakeBridgeUpstreamWebSocket("resp_durable_full_resend") - connect_calls: list[tuple[dict[str, str], str]] = [] + return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds @@ -11040,109 +7456,213 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del access_token, base_url, session - connect_calls.append((dict(headers), account_id_header)) + del headers, access_token, account_id_header, base_url, session return upstream + async def fake_submit_http_bridge_request( + self, + session, + *, + request_state, + text_data, + queue_limit, + ): + del self, session, request_state, text_data, queue_limit + raise proxy_module.ProxyResponseError( + 429, + proxy_module.openai_error( + "rate_limit_exceeded", + "HTTP responses session bridge queue is full", + error_type="rate_limit_error", + ), + ) + + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module.ProxyService, "_submit_http_bridge_request", fake_submit_http_bridge_request) - full_resend = [ - *historical_input, - { - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "first answer"}], - }, - { - "role": "user", - "content": [{"type": "input_text", "text": "second question"}], - }, - ] - first_events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": full_resend, - "stream": True, - }, - headers={"session_id": session_id, "x-request-trace": "keep-me"}, + payload = proxy_module.ResponsesRequest( + model="gpt-5.1", + instructions="Return exactly OK.", + input="hello", + prompt_cache_key="bridge-alias-after-admission", ) - second_events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "third question", - "stream": True, - }, - headers={"session_id": session_id}, + stream = service.stream_http_responses( + payload, + {}, + openai_cache_affinity=True, + downstream_turn_state="http_turn_unadmitted", ) - _assert_created_text_delta_completed(first_events) - _assert_created_text_delta_completed(second_events) - assert connect_calls[0][1] == owner_chatgpt_account_id - assert len(connect_calls) == 1 - connect_headers = {key.lower(): value for key, value in connect_calls[0][0].items()} - assert connect_headers["x-request-trace"] == "keep-me" - assert ( - not { - "session_id", - "session-id", - "thread-id", - "x-codex-conversation-id", - "x-codex-session-id", - "x-codex-turn-state", - } - & connect_headers.keys() - ) - assert len(upstream.sent_text) == 2 - replay_payload = json.loads(upstream.sent_text[0]) - assert "previous_response_id" not in replay_payload - assert replay_payload["input"] == full_resend - bridge_key = proxy_module._HTTPBridgeSessionKey("session_header", session_id, None) - bridge_session = service._http_bridge_sessions[bridge_key] - assert bridge_session.account.id == owner_account.id - assert bridge_session.codex_session is True - assert bridge_session.affinity.kind == proxy_module.StickySessionKind.CODEX_SESSION - assert bridge_session.affinity.key is None - async with SessionLocal() as session: - assert ( - await StickySessionsRepository(session).get_account_id( - session_id, - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ) - == stale_account.id - ) + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await stream.__anext__() + + exc = exc_info.value + assert exc.status_code == 429 + async with service._http_bridge_lock: + sessions = list(service._http_bridge_sessions.values()) + assert len(sessions) == 1 + bridge_session = sessions[0] + assert bridge_session.downstream_turn_state is None + assert bridge_session.downstream_turn_state_aliases == set() + assert service._http_bridge_turn_state_index == {} @pytest.mark.asyncio -async def test_backend_responses_verified_full_resend_fails_over_to_new_account_after_owner_loss( - async_client, - app_instance, - monkeypatch, +async def test_v1_responses_http_bridge_reconnects_after_clean_upstream_close(async_client, monkeypatch): + # The app lifespan registers the process hostname in the durable bridge + # ring before this test installs its settings. Keep the test on that same + # instance so the startup heartbeat cannot make the reconnect path look + # like a cross-replica ownership conflict. + _install_bridge_settings_with_limits(monkeypatch, enabled=True, instance_id=socket.gethostname()) + account_id = await _import_account(async_client, "acc_http_bridge_reconnect", "http-bridge-reconnect@example.com") + account = await _get_account(account_id) + first_upstream = _ClosingBridgeUpstreamWebSocket() + second_upstream = _FakeBridgeUpstreamWebSocket() + upstreams = [first_upstream, second_upstream] + connect_count = 0 + + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + ) + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + nonlocal connect_count + upstream = upstreams[connect_count] + connect_count += 1 + return upstream + + async def fail_legacy_stream(*args, **kwargs): + raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) + + payload = { + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + # Scope the soft-affinity key to this test's account so a parallel or + # ordered integration run cannot inherit another instance's durable + # owner and turn the reconnect assertion into a 409 race. + "prompt_cache_key": f"http-bridge-reconnect-thread-{account_id}", + } + first = await asyncio.wait_for(async_client.post("/v1/responses", json=payload), timeout=_TEST_SYNC_TIMEOUT_SECONDS) + second = await asyncio.wait_for( + async_client.post("/v1/responses", json=payload), timeout=_TEST_SYNC_TIMEOUT_SECONDS + ) + + assert first.status_code == 200 + assert second.status_code == 200 + assert connect_count == 2 + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_opens_fresh_session_for_previous_response_id_recovery( + async_client, monkeypatch ): _install_bridge_settings(monkeypatch, enabled=True) - owner_account_id = await _import_account( + account_id = await _import_account( async_client, - "acc_backend_full_resend_failover_owner", - "backend-full-resend-failover-owner@example.com", + "acc_http_bridge_previous_response_reconnect", + "http-bridge-previous-response-reconnect@example.com", ) - owner_account = await _get_account(owner_account_id) - owner_chatgpt_account_id = cast(str, owner_account.chatgpt_account_id) - owner_upstream = _ClosingBridgeUpstreamWebSocket("resp_failover_owner") - alternate_upstream = _FakeBridgeUpstreamWebSocket("resp_failover_alternate") - connected_account_ids: list[str] = [] - connect_headers_by_account: dict[str, dict[str, str]] = {} - degraded_reasons: list[str] = [] + account = await _get_account(account_id) + first_upstream = _ClosingBridgeUpstreamWebSocket() + second_upstream = _FakeBridgeUpstreamWebSocket() + upstreams = [first_upstream, second_upstream] + connect_count = 0 - async def fake_ensure_fresh_with_budget(self, account, *, force=False, timeout_seconds): + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + ) + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds - return account + return target async def fake_connect_responses_websocket( headers, @@ -11152,129 +7672,126 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del access_token, base_url, session - connected_account_ids.append(account_id_header) - connect_headers_by_account[account_id_header] = dict(headers) - if account_id_header == owner_chatgpt_account_id: - return owner_upstream - return alternate_upstream + del headers, access_token, account_id_header, base_url, session + nonlocal connect_count + upstream = upstreams[connect_count] + connect_count += 1 + return upstream + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(load_balancer_module, "set_degraded", degraded_reasons.append) - - session_id = "backend-full-resend-failover-session" - historical_input: list[proxy_module.JsonValue] = [ - { - "role": "user", - "content": [{"type": "input_text", "text": "first question"}], - } - ] - first_events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": historical_input, - "stream": True, - }, - headers={"session_id": session_id}, - ) - first_response = first_events[-1]["response"] - assert first_response["id"] == "resp_failover_owner_1" - - service = get_proxy_service_for_app(app_instance) - durable_lookup = await service._durable_bridge.lookup_request_targets( - session_key_kind="session_header", - session_key_value=session_id, - api_key_id=None, - turn_state=None, - session_header=session_id, - previous_response_id=None, - ) - assert durable_lookup is not None - assert durable_lookup.account_id == owner_account.id - assert durable_lookup.latest_input_item_count == len(historical_input) - assert durable_lookup.latest_input_full_fingerprint is not None - alternate_account_id = await _import_account( - async_client, - "acc_backend_full_resend_failover_alternate", - "backend-full-resend-failover-alternate@example.com", + first = await asyncio.wait_for( + async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + "prompt_cache_key": "http-bridge-previous-response-reconnect", + }, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - alternate_account = await _get_account(alternate_account_id) - alternate_chatgpt_account_id = cast(str, alternate_account.chatgpt_account_id) - pause = await async_client.post(f"/api/accounts/{owner_account_id}/pause") - assert pause.status_code == 200, pause.text + assert first.status_code == 200 + first_body = first.json() - full_resend = [ - *historical_input, - first_response["output"][0], - { - "role": "user", - "content": [{"type": "input_text", "text": "second question"}], - }, - ] - second_events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": full_resend, - "stream": True, - }, - headers={"session_id": session_id, "x-request-trace": "keep-me"}, + second = await asyncio.wait_for( + async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello-again", + "prompt_cache_key": "http-bridge-previous-response-reconnect", + "previous_response_id": first_body["id"], + }, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - second_response = second_events[-1]["response"] - assert second_response["id"] == "resp_failover_alternate_1" - assert connected_account_ids == [owner_chatgpt_account_id, alternate_chatgpt_account_id] - alternate_connect_headers = { - key.lower(): value for key, value in connect_headers_by_account[alternate_chatgpt_account_id].items() - } - assert alternate_connect_headers["x-request-trace"] == "keep-me" - assert ( - not { - "session_id", - "session-id", - "thread-id", - "x-codex-conversation-id", - "x-codex-session-id", - "x-codex-turn-state", - } - & alternate_connect_headers.keys() - ) - assert len(owner_upstream.sent_text) == 1 - assert len(alternate_upstream.sent_text) == 1 - replay_payload = json.loads(alternate_upstream.sent_text[0]) - assert "previous_response_id" not in replay_payload - assert replay_payload["input"] == full_resend - assert degraded_reasons == [] + assert second.status_code == 200 + assert second.json()["output"][0]["content"][0]["text"] == "OK" + assert connect_count == 2 +@pytest.mark.parametrize( + ("developer_message_extra", "fresh_developer_message", "leading_input_item", "preserves_full_resend"), + [ + pytest.param({}, None, None, True, id="unowned-developer-message"), + pytest.param({"id": "msg_response_owned"}, None, None, False, id="response-owned-developer-message"), + pytest.param( + {}, + { + "type": "message", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_fresh"}, + "content": [{"type": "input_text", "text": "fresh control"}], + }, + None, + True, + id="fresh-developer-interleave", + ), + pytest.param( + {}, + None, + { + "role": "user", + "content": [{"type": "input_text", "text": "leading question"}], + }, + False, + id="lite-bundle-not-at-prefix-start", + ), + ], +) @pytest.mark.asyncio -async def test_backend_responses_http_bridge_real_selector_recovers_full_resend_without_degrading_pool( - async_client, monkeypatch +async def test_v1_responses_http_bridge_classifies_responses_lite_developer_interleaved_full_resend( + async_client, + app_instance, + monkeypatch, + developer_message_extra, + fresh_developer_message, + leading_input_item, + preserves_full_resend, ): _install_bridge_settings(monkeypatch, enabled=True) - owner_account_id = await _import_account( + account_id = await _import_account( async_client, - "acc_backend_replay_owner", - "backend-replay-owner@example.com", + "acc_http_bridge_preserve_fresh_reattach", + "http-bridge-preserve-fresh-reattach@example.com", ) - owner_account = await _get_account(owner_account_id) - owner_chatgpt_account_id = cast(str, owner_account.chatgpt_account_id) - owner_upstream = _ClosingBridgeUpstreamWebSocket("resp_backend_owner") - alternate_upstream = _FakeBridgeUpstreamWebSocket("resp_backend_alternate") - connected_account_ids: list[str] = [] - connect_headers_by_account: dict[str, dict[str, str]] = {} - degraded_reasons: list[str] = [] + account = await _get_account(account_id) + first_upstream = _ClosingInterruptedCustomToolUpstreamWebSocket("resp_preserve_source") + replay_upstream = _FakeBridgeUpstreamWebSocket("resp_preserve_replay") + upstreams = [first_upstream, replay_upstream] + connect_headers: list[dict[str, str]] = [] + service = get_proxy_service_for_app(app_instance) + predecessor_release_started = asyncio.Event() + allow_predecessor_release = asyncio.Event() + replacement_claimed = asyncio.Event() + original_release_live_session = service._durable_bridge.release_live_session + original_claim_live_session = service._durable_bridge.claim_live_session + + async def delay_predecessor_release(**kwargs): + if kwargs["owner_epoch"] == 1 and not predecessor_release_started.is_set(): + predecessor_release_started.set() + await allow_predecessor_release.wait() + return await original_release_live_session(**kwargs) + + async def observe_replacement_claim(**kwargs): + lookup = await original_claim_live_session(**kwargs) + if lookup.owner_epoch > 1: + replacement_claimed.set() + return lookup - async def fake_ensure_fresh_with_budget(self, account, *, force=False, timeout_seconds): + async def fake_select_account_with_budget(self, deadline, **kwargs): + del self, deadline, kwargs + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds - return account + return target async def fake_connect_responses_websocket( headers, @@ -11284,310 +7801,272 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del access_token, base_url, session - connected_account_ids.append(account_id_header) - connect_headers_by_account[account_id_header] = dict(headers) - if account_id_header == owner_chatgpt_account_id: - return owner_upstream - return alternate_upstream + del access_token, account_id_header, base_url, session + connect_headers.append(dict(headers)) + return upstreams[len(connect_headers) - 1] + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(load_balancer_module, "set_degraded", degraded_reasons.append) + monkeypatch.setattr(service._durable_bridge, "release_live_session", delay_predecessor_release) + monkeypatch.setattr(service._durable_bridge, "claim_live_session", observe_replacement_claim) + session_headers = {"x-codex-session-id": "fresh-reattach-full-resend"} historical_input = [ + *([leading_input_item] if leading_input_item is not None else []), + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "custom", "name": "shell"}], + }, + { + "type": "message", + "role": "developer", + "content": [{"type": "input_text", "text": "canonical Lite instructions"}], + }, { "role": "user", "content": [{"type": "input_text", "text": "first question"}], - } - ] - first_events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": historical_input, - "prompt_cache_key": "backend-http-bridge-full-replay", - "stream": True, }, + { + "type": "custom_tool_call", + "call_id": "call_historical_shell", + "name": "shell", + "input": "printf historical", + }, + { + "role": "developer", + "content": [{"type": "input_text", "text": "historical control"}], + **developer_message_extra, + }, + { + "type": "custom_tool_call_output", + "call_id": "call_historical_shell", + "output": "historical", + }, + ] + first = await asyncio.wait_for( + async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": historical_input, + }, + headers=session_headers, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - first_response = first_events[-1]["response"] - assert first_response["id"] == "resp_backend_owner_1" - - alternate_account_id = await _import_account( - async_client, - "acc_backend_replay_alternate", - "backend-replay-alternate@example.com", - ) - alternate_account = await _get_account(alternate_account_id) - alternate_chatgpt_account_id = cast(str, alternate_account.chatgpt_account_id) - pause = await async_client.post(f"/api/accounts/{owner_account_id}/pause") - assert pause.status_code == 200, pause.text + assert first.status_code == 200, first.text + await asyncio.wait_for(predecessor_release_started.wait(), timeout=_TEST_SYNC_TIMEOUT_SECONDS) - retained_prior_output = first_response["output"][0] full_resend = [ *historical_input, - retained_prior_output, { - "role": "user", - "content": [{"type": "input_text", "text": "second question"}], - }, - ] - stale_headers = { - "session_id": "stale-session", - "session-id": "stale-session-dash", - "thread-id": "stale-thread", - "x-codex-conversation-id": "stale-conversation", - "x-codex-session-id": "stale-codex-session", - "x-codex-turn-state": "http_turn_stale", - "x-request-trace": "keep-me", - } - second_events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": full_resend, - "prompt_cache_key": "backend-http-bridge-full-replay", - "previous_response_id": first_response["id"], - "stream": True, + "type": "custom_tool_call", + "call_id": "call_custom_shell", + "name": "shell", + "input": "pwd", }, - headers=stale_headers, - ) - second_response = second_events[-1]["response"] - assert second_response["id"] == "resp_backend_alternate_1" - - third_events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "third question", - "prompt_cache_key": "backend-http-bridge-full-replay", - "previous_response_id": second_response["id"], - "stream": True, + *([fresh_developer_message] if fresh_developer_message is not None else []), + { + "type": "custom_tool_call_output", + "call_id": "call_custom_shell", + "output": "/workspace", }, + ] + second_task = asyncio.create_task( + async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": full_resend, + }, + headers=session_headers, + ) ) - assert third_events[-1]["response"]["id"] == "resp_backend_alternate_2" + try: + await asyncio.wait_for(replacement_claimed.wait(), timeout=_TEST_SYNC_TIMEOUT_SECONDS) + except BaseException: + allow_predecessor_release.set() + second_task.cancel() + try: + await second_task + except asyncio.CancelledError: + pass + raise + allow_predecessor_release.set() + second = await asyncio.wait_for(second_task, timeout=_TEST_SYNC_TIMEOUT_SECONDS) - assert connected_account_ids == [owner_chatgpt_account_id, alternate_chatgpt_account_id] - alternate_connect_headers = { - key.lower(): value for key, value in connect_headers_by_account[alternate_chatgpt_account_id].items() - } - assert alternate_connect_headers["x-request-trace"] == "keep-me" - assert ( - not { - "session_id", - "session-id", - "thread-id", - "x-codex-conversation-id", - "x-codex-session-id", - "x-codex-turn-state", - } - & alternate_connect_headers.keys() - ) - assert len(owner_upstream.sent_text) == 1 - assert len(alternate_upstream.sent_text) == 2 - replay_payload = json.loads(alternate_upstream.sent_text[0]) - assert "previous_response_id" not in replay_payload - assert replay_payload["input"] == full_resend - follow_up_payload = json.loads(alternate_upstream.sent_text[1]) - assert follow_up_payload["previous_response_id"] == second_response["id"] - assert degraded_reasons == [] + assert second.status_code == 200, second.text + assert second.json()["id"] == "resp_preserve_replay_1" + assert len(connect_headers) == 2 + replay_connect_headers = {key.lower(): value for key, value in connect_headers[1].items()} + assert replay_connect_headers["x-codex-session-id"] == session_headers["x-codex-session-id"] + assert len(first_upstream.sent_text) == 1 + assert len(replay_upstream.sent_text) == 1 + replay_payload = json.loads(replay_upstream.sent_text[0]) + if preserves_full_resend: + assert "previous_response_id" not in replay_payload + assert replay_payload["input"] == full_resend + else: + assert replay_payload["previous_response_id"] == "resp_bridge_custom_1" @pytest.mark.asyncio -async def test_backend_responses_http_bridge_declines_cross_account_anchor_and_settles( - async_client, app_instance, monkeypatch +async def test_v1_responses_http_bridge_reports_unavailable_required_owner_when_other_account_exists( + async_client, monkeypatch ): - """A restored durable anchor owned by another account must not be injected. - - The durable record still names the owner account, but the owner is out of the - rotation so the bridge session is created on another account. Replaying the - owner's ``previous_response_id`` there would send an anchor upstream cannot - resolve with the history trimmed away: upstream never emits - ``response.created`` and the per-bridge response-create gate wedges. The turn - must go upstream as a full-history resend instead, and it must settle. - """ - _install_bridge_settings(monkeypatch, enabled=True) owner_account_id = await _import_account( async_client, - "acc_cross_account_anchor_owner", - "cross-account-anchor-owner@example.com", + "acc_http_bridge_required_owner", + "http-bridge-required-owner@example.com", ) - serving_account_id = await _import_account( + alternate_account_id = await _import_account( async_client, - "acc_cross_account_anchor_serving", - "cross-account-anchor-serving@example.com", + "acc_http_bridge_available_other", + "http-bridge-available-other@example.com", ) - owner_account = await _get_account(owner_account_id) - serving_account = await _get_account(serving_account_id) - serving_chatgpt_account_id = cast(str, serving_account.chatgpt_account_id) - serving_upstream = _AccountScopedAnchorUpstreamWebSocket("resp_cross_account_serving") - service = get_proxy_service_for_app(app_instance) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, base_url, session - assert account_id_header == serving_chatgpt_account_id - return serving_upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - # The durable owner account left the rotation, so selection lands on the - # other account while the restored durable record still names the owner. - pause = await async_client.post(f"/api/accounts/{owner_account_id}/pause") - assert pause.status_code == 200, pause.text - - stored_input: list[proxy_module.JsonValue] = [ - {"role": "user", "content": [{"type": "input_text", "text": "first question"}]}, - ] - - def _durable_record( - *, - account_id: str, - latest_response_id: str, - stored_items: list[proxy_module.JsonValue], - ) -> proxy_module.DurableBridgeLookup: - return proxy_module.DurableBridgeLookup( - session_id="durable-cross-account-anchor", - canonical_kind="prompt_cache", - canonical_key="cross-account-anchor-cache-key", - api_key_scope="__anonymous__", - account_id=account_id, - owner_instance_id=None, - owner_epoch=1, - lease_expires_at=None, - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state=None, - latest_response_id=latest_response_id, - latest_input_item_count=len(stored_items), - latest_input_full_fingerprint=proxy_module._fingerprint_input_items(stored_items), + owner_account = await _get_account(owner_account_id) + alternate_account = await _get_account(alternate_account_id) + first_upstream = _ClosingBridgeUpstreamWebSocket() + selection_calls: list[tuple[str, str | None, bool, bool]] = [] + connect_count = 0 + + async def fake_select_account_with_budget(self, deadline, **kwargs): + del self, deadline + request_stage = cast(str, kwargs.get("request_stage", "first_turn")) + preferred_account_id = cast(str | None, kwargs.get("preferred_account_id")) + reallocate_sticky = bool(kwargs.get("reallocate_sticky")) + fallback_enabled = bool(kwargs.get("fallback_on_preferred_account_unavailable", True)) + selection_calls.append((request_stage, preferred_account_id, reallocate_sticky, fallback_enabled)) + if preferred_account_id is None: + return AccountSelection(account=owner_account, error_message=None, error_code=None) + if fallback_enabled: + return AccountSelection(account=alternate_account, error_message=None, error_code=None) + assert kwargs.get("preferred_account_is_continuity_owner") is True + return AccountSelection( + account=None, + error_message="Required continuity owner account no longer exists", + error_code=CONTINUITY_OWNER_UNAVAILABLE, ) - durable_record = _durable_record( - account_id=owner_account.id, - latest_response_id="resp_cross_account_owner_1", - stored_items=stored_input, - ) + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target - async def fake_lookup_request_targets(**kwargs): - del kwargs - return durable_record + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + nonlocal connect_count + connect_count += 1 + return first_upstream - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", fake_lookup_request_targets) + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - # Compaction-shaped follow-up: the stored prefix still matches, but the - # suffix carries no prior assistant output, so the account-neutral fresh - # resend projection is unavailable and the restored durable anchor is the - # only continuity candidate the session-level injection can reach for. - compacted_resend: list[proxy_module.JsonValue] = [ - *stored_input, - {"role": "user", "content": [{"type": "input_text", "text": "second question"}]}, - ] - session_id = "cross-account-anchor-session" - first_events, first_headers = await asyncio.wait_for( - _collect_sse_events_with_headers( - async_client, - "/backend-api/codex/responses", - json_body={ + first = await asyncio.wait_for( + async_client.post( + "/v1/responses", + json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": compacted_resend, - "stream": True, + "input": "hello", + "prompt_cache_key": "http-bridge-required-owner", }, - headers={"session_id": session_id}, ), timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - _assert_created_text_delta_completed(first_events) - turn_state = first_headers["x-codex-turn-state"] - - assert len(serving_upstream.sent_text) == 1 - resend_payload = json.loads(serving_upstream.sent_text[0]) - assert "previous_response_id" not in resend_payload - assert resend_payload["input"] == compacted_resend - - bridge_session = next( - candidate for candidate in service._http_bridge_sessions.values() if candidate.account.id == serving_account.id - ) - assert bridge_session.codex_session is True - # The turn settled on the serving account, so the gate is free and the - # session anchor is now owned by the account that actually created it. - assert bridge_session.response_create_gate.locked() is False - assert bridge_session.last_completed_response_id == "resp_cross_account_serving_1" - assert bridge_session.last_completed_response_account_id == serving_account.id + assert first.status_code == 200 - # Same-account continuity is untouched: once the durable record names the - # account that actually created the response, the very next turn anchors on - # it instead of resending the whole history. - durable_record = _durable_record( - account_id=serving_account.id, - latest_response_id="resp_cross_account_serving_1", - stored_items=compacted_resend, - ) - second_events = await asyncio.wait_for( - _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ + second = await asyncio.wait_for( + async_client.post( + "/v1/responses", + json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": [ - *compacted_resend, - { - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "OK"}], - }, - {"role": "user", "content": [{"type": "input_text", "text": "third question"}]}, - ], - "stream": True, + "input": "continue", + "prompt_cache_key": "http-bridge-required-owner", + "previous_response_id": first.json()["id"], }, - headers={"session_id": session_id, "x-codex-turn-state": turn_state}, ), timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - _assert_created_text_delta_completed(second_events) - assert len(serving_upstream.sent_text) == 2 - follow_up_payload = json.loads(serving_upstream.sent_text[1]) - assert follow_up_payload["previous_response_id"] == "resp_cross_account_serving_1" - assert bridge_session.response_create_gate.locked() is False + + assert second.status_code == 502 + assert second.json()["error"] == { + "message": "Previous response owner account is unavailable; retry later.", + "type": "server_error", + "code": "previous_response_owner_unavailable", + } + assert selection_calls == [ + ("first_turn", None, False, True), + ("follow_up", owner_account.id, False, False), + ] + assert connect_count == 1 @pytest.mark.asyncio -async def test_backend_responses_projects_retained_encrypted_reasoning_before_replaying_to_available_account( +async def test_backend_responses_soft_prompt_cache_follow_up_uses_durable_owner_over_stale_local_lane( async_client, + app_instance, monkeypatch, ): _install_bridge_settings(monkeypatch, enabled=True) owner_account_id = await _import_account( async_client, - "acc_backend_encrypted_owner", - "backend-encrypted-owner@example.com", + "acc_backend_soft_owner", + "backend-soft-owner@example.com", + ) + stale_account_id = await _import_account( + async_client, + "acc_backend_soft_stale", + "backend-soft-stale@example.com", ) owner_account = await _get_account(owner_account_id) + stale_account = await _get_account(stale_account_id) owner_chatgpt_account_id = cast(str, owner_account.chatgpt_account_id) - owner_upstream = _ClosingBridgeUpstreamWebSocket("resp_backend_encrypted_owner") - alternate_upstream = _FakeBridgeUpstreamWebSocket("resp_backend_encrypted_alternate") + service = get_proxy_service_for_app(app_instance) + prompt_cache_key = "backend-soft-owner-route" + turn_state = "http_turn_backend_soft_owner" + await service._durable_bridge.claim_live_session( + session_key_kind="prompt_cache", + session_key_value=prompt_cache_key, + api_key_id=None, + instance_id="instance-a", + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id=owner_account.id, + model="gpt-5.1", + service_tier=None, + latest_turn_state=turn_state, + latest_response_id="resp_backend_soft_previous", + allow_takeover=True, + ) + + key = proxy_module._HTTPBridgeSessionKey("prompt_cache", prompt_cache_key, None) + stale_upstream = _FakeBridgeUpstreamWebSocket("resp_backend_soft_stale") + stale_session = _make_dummy_bridge_session(key) + stale_session.account = stale_account + stale_session.upstream = cast(proxy_module.UpstreamWebSocket, stale_upstream) + stale_session.request_model = "gpt-5.1" + stale_session.affinity = proxy_module._AffinityPolicy( + key=prompt_cache_key, + kind=proxy_module.StickySessionKind.PROMPT_CACHE, + ) + service._http_bridge_sessions[key] = stale_session + + owner_upstream = _FakeBridgeUpstreamWebSocket("resp_backend_soft_owner") connected_account_ids: list[str] = [] - degraded_reasons: list[str] = [] async def fake_ensure_fresh_with_budget(self, account, *, force=False, timeout_seconds): del self, force, timeout_seconds @@ -11603,135 +8082,80 @@ async def fake_connect_responses_websocket( ): del headers, access_token, base_url, session connected_account_ids.append(account_id_header) - if account_id_header == owner_chatgpt_account_id: - return owner_upstream - return alternate_upstream + assert account_id_header == owner_chatgpt_account_id + return owner_upstream monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(load_balancer_module, "set_degraded", degraded_reasons.append) - historical_input = [ - { - "role": "user", - "content": [{"type": "input_text", "text": "first question"}], - } - ] - first_events = await _collect_sse_events( - async_client, - "/backend-api/codex/responses", - json_body={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": historical_input, - "prompt_cache_key": "backend-http-bridge-encrypted-replay", - "stream": True, - }, - ) - first_response = first_events[-1]["response"] - - alternate_account_id = await _import_account( - async_client, - "acc_backend_encrypted_alternate", - "backend-encrypted-alternate@example.com", - ) - alternate_account = await _get_account(alternate_account_id) - alternate_chatgpt_account_id = cast(str, alternate_account.chatgpt_account_id) - pause = await async_client.post(f"/api/accounts/{owner_account_id}/pause") - assert pause.status_code == 200, pause.text - - full_resend = [ - *historical_input, - { - "type": "reasoning", - "id": "rs_owner_scoped", - "encrypted_content": "owner-scoped-ciphertext", - "summary": [], - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-owner"}, - }, - { - "type": "web_search_call", - "id": "ws_owner_scoped", - "action": {"type": "search", "query": "portable result"}, - "status": "completed", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-owner"}, - }, - first_response["output"][0], - { - "role": "user", - "content": [{"type": "input_text", "text": "second question"}], - }, - ] - second_events = await _collect_sse_events( + events = await _collect_sse_events( async_client, "/backend-api/codex/responses", json_body={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": full_resend, - "prompt_cache_key": "backend-http-bridge-encrypted-replay", - "previous_response_id": first_response["id"], + "input": "continue", + "prompt_cache_key": prompt_cache_key, "stream": True, }, + headers={"x-codex-turn-state": turn_state}, ) - assert second_events[-1]["response"]["id"] == "resp_backend_encrypted_alternate_1" - assert connected_account_ids == [owner_chatgpt_account_id, alternate_chatgpt_account_id] + assert events[-1]["response"]["id"] == "resp_backend_soft_owner_1" + assert connected_account_ids == [owner_chatgpt_account_id] + assert stale_upstream.sent_text == [] assert len(owner_upstream.sent_text) == 1 - assert len(alternate_upstream.sent_text) == 1 - replay_payload = json.loads(alternate_upstream.sent_text[0]) - assert "previous_response_id" not in replay_payload - assert all(item.get("type") not in {"reasoning", "web_search_call"} for item in replay_payload["input"]) - assert all("id" not in item for item in replay_payload["input"]) - assert "encrypted_content" not in alternate_upstream.sent_text[0] - assert degraded_reasons == [] + assert stale_session.closed is True +@pytest.mark.parametrize( + "fresh_developer_followup", + [ + pytest.param(False, id="ordinary-user-followup"), + pytest.param(True, id="fresh-developer-followup"), + ], +) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_reuses_derived_prompt_cache_key_when_client_omits_it(async_client, monkeypatch): +async def test_v1_responses_http_bridge_replays_full_resend_once_then_stays_on_new_owner( + async_client, app_instance, monkeypatch, fresh_developer_followup +): _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account(async_client, "acc_http_bridge_derived", "http-bridge-derived@example.com") - account = await _get_account(account_id) - fake_upstream = _FakeBridgeUpstreamWebSocket() - connect_count = 0 + owner_account_id = await _import_account( + async_client, + "acc_http_bridge_replay_owner", + "http-bridge-replay-owner@example.com", + ) + alternate_account_id = await _import_account( + async_client, + "acc_http_bridge_replay_alternate", + "http-bridge-replay-alternate@example.com", + ) + owner_account = await _get_account(owner_account_id) + alternate_account = await _get_account(alternate_account_id) + owner_chatgpt_account_id = cast(str, owner_account.chatgpt_account_id) + alternate_chatgpt_account_id = cast(str, alternate_account.chatgpt_account_id) + owner_upstream = _ClosingBridgeUpstreamWebSocket("resp_owner") + alternate_upstream = _FakeBridgeUpstreamWebSocket("resp_alternate") + selection_calls: list[dict[str, object]] = [] + connected_account_ids: list[str] = [] + connect_headers_by_account: dict[str, dict[str, str]] = {} - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - ) - return AccountSelection(account=account, error_message=None, error_code=None) + async def fake_select_account_with_budget(self, deadline, **kwargs): + del self, deadline + selection_calls.append(dict(kwargs)) + preferred_account_id = cast(str | None, kwargs.get("preferred_account_id")) + excluded_account_ids = cast(set[str], kwargs.get("exclude_account_ids") or set()) + fallback_enabled = bool(kwargs.get("fallback_on_preferred_account_unavailable", True)) + if preferred_account_id == owner_account.id and not fallback_enabled: + assert kwargs.get("preferred_account_is_continuity_owner") is True + return AccountSelection( + account=None, + error_message="Required continuity owner account no longer exists", + error_code=CONTINUITY_OWNER_UNAVAILABLE, + ) + if owner_account.id in excluded_account_ids or preferred_account_id == alternate_account.id: + return AccountSelection(account=alternate_account, error_message=None, error_code=None) + return AccountSelection(account=owner_account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds @@ -11745,233 +8169,250 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, account_id_header, base_url, session - nonlocal connect_count - connect_count += 1 - return fake_upstream - - async def fail_legacy_stream(*args, **kwargs): - raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + del access_token, base_url, session + connected_account_ids.append(account_id_header) + connect_headers_by_account[account_id_header] = dict(headers) + if account_id_header == owner_chatgpt_account_id: + return owner_upstream + assert account_id_header == alternate_chatgpt_account_id + return alternate_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - payload = { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "same-first-user-input", - } - first = await async_client.post("/v1/responses", json=payload) - second = await async_client.post("/v1/responses", json=payload) + historical_input = [ + *( + [ + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "custom", "name": "shell"}], + } + ] + if fresh_developer_followup + else [] + ), + { + "role": "user", + "content": [{"type": "input_text", "text": "first question"}], + }, + ] + first = await asyncio.wait_for( + async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": historical_input, + "prompt_cache_key": "http-bridge-full-replay", + }, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, + ) + assert first.status_code == 200, first.text + service = get_proxy_service_for_app(app_instance) + durable_lookup = await service._durable_bridge.lookup_request_targets( + session_key_kind="prompt_cache", + session_key_value="http-bridge-full-replay", + api_key_id=None, + turn_state=None, + session_header=None, + previous_response_id=first.json()["id"], + ) + assert durable_lookup is not None + assert durable_lookup.latest_input_item_count == len(historical_input) + assert durable_lookup.latest_input_full_fingerprint is not None - assert first.status_code == 200 - assert second.status_code == 200 - assert connect_count == 1 + if fresh_developer_followup: + retained_prior_output = { + "type": "message", + "role": "assistant", + "phase": "final_answer", + "status": "completed", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_previous"}, + "content": [{"type": "output_text", "text": "first answer"}], + } + fresh_followup_items = [ + { + "type": "message", + "role": "user", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_current"}, + "content": [{"type": "input_text", "text": "second question"}], + }, + { + "type": "message", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_current"}, + "content": [{"type": "input_text", "text": "fresh control"}], + }, + ] + else: + retained_prior_output = { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "first answer"}], + } + fresh_followup_items = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "second question"}], + } + ] + full_resend = [ + *historical_input, + retained_prior_output, + *fresh_followup_items, + ] + second = await asyncio.wait_for( + async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": full_resend, + "prompt_cache_key": "http-bridge-full-replay", + "previous_response_id": first.json()["id"], + }, + headers={ + "session_id": "stale-session", + "session-id": "stale-session-dash", + "thread-id": "stale-thread", + "x-codex-conversation-id": "stale-conversation", + "x-codex-session-id": "stale-codex-session", + "x-codex-turn-state": "http_turn_stale", + "x-request-trace": "keep-me", + }, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, + ) + assert second.status_code == 200, second.text + assert second.json()["id"] == "resp_alternate_1" + + third = await asyncio.wait_for( + async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "third question", + "prompt_cache_key": "http-bridge-full-replay", + "previous_response_id": second.json()["id"], + }, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, + ) + assert third.status_code == 200, third.text + assert third.json()["id"] == "resp_alternate_2" + + assert connected_account_ids == [ + owner_chatgpt_account_id, + alternate_chatgpt_account_id, + ] + alternate_connect_headers = { + key.lower(): value for key, value in connect_headers_by_account[alternate_chatgpt_account_id].items() + } + assert alternate_connect_headers["x-request-trace"] == "keep-me" + assert ( + not { + "session_id", + "session-id", + "thread-id", + "x-codex-conversation-id", + "x-codex-session-id", + "x-codex-turn-state", + } + & alternate_connect_headers.keys() + ) + assert len(owner_upstream.sent_text) == 1 + assert len(alternate_upstream.sent_text) == 2 + replay_payload = json.loads(alternate_upstream.sent_text[0]) + assert "previous_response_id" not in replay_payload + assert replay_payload["input"] == full_resend + follow_up_payload = json.loads(alternate_upstream.sent_text[1]) + assert follow_up_payload["previous_response_id"] == second.json()["id"] + owner_miss = next( + call + for call in selection_calls + if call.get("preferred_account_id") == owner_account.id + and call.get("fallback_on_preferred_account_unavailable") is False + ) + assert owner_miss["preferred_account_is_continuity_owner"] is True @pytest.mark.asyncio -async def test_v1_responses_http_bridge_terminal_release_admits_second_session_before_idle_ttl( +async def test_backend_responses_verified_full_resend_ignores_stale_broad_owner_on_durable_account( async_client, app_instance, monkeypatch, ): - app_settings = _make_app_settings(enabled=True, codex_idle_ttl_seconds=900.0).model_copy( - update={ - "proxy_account_stream_limit": 1, - "proxy_account_stream_recovery_reserve": 0, - } - ) - _install_proxy_settings( - monkeypatch, - app_settings=app_settings, - dashboard_settings=_make_dashboard_settings(), + _install_bridge_settings(monkeypatch, enabled=True) + owner_account_id = await _import_account( + async_client, + "acc_backend_durable_full_resend_owner", + "backend-durable-full-resend-owner@example.com", ) - account_id = await _import_account( + stale_account_id = await _import_account( async_client, - "acc_http_bridge_idle_release", - "http-bridge-idle-release@example.com", + "acc_backend_durable_full_resend_stale", + "backend-durable-full-resend-stale@example.com", ) - account = await _get_account(account_id) + owner_account = await _get_account(owner_account_id) + stale_account = await _get_account(stale_account_id) + owner_chatgpt_account_id = cast(str, owner_account.chatgpt_account_id) service = get_proxy_service_for_app(app_instance) - upstreams = deque([_FakeBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()]) - - async def fake_select_account_with_budget(*_args: object, **_kwargs: object) -> AccountSelection: - lease = await service._load_balancer.acquire_account_lease(account.id, kind="stream") - if lease is None: - return AccountSelection( - account=None, - error_message="Account stream capacity is exhausted; wait for active streams to finish.", - error_code="account_stream_cap", - ) - return AccountSelection(account=account, error_message=None, lease=lease) - - async def fake_ensure_fresh_with_budget( - _self: object, - target: Account, - **_kwargs: object, - ) -> Account: - return target - - async def fake_connect_responses_websocket(*_args: object, **_kwargs: object) -> _FakeBridgeUpstreamWebSocket: - return upstreams.popleft() - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - payload = { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "release the idle stream lease", - } - first = await async_client.post("/v1/responses", json=payload, headers={"session_id": "idle-release-a"}) - assert first.status_code == 200 - assert await service._load_balancer.account_pressure_snapshot(account.id) == (0, 0, 0.0) - - second = await async_client.post("/v1/responses", json=payload, headers={"session_id": "idle-release-b"}) - assert second.status_code == 200 - assert await service._load_balancer.account_pressure_snapshot(account.id) == (0, 0, 0.0) - assert not upstreams - assert len(service._http_bridge_sessions) == 2 - assert all(not session.closed for session in service._http_bridge_sessions.values()) - assert all(session.idle_ttl_seconds >= 900.0 for session in service._http_bridge_sessions.values()) - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_prefers_session_header_for_isolation(async_client, monkeypatch): - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_session_key", - "http-bridge-session-key@example.com", + session_id = "backend-durable-full-resend-session" + historical_input: list[proxy_module.JsonValue] = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "first question"}], + } + ] + claimed = await service._durable_bridge.claim_live_session( + session_key_kind="session_header", + session_key_value=session_id, + api_key_id=None, + instance_id="instance-a", + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id=owner_account.id, + model="gpt-5.1", + service_tier=None, + latest_turn_state="http_turn_durable_full_resend", + latest_response_id="resp_durable_full_resend_previous", + allow_takeover=True, ) - account = await _get_account(account_id) - upstreams = [_FakeBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] - connect_count = 0 - - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - ) - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - nonlocal connect_count - upstream = upstreams[connect_count] - connect_count += 1 - return upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - payload = { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "same-first-user-input", - } - first = await async_client.post("/v1/responses", json=payload, headers={"session_id": "session-a"}) - second = await async_client.post("/v1/responses", json=payload, headers={"session_id": "session-b"}) - - assert first.status_code == 200 - assert second.status_code == 200 - assert connect_count == 2 - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_retries_once_when_upstream_closes_before_response_created( - async_client, - monkeypatch, -): - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account(async_client, "acc_http_bridge_retry", "http-bridge-retry@example.com") - account = await _get_account(account_id) - upstreams = [_PrecreatedCloseUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] - connect_count = 0 - - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, + renewed = await service._durable_bridge.renew_live_session( + session_id=claimed.session_id, + api_key_id=None, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + lease_ttl_seconds=60.0, + latest_turn_state="http_turn_durable_full_resend", + latest_response_id="resp_durable_full_resend_previous", + latest_input_item_count=len(historical_input), + latest_input_full_fingerprint=proxy_module._fingerprint_input_items(historical_input), + ) + assert renewed is not None + released = await service._durable_bridge.release_live_session( + session_id=claimed.session_id, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + draining=False, + ) + assert released is not None + assert released.account_id == owner_account.id + + async with SessionLocal() as session: + await StickySessionsRepository(session).upsert( + session_id, + stale_account.id, + kind=proxy_module.StickySessionKind.CODEX_SESSION, ) - return AccountSelection(account=account, error_message=None, error_code=None) + + upstream = _FakeBridgeUpstreamWebSocket("resp_durable_full_resend") + connect_calls: list[tuple[dict[str, str], str]] = [] async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds @@ -11985,84 +8426,241 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, account_id_header, base_url, session - nonlocal connect_count - upstream = upstreams[connect_count] - connect_count += 1 + del access_token, base_url, session + connect_calls.append((dict(headers), account_id_header)) return upstream - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - response = await async_client.post( - "/v1/responses", - json={ + full_resend = [ + *historical_input, + { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "first answer"}], + }, + { + "role": "user", + "content": [{"type": "input_text", "text": "second question"}], + }, + ] + first_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": "retry-me", - "prompt_cache_key": "retry-key", + "input": full_resend, + "stream": True, + }, + headers={"session_id": session_id, "x-request-trace": "keep-me"}, + ) + second_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "third question", + "stream": True, }, + headers={"session_id": session_id}, ) - assert response.status_code == 200 - assert connect_count == 2 + _assert_created_text_delta_completed(first_events) + _assert_created_text_delta_completed(second_events) + assert connect_calls[0][1] == owner_chatgpt_account_id + assert len(connect_calls) == 1 + connect_headers = {key.lower(): value for key, value in connect_calls[0][0].items()} + assert connect_headers["x-request-trace"] == "keep-me" + assert ( + not { + "session_id", + "session-id", + "thread-id", + "x-codex-conversation-id", + "x-codex-session-id", + "x-codex-turn-state", + } + & connect_headers.keys() + ) + assert len(upstream.sent_text) == 2 + replay_payload = json.loads(upstream.sent_text[0]) + assert "previous_response_id" not in replay_payload + assert replay_payload["input"] == full_resend + bridge_key = proxy_module._HTTPBridgeSessionKey("session_header", session_id, None) + bridge_session = service._http_bridge_sessions[bridge_key] + assert bridge_session.account.id == owner_account.id + assert bridge_session.codex_session is True + assert bridge_session.affinity.kind == proxy_module.StickySessionKind.CODEX_SESSION + assert bridge_session.affinity.key is None + async with SessionLocal() as session: + assert ( + await StickySessionsRepository(session).get_account_id( + session_id, + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ) + == stale_account.id + ) @pytest.mark.asyncio -async def test_backend_responses_http_bridge_retries_precreated_server_overload(async_client, monkeypatch): +async def test_backend_responses_verified_full_resend_fails_over_to_new_account_after_owner_loss( + async_client, + app_instance, + monkeypatch, +): _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( + owner_account_id = await _import_account( async_client, - "acc_http_bridge_server_overload", - "http-bridge-server-overload@example.com", + "acc_backend_full_resend_failover_owner", + "backend-full-resend-failover-owner@example.com", ) - account = await _get_account(account_id) - upstreams = [_PrecreatedOverloadUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] - connect_count = 0 + owner_account = await _get_account(owner_account_id) + owner_chatgpt_account_id = cast(str, owner_account.chatgpt_account_id) + owner_upstream = _ClosingBridgeUpstreamWebSocket("resp_failover_owner") + alternate_upstream = _FakeBridgeUpstreamWebSocket("resp_failover_alternate") + connected_account_ids: list[str] = [] + connect_headers_by_account: dict[str, dict[str, str]] = {} + degraded_reasons: list[str] = [] - async def fake_select_account_with_budget( - self, - deadline, + async def fake_ensure_fresh_with_budget(self, account, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return account + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, + base_url=None, + session=None, ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - api_key, - ) - return AccountSelection(account=account, error_message=None, error_code=None) + del access_token, base_url, session + connected_account_ids.append(account_id_header) + connect_headers_by_account[account_id_header] = dict(headers) + if account_id_header == owner_chatgpt_account_id: + return owner_upstream + return alternate_upstream + + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(load_balancer_module, "set_degraded", degraded_reasons.append) + + session_id = "backend-full-resend-failover-session" + historical_input: list[proxy_module.JsonValue] = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "first question"}], + } + ] + first_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": historical_input, + "stream": True, + }, + headers={"session_id": session_id}, + ) + first_response = first_events[-1]["response"] + assert first_response["id"] == "resp_failover_owner_1" + + service = get_proxy_service_for_app(app_instance) + durable_lookup = await service._durable_bridge.lookup_request_targets( + session_key_kind="session_header", + session_key_value=session_id, + api_key_id=None, + turn_state=None, + session_header=session_id, + previous_response_id=None, + ) + assert durable_lookup is not None + assert durable_lookup.account_id == owner_account.id + assert durable_lookup.latest_input_item_count == len(historical_input) + assert durable_lookup.latest_input_full_fingerprint is not None + + alternate_account_id = await _import_account( + async_client, + "acc_backend_full_resend_failover_alternate", + "backend-full-resend-failover-alternate@example.com", + ) + alternate_account = await _get_account(alternate_account_id) + alternate_chatgpt_account_id = cast(str, alternate_account.chatgpt_account_id) + pause = await async_client.post(f"/api/accounts/{owner_account_id}/pause") + assert pause.status_code == 200, pause.text + + full_resend = [ + *historical_input, + first_response["output"][0], + { + "role": "user", + "content": [{"type": "input_text", "text": "second question"}], + }, + ] + second_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": full_resend, + "stream": True, + }, + headers={"session_id": session_id, "x-request-trace": "keep-me"}, + ) + second_response = second_events[-1]["response"] + assert second_response["id"] == "resp_failover_alternate_1" + + assert connected_account_ids == [owner_chatgpt_account_id, alternate_chatgpt_account_id] + alternate_connect_headers = { + key.lower(): value for key, value in connect_headers_by_account[alternate_chatgpt_account_id].items() + } + assert alternate_connect_headers["x-request-trace"] == "keep-me" + assert ( + not { + "session_id", + "session-id", + "thread-id", + "x-codex-conversation-id", + "x-codex-session-id", + "x-codex-turn-state", + } + & alternate_connect_headers.keys() + ) + assert len(owner_upstream.sent_text) == 1 + assert len(alternate_upstream.sent_text) == 1 + replay_payload = json.loads(alternate_upstream.sent_text[0]) + assert "previous_response_id" not in replay_payload + assert replay_payload["input"] == full_resend + assert degraded_reasons == [] - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + +@pytest.mark.asyncio +async def test_backend_responses_http_bridge_real_selector_recovers_full_resend_without_degrading_pool( + async_client, monkeypatch +): + _install_bridge_settings(monkeypatch, enabled=True) + owner_account_id = await _import_account( + async_client, + "acc_backend_replay_owner", + "backend-replay-owner@example.com", + ) + owner_account = await _get_account(owner_account_id) + owner_chatgpt_account_id = cast(str, owner_account.chatgpt_account_id) + owner_upstream = _ClosingBridgeUpstreamWebSocket("resp_backend_owner") + alternate_upstream = _FakeBridgeUpstreamWebSocket("resp_backend_alternate") + connected_account_ids: list[str] = [] + connect_headers_by_account: dict[str, dict[str, str]] = {} + degraded_reasons: list[str] = [] + + async def fake_ensure_fresh_with_budget(self, account, *, force=False, timeout_seconds): del self, force, timeout_seconds - return target + return account async def fake_connect_responses_websocket( headers, @@ -12072,147 +8670,151 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, account_id_header, base_url, session - nonlocal connect_count - upstream = upstreams[connect_count] - connect_count += 1 - return upstream - - async def fail_legacy_stream(*args, **kwargs): - raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + del access_token, base_url, session + connected_account_ids.append(account_id_header) + connect_headers_by_account[account_id_header] = dict(headers) + if account_id_header == owner_chatgpt_account_id: + return owner_upstream + return alternate_upstream - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) + monkeypatch.setattr(load_balancer_module, "set_degraded", degraded_reasons.append) - events = await _collect_sse_events( + historical_input = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "first question"}], + } + ] + first_events = await _collect_sse_events( async_client, "/backend-api/codex/responses", json_body={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": "retry-overload", - "prompt_cache_key": "server-overload-retry-key", + "input": historical_input, + "prompt_cache_key": "backend-http-bridge-full-replay", "stream": True, }, ) + first_response = first_events[-1]["response"] + assert first_response["id"] == "resp_backend_owner_1" - _assert_created_text_delta_completed(events) - assert events[-1]["response"]["id"] == "resp_bridge_1" - assert connect_count == 2 - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_rejects_oversized_response_create_before_upstream( - async_client, - monkeypatch, - tmp_path, -): - _install_bridge_settings(monkeypatch, enabled=True) - monkeypatch.setattr(proxy_module, "_UPSTREAM_RESPONSE_CREATE_WARN_BYTES", 64) - monkeypatch.setattr(proxy_module, "_UPSTREAM_RESPONSE_CREATE_MAX_BYTES", 128) - monkeypatch.setattr(proxy_module, "_OVERSIZED_RESPONSE_CREATE_DUMP_DIR", tmp_path) - - async def fail_get_or_create_http_bridge_session(self, *args, **kwargs): - del self, args, kwargs - raise AssertionError("oversized response.create must fail before upstream bridge session allocation") - - monkeypatch.setattr( - proxy_module.ProxyService, - "_get_or_create_http_bridge_session", - fail_get_or_create_http_bridge_session, + alternate_account_id = await _import_account( + async_client, + "acc_backend_replay_alternate", + "backend-replay-alternate@example.com", ) + alternate_account = await _get_account(alternate_account_id) + alternate_chatgpt_account_id = cast(str, alternate_account.chatgpt_account_id) + pause = await async_client.post(f"/api/accounts/{owner_account_id}/pause") + assert pause.status_code == 200, pause.text - request_json = { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": [{"role": "user", "content": [{"type": "input_text", "text": "x" * 256}]}], - "prompt_cache_key": "oversized-http-bridge", + retained_prior_output = first_response["output"][0] + full_resend = [ + *historical_input, + retained_prior_output, + { + "role": "user", + "content": [{"type": "input_text", "text": "second question"}], + }, + ] + stale_headers = { + "session_id": "stale-session", + "session-id": "stale-session-dash", + "thread-id": "stale-thread", + "x-codex-conversation-id": "stale-conversation", + "x-codex-session-id": "stale-codex-session", + "x-codex-turn-state": "http_turn_stale", + "x-request-trace": "keep-me", } + second_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": full_resend, + "prompt_cache_key": "backend-http-bridge-full-replay", + "previous_response_id": first_response["id"], + "stream": True, + }, + headers=stale_headers, + ) + second_response = second_events[-1]["response"] + assert second_response["id"] == "resp_backend_alternate_1" - response = await async_client.post("/v1/responses", json=request_json) - - assert response.status_code == 400 - payload = response.json() - assert payload["error"]["code"] == "payload_too_large" - assert payload["error"]["type"] == "invalid_request_error" - assert payload["error"]["param"] == "input" - assert "response.create is too large for upstream websocket" in payload["error"]["message"] - - meta_files = list(tmp_path.glob("*.meta.json")) - assert len(meta_files) == 1 - meta = json.loads(meta_files[0].read_text(encoding="utf-8")) - assert meta["reason"]["error_code"] == "payload_too_large" - assert meta["request"]["transport"] == "http" - assert meta["request"]["request_text_bytes"] > 128 - - duplicate_response = await async_client.post("/v1/responses", json=request_json) - assert duplicate_response.status_code == 400 - assert len(list(tmp_path.glob("*.response-create.json.gz"))) == 1 - assert len(list(tmp_path.glob("*.meta.json"))) == 1 + third_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "third question", + "prompt_cache_key": "backend-http-bridge-full-replay", + "previous_response_id": second_response["id"], + "stream": True, + }, + ) + assert third_events[-1]["response"]["id"] == "resp_backend_alternate_2" - meta_files[0].unlink() - orphan_retry_response = await async_client.post("/v1/responses", json=request_json) - assert orphan_retry_response.status_code == 400 - complete_pairs = [ - dump_path - for dump_path in tmp_path.glob("*.response-create.json.gz") - if (tmp_path / f"{dump_path.name[: -len('.response-create.json.gz')]}.meta.json").exists() - ] - assert complete_pairs + assert connected_account_ids == [owner_chatgpt_account_id, alternate_chatgpt_account_id] + alternate_connect_headers = { + key.lower(): value for key, value in connect_headers_by_account[alternate_chatgpt_account_id].items() + } + assert alternate_connect_headers["x-request-trace"] == "keep-me" + assert ( + not { + "session_id", + "session-id", + "thread-id", + "x-codex-conversation-id", + "x-codex-session-id", + "x-codex-turn-state", + } + & alternate_connect_headers.keys() + ) + assert len(owner_upstream.sent_text) == 1 + assert len(alternate_upstream.sent_text) == 2 + replay_payload = json.loads(alternate_upstream.sent_text[0]) + assert "previous_response_id" not in replay_payload + assert replay_payload["input"] == full_resend + follow_up_payload = json.loads(alternate_upstream.sent_text[1]) + assert follow_up_payload["previous_response_id"] == second_response["id"] + assert degraded_reasons == [] @pytest.mark.asyncio -async def test_v1_responses_http_bridge_slims_historical_inline_artifacts_and_succeeds( - async_client, - monkeypatch, +async def test_backend_responses_http_bridge_declines_cross_account_anchor_and_settles( + async_client, app_instance, monkeypatch ): - _install_bridge_settings(monkeypatch, enabled=True) - monkeypatch.setattr(proxy_module, "_UPSTREAM_RESPONSE_CREATE_WARN_BYTES", 64) - monkeypatch.setattr(proxy_module, "_UPSTREAM_RESPONSE_CREATE_MAX_BYTES", 640) - account_id = await _import_account(async_client, "acc_http_bridge_slim", "http-bridge-slim@example.com") - account = await _get_account(account_id) - fake_upstream = _FakeBridgeUpstreamWebSocket() + """A restored durable anchor owned by another account must not be injected. - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - api_key, - ) - return AccountSelection(account=account, error_message=None, error_code=None) + The durable record still names the owner account, but the owner is out of the + rotation so the bridge session is created on another account. Replaying the + owner's ``previous_response_id`` there would send an anchor upstream cannot + resolve with the history trimmed away: upstream never emits + ``response.created`` and the per-bridge response-create gate wedges. The turn + must go upstream as a full-history resend instead, and it must settle. + """ + + _install_bridge_settings(monkeypatch, enabled=True) + owner_account_id = await _import_account( + async_client, + "acc_cross_account_anchor_owner", + "cross-account-anchor-owner@example.com", + ) + serving_account_id = await _import_account( + async_client, + "acc_cross_account_anchor_serving", + "cross-account-anchor-serving@example.com", + ) + owner_account = await _get_account(owner_account_id) + serving_account = await _get_account(serving_account_id) + serving_chatgpt_account_id = cast(str, serving_account.chatgpt_account_id) + serving_upstream = _AccountScopedAnchorUpstreamWebSocket("resp_cross_account_serving") + service = get_proxy_service_for_app(app_instance) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): del self, force, timeout_seconds @@ -12226,256 +8828,156 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, account_id_header, base_url, session - return fake_upstream + del headers, access_token, base_url, session + assert account_id_header == serving_chatgpt_account_id + return serving_upstream - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - response = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": [ - {"role": "user", "content": [{"type": "input_text", "text": "old turn"}]}, - { - "type": "function_call_output", - "call_id": "call_1", - "output": "data:image/png;base64," + ("A" * 1500), - }, - {"role": "assistant", "content": [{"type": "output_text", "text": "done"}]}, - {"role": "user", "content": [{"type": "input_text", "text": "ping"}]}, - ], - "prompt_cache_key": "slim-http-bridge", - }, - ) - - assert response.status_code == 200 - sent_payload = json.loads(fake_upstream.sent_text[0]) - assert sent_payload["input"][-1]["content"][0]["text"] == "ping" - assert "data:image/" not in json.dumps(sent_payload["input"], ensure_ascii=True) - assert "historical tool output" in json.dumps(sent_payload["input"], ensure_ascii=True) - + # The durable owner account left the rotation, so selection lands on the + # other account while the restored durable record still names the owner. + pause = await async_client.post(f"/api/accounts/{owner_account_id}/pause") + assert pause.status_code == 200, pause.text -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_does_not_evict_active_session_when_pool_is_full( - async_client, - app_instance, - monkeypatch, -): - _install_bridge_settings_with_limits(monkeypatch, enabled=True, max_sessions=1) - account_id = await _import_account(async_client, "acc_http_bridge_capacity", "http-bridge-capacity@example.com") - service = get_proxy_service_for_app(app_instance) - account = await _get_account(account_id) - hanging_upstream = _CreatedOnlyUpstreamWebSocket() + stored_input: list[proxy_module.JsonValue] = [ + {"role": "user", "content": [{"type": "input_text", "text": "first question"}]}, + ] - async def fake_select_account_with_budget( - self, - deadline, + def _durable_record( *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, + account_id: str, + latest_response_id: str, + stored_items: list[proxy_module.JsonValue], + ) -> proxy_module.DurableBridgeLookup: + return proxy_module.DurableBridgeLookup( + session_id="durable-cross-account-anchor", + canonical_kind="prompt_cache", + canonical_key="cross-account-anchor-cache-key", + api_key_scope="__anonymous__", + account_id=account_id, + owner_instance_id=None, + owner_epoch=1, + lease_expires_at=None, + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state=None, + latest_response_id=latest_response_id, + latest_input_item_count=len(stored_items), + latest_input_full_fingerprint=proxy_module._fingerprint_input_items(stored_items), ) - return AccountSelection(account=account, error_message=None, error_code=None) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target + durable_record = _durable_record( + account_id=owner_account.id, + latest_response_id="resp_cross_account_owner_1", + stored_items=stored_input, + ) - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - return hanging_upstream + async def fake_lookup_request_targets(**kwargs): + del kwargs + return durable_record - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - first_payload = proxy_module.ResponsesRequest( - model="gpt-5.1", - instructions="Return exactly OK.", - input="hold-open", - prompt_cache_key="active-session-a", - ) - first_affinity = proxy_module._sticky_key_for_responses_request( - first_payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - first_key = proxy_module._make_http_bridge_session_key( - first_payload, - headers={}, - affinity=first_affinity, - api_key=None, - request_id="req_a", - ) - first_session = await service._get_or_create_http_bridge_session( - first_key, - headers={}, - affinity=first_affinity, - api_key=None, - request_model="gpt-5.1", - idle_ttl_seconds=120.0, - max_sessions=1, + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", fake_lookup_request_targets) + + # Compaction-shaped follow-up: the stored prefix still matches, but the + # suffix carries no prior assistant output, so the account-neutral fresh + # resend projection is unavailable and the restored durable anchor is the + # only continuity candidate the session-level injection can reach for. + compacted_resend: list[proxy_module.JsonValue] = [ + *stored_input, + {"role": "user", "content": [{"type": "input_text", "text": "second question"}]}, + ] + session_id = "cross-account-anchor-session" + first_events, first_headers = await asyncio.wait_for( + _collect_sse_events_with_headers( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": compacted_resend, + "stream": True, + }, + headers={"session_id": session_id}, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - async with first_session.pending_lock: - first_session.pending_requests.append( - proxy_module._WebSocketRequestState( - request_id="req-active", - model="gpt-5.1", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, - response_create_gate_acquired=True, - event_queue=asyncio.Queue(), - transport="http", - ) - ) - second_payload = proxy_module.ResponsesRequest( - model="gpt-5.1", - instructions="Return exactly OK.", - input="new-session", - prompt_cache_key="active-session-b", + _assert_created_text_delta_completed(first_events) + turn_state = first_headers["x-codex-turn-state"] + + assert len(serving_upstream.sent_text) == 1 + resend_payload = json.loads(serving_upstream.sent_text[0]) + assert "previous_response_id" not in resend_payload + assert resend_payload["input"] == compacted_resend + + bridge_session = next( + candidate for candidate in service._http_bridge_sessions.values() if candidate.account.id == serving_account.id ) - second_affinity = proxy_module._sticky_key_for_responses_request( - second_payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, + assert bridge_session.codex_session is True + # The turn settled on the serving account, so the gate is free and the + # session anchor is now owned by the account that actually created it. + assert bridge_session.response_create_gate.locked() is False + assert bridge_session.last_completed_response_id == "resp_cross_account_serving_1" + assert bridge_session.last_completed_response_account_id == serving_account.id + + # Same-account continuity is untouched: once the durable record names the + # account that actually created the response, the very next turn anchors on + # it instead of resending the whole history. + durable_record = _durable_record( + account_id=serving_account.id, + latest_response_id="resp_cross_account_serving_1", + stored_items=compacted_resend, ) - second_key = proxy_module._make_http_bridge_session_key( - second_payload, - headers={}, - affinity=second_affinity, - api_key=None, - request_id="req_b", + second_events = await asyncio.wait_for( + _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": [ + *compacted_resend, + { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "OK"}], + }, + {"role": "user", "content": [{"type": "input_text", "text": "third question"}]}, + ], + "stream": True, + }, + headers={"session_id": session_id, "x-codex-turn-state": turn_state}, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._get_or_create_http_bridge_session( - second_key, - headers={}, - affinity=second_affinity, - api_key=None, - request_model="gpt-5.1", - idle_ttl_seconds=120.0, - max_sessions=1, - ) - exc = exc_info.value - assert exc.status_code == 429 - assert hanging_upstream.closed is False - await service._close_http_bridge_session(first_session) + _assert_created_text_delta_completed(second_events) + assert len(serving_upstream.sent_text) == 2 + follow_up_payload = json.loads(serving_upstream.sent_text[1]) + assert follow_up_payload["previous_response_id"] == "resp_cross_account_serving_1" + assert bridge_session.response_create_gate.locked() is False @pytest.mark.asyncio -async def test_v1_responses_http_bridge_times_out_queued_request_on_bounded_startup_gate_wait( +async def test_backend_responses_projects_retained_encrypted_reasoning_before_replaying_to_available_account( async_client, - app_instance, monkeypatch, ): - _install_bridge_settings_with_limits( - monkeypatch, - enabled=True, - max_sessions=1, - # Queued HTTP bridge requests have already claimed the bridge queue slot, - # so we still wait for the per-session response-create gate, but only - # until the bounded startup timeout. - admission_wait_timeout_seconds=0.01, - ) - account_id = await _import_account( + _install_bridge_settings(monkeypatch, enabled=True) + owner_account_id = await _import_account( async_client, - "acc_http_bridge_queued_capacity", - "http-bridge-queued@example.com", + "acc_backend_encrypted_owner", + "backend-encrypted-owner@example.com", ) - service = get_proxy_service_for_app(app_instance) - account = await _get_account(account_id) - hanging_upstream = _SilentUpstreamWebSocket() - - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - ) - return AccountSelection(account=account, error_message=None, error_code=None) + owner_account = await _get_account(owner_account_id) + owner_chatgpt_account_id = cast(str, owner_account.chatgpt_account_id) + owner_upstream = _ClosingBridgeUpstreamWebSocket("resp_backend_encrypted_owner") + alternate_upstream = _FakeBridgeUpstreamWebSocket("resp_backend_encrypted_alternate") + connected_account_ids: list[str] = [] + degraded_reasons: list[str] = [] - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + async def fake_ensure_fresh_with_budget(self, account, *, force=False, timeout_seconds): del self, force, timeout_seconds - return target + return account async def fake_connect_responses_websocket( headers, @@ -12485,99 +8987,99 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - del headers, access_token, account_id_header, base_url, session - return hanging_upstream + del headers, access_token, base_url, session + connected_account_ids.append(account_id_header) + if account_id_header == owner_chatgpt_account_id: + return owner_upstream + return alternate_upstream - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(load_balancer_module, "set_degraded", degraded_reasons.append) - first_payload = proxy_module.ResponsesRequest( - model="gpt-5.1", - instructions="Return exactly OK.", - input="queued-session", - prompt_cache_key="queued-session-a", - ) - first_affinity = proxy_module._sticky_key_for_responses_request( - first_payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - first_key = proxy_module._make_http_bridge_session_key( - first_payload, - headers={}, - affinity=first_affinity, - api_key=None, - request_id="req_queue_a", - ) - first_session = await service._get_or_create_http_bridge_session( - first_key, - headers={}, - affinity=first_affinity, - api_key=None, - request_model="gpt-5.1", - idle_ttl_seconds=120.0, - max_sessions=1, + historical_input = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "first question"}], + } + ] + first_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": historical_input, + "prompt_cache_key": "backend-http-bridge-encrypted-replay", + "stream": True, + }, ) + first_response = first_events[-1]["response"] - await first_session.response_create_gate.acquire() - request_state, text_data = service._prepare_http_bridge_request( - first_payload, - {}, - api_key=None, - api_key_reservation=None, - ) - request_state.transport = "http" - submit_task = asyncio.create_task( - service._submit_http_bridge_request( - first_session, - request_state=request_state, - text_data=text_data, - queue_limit=8, - ) + alternate_account_id = await _import_account( + async_client, + "acc_backend_encrypted_alternate", + "backend-encrypted-alternate@example.com", ) - await asyncio.sleep(0) - await asyncio.sleep(0.05) - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await asyncio.wait_for(submit_task, timeout=0.2) - exc = exc_info.value - assert exc.status_code == 429 - assert exc.payload["error"]["code"] == "response_create_gate_timeout" - assert exc.payload["error"]["type"] == "rate_limit_error" - - assert await service._http_bridge_pending_count(first_session) == 0 - async with first_session.pending_lock: - assert list(first_session.pending_requests) == [] - assert first_session.queued_request_count == 0 - - first_session.response_create_gate.release() - await service._close_http_bridge_session(first_session) - + alternate_account = await _get_account(alternate_account_id) + alternate_chatgpt_account_id = cast(str, alternate_account.chatgpt_account_id) + pause = await async_client.post(f"/api/accounts/{owner_account_id}/pause") + assert pause.status_code == 200, pause.text -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_gate_wait_is_clamped_to_remaining_budget( - async_client, - app_instance, - monkeypatch, -): - _install_bridge_settings_with_limits( - monkeypatch, - enabled=True, - max_sessions=1, - admission_wait_timeout_seconds=5.0, - ) - account_id = await _import_account( + full_resend = [ + *historical_input, + { + "type": "reasoning", + "id": "rs_owner_scoped", + "encrypted_content": "owner-scoped-ciphertext", + "summary": [], + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-owner"}, + }, + { + "type": "web_search_call", + "id": "ws_owner_scoped", + "action": {"type": "search", "query": "portable result"}, + "status": "completed", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-owner"}, + }, + first_response["output"][0], + { + "role": "user", + "content": [{"type": "input_text", "text": "second question"}], + }, + ] + second_events = await _collect_sse_events( async_client, - "acc_http_bridge_budget_clamp", - "http-bridge-budget-clamp@example.com", + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": full_resend, + "prompt_cache_key": "backend-http-bridge-encrypted-replay", + "previous_response_id": first_response["id"], + "stream": True, + }, ) - service = get_proxy_service_for_app(app_instance) + + assert second_events[-1]["response"]["id"] == "resp_backend_encrypted_alternate_1" + assert connected_account_ids == [owner_chatgpt_account_id, alternate_chatgpt_account_id] + assert len(owner_upstream.sent_text) == 1 + assert len(alternate_upstream.sent_text) == 1 + replay_payload = json.loads(alternate_upstream.sent_text[0]) + assert "previous_response_id" not in replay_payload + assert all(item.get("type") not in {"reasoning", "web_search_call"} for item in replay_payload["input"]) + assert all("id" not in item for item in replay_payload["input"]) + assert "encrypted_content" not in alternate_upstream.sent_text[0] + assert degraded_reasons == [] + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_reuses_derived_prompt_cache_key_when_client_omits_it(async_client, monkeypatch): + _install_bridge_settings(monkeypatch, enabled=True) + account_id = await _import_account(async_client, "acc_http_bridge_derived", "http-bridge-derived@example.com") account = await _get_account(account_id) - hanging_upstream = _SilentUpstreamWebSocket() + fake_upstream = _FakeBridgeUpstreamWebSocket() + connect_count = 0 async def fake_select_account_with_budget( self, @@ -12630,98 +9132,110 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - return hanging_upstream + nonlocal connect_count + connect_count += 1 + return fake_upstream + + async def fail_legacy_stream(*args, **kwargs): + raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) - payload = proxy_module.ResponsesRequest( - model="gpt-5.1", - instructions="Return exactly OK.", - input="budget-clamp-session", - prompt_cache_key="budget-clamp-session-a", - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={}, - affinity=affinity, - api_key=None, - request_id="req_budget_clamp_a", - ) - session = await service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=affinity, - api_key=None, - request_model="gpt-5.1", - idle_ttl_seconds=120.0, - max_sessions=1, - ) - - await session.response_create_gate.acquire() - request_state, text_data = service._prepare_http_bridge_request( - payload, - {}, - api_key=None, - api_key_reservation=None, - ) - request_state.transport = "http" - # Age the request so only ~0.2s of the bridge request budget remains: - # the gate wait must be clamped to that tail, not run the full 5s - # admission timeout past the budget. - budget_seconds = proxy_module._http_bridge_request_budget_seconds(proxy_module.get_settings()) - request_state.started_at = time.monotonic() - (budget_seconds - 0.2) - - started = time.monotonic() - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._submit_http_bridge_request( - session, - request_state=request_state, - text_data=text_data, - queue_limit=8, - ) - elapsed = time.monotonic() - started - - assert exc_info.value.status_code == 429 - assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" - assert elapsed < 2.0, f"gate wait ran past the remaining budget: {elapsed:.2f}s" + payload = { + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "same-first-user-input", + } + first = await async_client.post("/v1/responses", json=payload) + second = await async_client.post("/v1/responses", json=payload) - session.response_create_gate.release() - await service._close_http_bridge_session(session) + assert first.status_code == 200 + assert second.status_code == 200 + assert connect_count == 1 @pytest.mark.asyncio -async def test_v1_responses_http_bridge_gate_contention_waits_and_completes_after_release( +async def test_v1_responses_http_bridge_terminal_release_admits_second_session_before_idle_ttl( async_client, app_instance, monkeypatch, ): - _install_bridge_settings_with_limits( + app_settings = _make_app_settings(enabled=True, codex_idle_ttl_seconds=900.0).model_copy( + update={ + "proxy_account_stream_limit": 1, + "proxy_account_stream_recovery_reserve": 0, + } + ) + _install_proxy_settings( monkeypatch, - enabled=True, - max_sessions=1, - admission_wait_timeout_seconds=0.05, + app_settings=app_settings, + dashboard_settings=_make_dashboard_settings(), ) - monkeypatch.setattr(http_bridge_streaming_module, "_RESPONSE_CREATE_GATE_RETRY_SLEEP_SECONDS", 0.02) - monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 0.02) account_id = await _import_account( async_client, - "acc_http_bridge_gate_wait", - "http-bridge-gate-wait@example.com", + "acc_http_bridge_idle_release", + "http-bridge-idle-release@example.com", ) + account = await _get_account(account_id) service = get_proxy_service_for_app(app_instance) + upstreams = deque([_FakeBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()]) + + async def fake_select_account_with_budget(*_args: object, **_kwargs: object) -> AccountSelection: + lease = await service._load_balancer.acquire_account_lease(account.id, kind="stream") + if lease is None: + return AccountSelection( + account=None, + error_message="Account stream capacity is exhausted; wait for active streams to finish.", + error_code="account_stream_cap", + ) + return AccountSelection(account=account, error_message=None, lease=lease) + + async def fake_ensure_fresh_with_budget( + _self: object, + target: Account, + **_kwargs: object, + ) -> Account: + return target + + async def fake_connect_responses_websocket(*_args: object, **_kwargs: object) -> _FakeBridgeUpstreamWebSocket: + return upstreams.popleft() + + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + payload = { + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "release the idle stream lease", + } + first = await async_client.post("/v1/responses", json=payload, headers={"session_id": "idle-release-a"}) + assert first.status_code == 200 + assert await service._load_balancer.account_pressure_snapshot(account.id) == (0, 0, 0.0) + + second = await async_client.post("/v1/responses", json=payload, headers={"session_id": "idle-release-b"}) + assert second.status_code == 200 + assert await service._load_balancer.account_pressure_snapshot(account.id) == (0, 0, 0.0) + assert not upstreams + assert len(service._http_bridge_sessions) == 2 + assert all(not session.closed for session in service._http_bridge_sessions.values()) + assert all(session.idle_ttl_seconds >= 900.0 for session in service._http_bridge_sessions.values()) + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_prefers_session_header_for_isolation(async_client, monkeypatch): + _install_bridge_settings(monkeypatch, enabled=True) + account_id = await _import_account( + async_client, + "acc_http_bridge_session_key", + "http-bridge-session-key@example.com", + ) account = await _get_account(account_id) - hanging_upstream = _SilentUpstreamWebSocket() + upstreams = [_FakeBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] + connect_count = 0 async def fake_select_account_with_budget( self, @@ -12774,316 +9288,176 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - return hanging_upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - payload = proxy_module.ResponsesRequest( - model="gpt-5.1", - instructions="Return exactly OK.", - input="gate-wait-session", - prompt_cache_key="gate-wait-session-a", - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={}, - affinity=affinity, - api_key=None, - request_id="req_gate_wait_a", - ) - session = await service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=affinity, - api_key=None, - request_model="gpt-5.1", - idle_ttl_seconds=120.0, - max_sessions=1, - ) - - # Simulate a legitimate in-flight turn holding the per-session gate. - await session.response_create_gate.acquire() - request_state, text_data = service._prepare_http_bridge_request( - payload, - {}, - api_key=None, - api_key_reservation=None, - ) - request_state.transport = "http" - assert request_state.event_queue is not None - - async def consume() -> list[str]: - return [ - chunk - async for chunk in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data=text_data, - queue_limit=8, - propagate_http_errors=False, - downstream_turn_state=None, - ) - ] - - consume_task = asyncio.create_task(consume()) - # Let at least one bounded gate acquisition attempt expire while held. - await asyncio.sleep(0.15) - assert not consume_task.done() - session.response_create_gate.release() - - enqueued = False - for _ in range(200): - async with session.pending_lock: - enqueued = request_state in session.pending_requests - if enqueued: - break - await asyncio.sleep(0.01) - assert enqueued, "queued request should submit after the gate frees" - - request_state.event_queue.put_nowait('data: {"type":"response.completed","response":{"id":"resp_gate_wait"}}\n\n') - request_state.event_queue.put_nowait(None) - chunks = await asyncio.wait_for(consume_task, timeout=5.0) + nonlocal connect_count + upstream = upstreams[connect_count] + connect_count += 1 + return upstream - event_payloads = [cast(dict[str, object], proxy_module.parse_sse_data_json(chunk)) for chunk in chunks] - event_types = [event["type"] for event in event_payloads] - assert "response.completed" in event_types - keepalives = [event for event in event_payloads if event["type"] == "codex.keepalive"] - assert keepalives, "gate contention should emit capacity-wait keepalives" - assert any(event.get("status") == "waiting_for_account_capacity" for event in keepalives) + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - await service._close_http_bridge_session(session) + payload = { + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "same-first-user-input", + } + first = await async_client.post("/v1/responses", json=payload, headers={"session_id": "session-a"}) + second = await async_client.post("/v1/responses", json=payload, headers={"session_id": "session-b"}) + + assert first.status_code == 200 + assert second.status_code == 200 + assert connect_count == 2 @pytest.mark.asyncio -async def test_http_bridge_stale_gate_retires_after_leading_rate_limit_telemetry( - app_instance, +async def test_v1_responses_http_bridge_retries_once_when_upstream_closes_before_response_created( + async_client, monkeypatch, ): - app_settings = _make_app_settings( - enabled=True, - admission_wait_timeout_seconds=0.001, - ) - app_settings.http_responses_session_bridge_stuck_gate_retire_after_seconds = 0.01 - _install_proxy_settings( - monkeypatch, - app_settings=app_settings, - dashboard_settings=_make_dashboard_settings(), - ) - service = get_proxy_service_for_app(app_instance) - upstream = _SilentUpstreamWebSocket() - key = proxy_module._HTTPBridgeSessionKey("session_header", "stale-after-rate-limits", None) - gate = asyncio.Semaphore(1) - await gate.acquire() - request_state = proxy_module._WebSocketRequestState( - request_id="req-stale-after-rate-limits", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort="high", - api_key_reservation=None, - started_at=time.monotonic() - 1.0, - transport="http", - response_create_gate=gate, - response_create_gate_acquired=True, - awaiting_response_created=True, - event_queue=asyncio.Queue(), - ) - session = proxy_module._HTTPBridgeSession( - key=key, - headers={}, - affinity=proxy_module._AffinityPolicy(key="stale-after-rate-limits"), - request_model="gpt-5.6-sol", - account=cast(Account, SimpleNamespace(id="acct-stale-rate-limits", status=AccountStatus.ACTIVE)), - upstream=cast(proxy_module.UpstreamWebSocket, upstream), - upstream_control=proxy_module._WebSocketUpstreamControl(), - pending_requests=deque([request_state]), - pending_lock=anyio.Lock(), - response_create_gate=gate, - queued_request_count=1, - last_used_at=time.monotonic(), - idle_ttl_seconds=120.0, - ) - service._http_bridge_sessions[key] = session - - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "codex.rate_limits", - "plan_type": "pro", - "rate_limits": {"allowed": True, "limit_reached": False}, - }, - separators=(",", ":"), - ), - ) - - assert request_state.latency_first_upstream_event_ms is not None - assert request_state.latency_response_created_ms is None - assert request_state.response_id is None - assert request_state.awaiting_response_created is True - assert gate.locked() is True + _install_bridge_settings(monkeypatch, enabled=True) + account_id = await _import_account(async_client, "acc_http_bridge_retry", "http-bridge-retry@example.com") + account = await _get_account(account_id) + upstreams = [_PrecreatedCloseUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] + connect_count = 0 - waiter = proxy_module._WebSocketRequestState( - request_id="req-waiting-after-rate-limits", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort="high", - api_key_reservation=None, - started_at=time.monotonic(), - transport="http", - downstream_visible=True, - ) - try: - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._acquire_request_state_response_create_admission( - waiter, - response_create_gate=gate, - bridge_session=session, - ) - finally: - if gate.locked(): - gate.release() + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + ) + return AccountSelection(account=account, error_message=None, error_code=None) - assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" - assert session.closed is True - assert key not in service._http_bridge_sessions - assert upstream.closed is True + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + nonlocal connect_count + upstream = upstreams[connect_count] + connect_count += 1 + return upstream -@pytest.mark.asyncio -async def test_http_bridge_stale_gate_direct_retirement_quarantines_wedged_reattach( - app_instance, - monkeypatch, -): - """Regression for the #1534 quarantine bypass: when the silent reattach is - the ONLY stale pending request, the stuck-gate watchdog retires the whole - session directly (no partial cleanup, no reader-failure funnel). That - direct retirement must still quarantine the key, or the next request - rebuilds the identical anchored wedge.""" - app_settings = _make_app_settings( - enabled=True, - admission_wait_timeout_seconds=0.001, - ) - app_settings.http_responses_session_bridge_stuck_gate_retire_after_seconds = 0.01 - _install_proxy_settings( - monkeypatch, - app_settings=app_settings, - dashboard_settings=_make_dashboard_settings(), - ) - service = get_proxy_service_for_app(app_instance) - http_bridge_quarantine_module._http_bridge_quarantine_registry(service).clear() - upstream = _SilentUpstreamWebSocket() - key = proxy_module._HTTPBridgeSessionKey("session_header", "quarantine-direct-retire-all-stale", None) - gate = asyncio.Semaphore(1) - await gate.acquire() - wedged_reattach = proxy_module._WebSocketRequestState( - request_id="req-wedged-direct-retire", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort="high", - api_key_reservation=None, - started_at=time.monotonic() - 1.0, - transport="http", - response_create_gate=gate, - response_create_gate_acquired=True, - awaiting_response_created=True, - event_queue=asyncio.Queue(), - ) - # The #1534 wedge shape: a proxy-injected reattach whose response.create - # was sent and that streamed response events, but whose response.created - # was never assigned. - wedged_reattach.proxy_injected_previous_response_id = True - wedged_reattach.previous_response_id = "resp_wedged_direct_retire" - wedged_reattach.response_create_sent_at = time.monotonic() - 1.0 - wedged_reattach.response_event_count = 3 - wedged_reattach.last_upstream_activity_at = time.monotonic() - 1.0 - session = proxy_module._HTTPBridgeSession( - key=key, - headers={}, - affinity=proxy_module._AffinityPolicy(key="quarantine-direct-retire-all-stale"), - request_model="gpt-5.6-sol", - account=cast(Account, SimpleNamespace(id="acct-quarantine-direct-retire", status=AccountStatus.ACTIVE)), - upstream=cast(proxy_module.UpstreamWebSocket, upstream), - upstream_control=proxy_module._WebSocketUpstreamControl(), - pending_requests=deque([wedged_reattach]), - pending_lock=anyio.Lock(), - response_create_gate=gate, - queued_request_count=1, - last_used_at=time.monotonic(), - idle_ttl_seconds=120.0, - ) - service._http_bridge_sessions[key] = session + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - waiter = proxy_module._WebSocketRequestState( - request_id="req-waiting-behind-wedged-reattach", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort="high", - api_key_reservation=None, - started_at=time.monotonic(), - transport="http", - downstream_visible=True, - ) - try: - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._acquire_request_state_response_create_admission( - waiter, - response_create_gate=gate, - bridge_session=session, - ) - finally: - if gate.locked(): - gate.release() + response = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "retry-me", + "prompt_cache_key": "retry-key", + }, + ) - assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" - assert session.closed is True - assert key not in service._http_bridge_sessions - assert upstream.closed is True - # The direct all-stale retirement must record the quarantine so the next - # request takes the fresh no-anchor path instead of re-attaching. - assert session.quarantined is True - assert http_bridge_quarantine_module._http_bridge_session_key_quarantined(service, key) is True - entry = http_bridge_quarantine_module._http_bridge_quarantine_registry(service)[key] - assert entry.reason == "reattach_missing_response_created" - http_bridge_quarantine_module._http_bridge_quarantine_registry(service).clear() + assert response.status_code == 200 + assert connect_count == 2 @pytest.mark.asyncio -async def test_codex_responses_http_bridge_replaces_retired_gate_without_client_retry( +async def test_v1_responses_http_bridge_retries_unanchored_request_when_upstream_never_acknowledges_response_create( async_client, - app_instance, monkeypatch, ): _install_bridge_settings_with_limits( monkeypatch, enabled=True, - admission_wait_timeout_seconds=0.001, ) proxy_module.get_settings().http_responses_session_bridge_stuck_gate_retire_after_seconds = 0.01 account_id = await _import_account( async_client, - "acc-http-bridge-retired-gate-replace", - "http-bridge-retired-gate-replace@example.com", + "acc_http_bridge_missing_created_retry", + "http-bridge-missing-created-retry@example.com", ) account = await _get_account(account_id) - service = get_proxy_service_for_app(app_instance) - replacement_upstream = _FakeBridgeUpstreamWebSocket() - monkeypatch.setattr( - service, - "_select_account_with_budget", - AsyncMock(return_value=AccountSelection(account=account, error_message=None, error_code=None)), - ) - monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + silent_upstream = _SilentUpstreamWebSocket() + recovered_upstream = _FakeBridgeUpstreamWebSocket() + upstreams = [silent_upstream, recovered_upstream] + connect_count = 0 + + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + api_key, + ) + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target async def fake_connect_responses_websocket( headers, @@ -13094,103 +9468,201 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - return replacement_upstream + nonlocal connect_count + upstream = upstreams[connect_count] + connect_count += 1 + return upstream + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - request_headers = { - "session_id": "retired-gate-replace", - "x-codex-turn-state": "http_turn_retired_gate_replace", - "user-agent": "codex_cli_rs/0.145.0", - } - payload = proxy_module.ResponsesRequest( - model="gpt-5.6-sol", - instructions="Return exactly OK.", - input="continue after stale gate", - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - request_headers, - codex_session_affinity=True, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, + response = await asyncio.wait_for( + async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "retry missing response.created", + "prompt_cache_key": "missing-created-retry-key", + }, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - key = proxy_module._HTTPBridgeSessionKey("session_header", "retired-gate-replace", None) - stale_upstream = _SilentUpstreamWebSocket() - gate = asyncio.Semaphore(1) - await gate.acquire() - stale_request = proxy_module._WebSocketRequestState( - request_id="req-stale-gate-owner", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort="high", - api_key_reservation=None, - started_at=time.monotonic() - 1.0, - transport="http", - response_create_gate=gate, - response_create_gate_acquired=True, - awaiting_response_created=True, - event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.6-sol"}', + + assert response.status_code == 200 + assert connect_count == 2 + assert silent_upstream.closed is True + assert len(silent_upstream.sent_text) == 1 + assert len(recovered_upstream.sent_text) == 1 + assert silent_upstream.sent_text == recovered_upstream.sent_text + + +@pytest.mark.asyncio +async def test_backend_responses_http_bridge_retries_precreated_server_overload(async_client, monkeypatch): + _install_bridge_settings(monkeypatch, enabled=True) + account_id = await _import_account( + async_client, + "acc_http_bridge_server_overload", + "http-bridge-server-overload@example.com", ) - stale_session = proxy_module._HTTPBridgeSession( - key=key, - headers=request_headers, - affinity=affinity, - request_model="gpt-5.6-sol", - account=account, - upstream=cast(proxy_module.UpstreamWebSocket, stale_upstream), - upstream_control=proxy_module._WebSocketUpstreamControl(), - pending_requests=deque([stale_request]), - pending_lock=anyio.Lock(), - response_create_gate=gate, - queued_request_count=1, - last_used_at=time.monotonic(), - idle_ttl_seconds=120.0, - codex_session=True, + account = await _get_account(account_id) + upstreams = [_PrecreatedOverloadUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] + connect_count = 0 + + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + api_key, + ) + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + nonlocal connect_count + upstream = upstreams[connect_count] + connect_count += 1 + return upstream + + async def fail_legacy_stream(*args, **kwargs): + raise AssertionError("legacy core_stream_responses path must not be used when HTTP bridge is enabled") + + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_legacy_stream) + + events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "retry-overload", + "prompt_cache_key": "server-overload-retry-key", + "stream": True, + }, ) - stale_session.downstream_turn_state_aliases.add("http_turn_retired_gate_replace") - service._http_bridge_sessions[key] = stale_session - service._http_bridge_turn_state_index[ - proxy_module._http_bridge_turn_state_alias_key("http_turn_retired_gate_replace", None) - ] = key - try: - response = await asyncio.wait_for( - async_client.post( - "/backend-api/codex/responses", - json=payload.to_payload(), - headers=request_headers, - ), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, - ) - finally: - if gate.locked(): - gate.release() + _assert_created_text_delta_completed(events) + assert events[-1]["response"]["id"] == "resp_bridge_1" + assert connect_count == 2 + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_rejects_oversized_response_create_before_upstream( + async_client, + monkeypatch, + tmp_path, +): + _install_bridge_settings(monkeypatch, enabled=True) + monkeypatch.setattr(proxy_module, "_UPSTREAM_RESPONSE_CREATE_WARN_BYTES", 64) + monkeypatch.setattr(proxy_module, "_UPSTREAM_RESPONSE_CREATE_MAX_BYTES", 128) + monkeypatch.setattr(proxy_module, "_OVERSIZED_RESPONSE_CREATE_DUMP_DIR", tmp_path) + + async def fail_get_or_create_http_bridge_session(self, *args, **kwargs): + del self, args, kwargs + raise AssertionError("oversized response.create must fail before upstream bridge session allocation") - assert response.status_code == 200 - assert stale_session.closed is True - assert stale_upstream.closed is True - assert len(replacement_upstream.sent_text) == 1 - assert any( - current_session is not stale_session and current_session.upstream is replacement_upstream - for current_session in service._http_bridge_sessions.values() + monkeypatch.setattr( + proxy_module.ProxyService, + "_get_or_create_http_bridge_session", + fail_get_or_create_http_bridge_session, ) + request_json = { + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "x" * 256}]}], + "prompt_cache_key": "oversized-http-bridge", + } + + response = await async_client.post("/v1/responses", json=request_json) + + assert response.status_code == 400 + payload = response.json() + assert payload["error"]["code"] == "payload_too_large" + assert payload["error"]["type"] == "invalid_request_error" + assert payload["error"]["param"] == "input" + assert "response.create is too large for upstream websocket" in payload["error"]["message"] + + meta_files = list(tmp_path.glob("*.meta.json")) + assert len(meta_files) == 1 + meta = json.loads(meta_files[0].read_text(encoding="utf-8")) + assert meta["reason"]["error_code"] == "payload_too_large" + assert meta["request"]["transport"] == "http" + assert meta["request"]["request_text_bytes"] > 128 + + duplicate_response = await async_client.post("/v1/responses", json=request_json) + assert duplicate_response.status_code == 400 + assert len(list(tmp_path.glob("*.response-create.json.gz"))) == 1 + assert len(list(tmp_path.glob("*.meta.json"))) == 1 + + meta_files[0].unlink() + orphan_retry_response = await async_client.post("/v1/responses", json=request_json) + assert orphan_retry_response.status_code == 400 + complete_pairs = [ + dump_path + for dump_path in tmp_path.glob("*.response-create.json.gz") + if (tmp_path / f"{dump_path.name[: -len('.response-create.json.gz')]}.meta.json").exists() + ] + assert complete_pairs + @pytest.mark.asyncio -async def test_v1_responses_http_bridge_enforces_queue_limit_atomically_for_same_session( +async def test_v1_responses_http_bridge_slims_historical_inline_artifacts_and_succeeds( async_client, - app_instance, monkeypatch, ): - _install_bridge_settings_with_limits(monkeypatch, enabled=True, queue_limit=1) - account_id = await _import_account(async_client, "acc_http_bridge_queue", "http-bridge-queue@example.com") - service = get_proxy_service_for_app(app_instance) + _install_bridge_settings(monkeypatch, enabled=True) + monkeypatch.setattr(proxy_module, "_UPSTREAM_RESPONSE_CREATE_WARN_BYTES", 64) + monkeypatch.setattr(proxy_module, "_UPSTREAM_RESPONSE_CREATE_MAX_BYTES", 640) + account_id = await _import_account(async_client, "acc_http_bridge_slim", "http-bridge-slim@example.com") account = await _get_account(account_id) - hanging_upstream = _SilentUpstreamWebSocket() + fake_upstream = _FakeBridgeUpstreamWebSocket() async def fake_select_account_with_budget( self, @@ -13227,6 +9699,7 @@ async def fake_select_account_with_budget( model, exclude_account_ids, additional_limit_name, + api_key, ) return AccountSelection(account=account, error_message=None, error_code=None) @@ -13243,889 +9716,1096 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - return hanging_upstream + return fake_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - payload = proxy_module.ResponsesRequest( - model="gpt-5.1", - instructions="Return exactly OK.", - input="same-session", - prompt_cache_key="same-session-key", - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={}, - affinity=affinity, - api_key=None, - request_id="req_queue", - ) - session = await service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=affinity, - api_key=None, - request_model="gpt-5.1", - idle_ttl_seconds=120.0, - max_sessions=128, - ) - - first_state, first_text = service._prepare_http_bridge_request(payload, {}, api_key=None, api_key_reservation=None) - first_state.transport = "http" - session.unanchored_reservation_id = "scope-submit" - request_scope_token = set_request_scope_id("scope-submit") - try: - await service._submit_http_bridge_request( - session, - request_state=first_state, - text_data=first_text, - queue_limit=1, - ) - finally: - reset_request_scope_id(request_scope_token) - - assert session.unanchored_reservation_id is None - - second_state, second_text = service._prepare_http_bridge_request( - payload, {}, api_key=None, api_key_reservation=None - ) - second_state.transport = "http" - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._submit_http_bridge_request( - session, - request_state=second_state, - text_data=second_text, - queue_limit=1, - ) - - exc = exc_info.value - assert exc.status_code == 429 - assert session.queued_request_count == 1 - await service._close_http_bridge_session(session) - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_creates_different_session_keys_in_parallel(app_instance, monkeypatch): - service = get_proxy_service_for_app(app_instance) - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() - - _install_proxy_settings( - monkeypatch, - app_settings=_make_app_settings( - enabled=True, - max_sessions=8, - codex_idle_ttl_seconds=120.0, - instance_id="instance-a", - instance_ring=[], - ), - dashboard_settings=_make_dashboard_settings(), - ) - - create_started: list[str] = [] - create_started_events = { - "bridge-a": asyncio.Event(), - "bridge-b": asyncio.Event(), - } - release_create = asyncio.Event() - - async def fake_create_http_bridge_session( - self, - key, - *, - headers, - affinity, - api_key, - request_model, - idle_ttl_seconds, - request_stage="first_turn", - preferred_account_id=None, - require_preferred_account=False, - fallback_on_preferred_account_unavailable=True, - ): - del ( - self, - headers, - affinity, - request_model, - idle_ttl_seconds, - request_stage, - preferred_account_id, - require_preferred_account, - fallback_on_preferred_account_unavailable, - ) - create_started.append(key.affinity_key) - create_started_events[key.affinity_key].set() - await _wait_for_event(release_create) - return _make_dummy_bridge_session(key) - - monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - - key_one = proxy_module._HTTPBridgeSessionKey("request", "bridge-a", None) - key_two = proxy_module._HTTPBridgeSessionKey("request", "bridge-b", None) - - try: - first = asyncio.create_task( - service._get_or_create_http_bridge_session( - key_one, - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - ) - ) - second = asyncio.create_task( - service._get_or_create_http_bridge_session( - key_two, - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - ) - ) - await _wait_for_event(create_started_events["bridge-a"]) - await _wait_for_event(create_started_events["bridge-b"]) - assert key_one in service._http_bridge_inflight_sessions - assert key_two in service._http_bridge_inflight_sessions - - release_create.set() - session_one, session_two = await asyncio.gather(first, second) + response = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": [ + {"role": "user", "content": [{"type": "input_text", "text": "old turn"}]}, + { + "type": "function_call_output", + "call_id": "call_1", + "output": "data:image/png;base64," + ("A" * 1500), + }, + {"role": "assistant", "content": [{"type": "output_text", "text": "done"}]}, + {"role": "user", "content": [{"type": "input_text", "text": "ping"}]}, + ], + "prompt_cache_key": "slim-http-bridge", + }, + ) - assert sorted(create_started) == ["bridge-a", "bridge-b"] - assert session_one.key == key_one - assert session_two.key == key_two - assert service._http_bridge_sessions[key_one] is session_one - assert service._http_bridge_sessions[key_two] is session_two - finally: - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() + assert response.status_code == 200 + sent_payload = json.loads(fake_upstream.sent_text[0]) + assert sent_payload["input"][-1]["content"][0]["text"] == "ping" + assert "data:image/" not in json.dumps(sent_payload["input"], ensure_ascii=True) + assert "historical tool output" in json.dumps(sent_payload["input"], ensure_ascii=True) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_singleflights_same_session_key_during_creation(app_instance, monkeypatch): +async def test_v1_responses_http_bridge_does_not_evict_active_session_when_pool_is_full( + async_client, + app_instance, + monkeypatch, +): + _install_bridge_settings_with_limits(monkeypatch, enabled=True, max_sessions=1) + account_id = await _import_account(async_client, "acc_http_bridge_capacity", "http-bridge-capacity@example.com") service = get_proxy_service_for_app(app_instance) - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() - - _install_proxy_settings( - monkeypatch, - app_settings=_make_app_settings( - enabled=True, - max_sessions=8, - admission_wait_timeout_seconds=1.0, - codex_idle_ttl_seconds=120.0, - instance_id="instance-a", - instance_ring=[], - ), - dashboard_settings=_make_dashboard_settings(), - ) - - create_started: list[str] = [] - create_started_event = asyncio.Event() - release_create = asyncio.Event() + account = await _get_account(account_id) + hanging_upstream = _CreatedOnlyUpstreamWebSocket() - async def fake_create_http_bridge_session( + async def fake_select_account_with_budget( self, - key, + deadline, *, - headers, - affinity, - api_key, - request_model, - idle_ttl_seconds, + request_id, + kind, request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, preferred_account_id=None, - require_preferred_account=False, - fallback_on_preferred_account_unavailable=True, ): + del preferred_account_id del ( self, - headers, - affinity, - request_model, - idle_ttl_seconds, + deadline, + request_id, + kind, request_stage, - preferred_account_id, - require_preferred_account, - fallback_on_preferred_account_unavailable, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, ) - create_started.append(key.affinity_key) - create_started_event.set() - await _wait_for_event(release_create) - return _make_dummy_bridge_session(key) - - monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) + return AccountSelection(account=account, error_message=None, error_code=None) - key = proxy_module._HTTPBridgeSessionKey("request", "bridge-singleflight", None) + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target - try: - first = asyncio.create_task( - service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - ) - ) - await _wait_for_event(create_started_event) - assert create_started == ["bridge-singleflight"] - assert key in service._http_bridge_inflight_sessions + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + return hanging_upstream - second = asyncio.create_task( - service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + first_payload = proxy_module.ResponsesRequest( + model="gpt-5.1", + instructions="Return exactly OK.", + input="hold-open", + prompt_cache_key="active-session-a", + ) + first_affinity = proxy_module._sticky_key_for_responses_request( + first_payload, + {}, + codex_session_affinity=False, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + first_key = proxy_module._make_http_bridge_session_key( + first_payload, + headers={}, + affinity=first_affinity, + api_key=None, + request_id="req_a", + ) + first_session = await service._get_or_create_http_bridge_session( + first_key, + headers={}, + affinity=first_affinity, + api_key=None, + request_model="gpt-5.1", + idle_ttl_seconds=120.0, + max_sessions=1, + ) + async with first_session.pending_lock: + first_session.pending_requests.append( + proxy_module._WebSocketRequestState( + request_id="req-active", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + response_create_gate_acquired=True, + event_queue=asyncio.Queue(), + transport="http", ) ) - await asyncio.sleep(0) - assert create_started == ["bridge-singleflight"] - assert not second.done() - - release_create.set() - session_one, session_two = await asyncio.gather(first, second) - - assert create_started == ["bridge-singleflight"] - assert session_one is session_two - assert service._http_bridge_sessions[key] is session_one - finally: - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_inflight_waiter_rejects_service_tier_provenance_mismatch( - app_instance, - monkeypatch, -): - service = get_proxy_service_for_app(app_instance) - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() - - _install_proxy_settings( - monkeypatch, - app_settings=_make_app_settings( - enabled=True, - max_sessions=8, - admission_wait_timeout_seconds=1.0, - codex_idle_ttl_seconds=120.0, - instance_id="instance-a", - instance_ring=[], - ), - dashboard_settings=_make_dashboard_settings(), + second_payload = proxy_module.ResponsesRequest( + model="gpt-5.1", + instructions="Return exactly OK.", + input="new-session", + prompt_cache_key="active-session-b", ) + second_affinity = proxy_module._sticky_key_for_responses_request( + second_payload, + {}, + codex_session_affinity=False, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + second_key = proxy_module._make_http_bridge_session_key( + second_payload, + headers={}, + affinity=second_affinity, + api_key=None, + request_id="req_b", + ) + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + second_key, + headers={}, + affinity=second_affinity, + api_key=None, + request_model="gpt-5.1", + idle_ttl_seconds=120.0, + max_sessions=1, + ) + exc = exc_info.value + assert exc.status_code == 429 + assert hanging_upstream.closed is False + await service._close_http_bridge_session(first_session) - account_id = None - - class Registry: - def account_ids_for_model(self, model: str) -> set[str]: - assert model == "gpt-5.3-codex-spark" - return set() - - def plan_types_for_model(self, model: str) -> set[str]: - assert model == "gpt-5.3-codex-spark" - return {"pro"} - - def account_ids_for_model_service_tier(self, model: str, service_tier: str) -> set[str]: - assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") - return set() - - def plan_types_for_model_service_tier(self, model: str, service_tier: str) -> set[str]: - assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") - return {"pro"} - - def get_snapshot(self): - return SimpleNamespace(account_plans={account_id: "pro"}) - - monkeypatch.setattr(proxy_support, "get_model_registry", lambda: Registry()) - create_service_tiers: list[str | None] = [] - create_started_event = asyncio.Event() - release_first_create = asyncio.Event() +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_times_out_queued_request_on_bounded_startup_gate_wait( + async_client, + app_instance, + monkeypatch, +): + _install_bridge_settings_with_limits( + monkeypatch, + enabled=True, + max_sessions=1, + # Queued HTTP bridge requests have already claimed the bridge queue slot, + # so we still wait for the per-session response-create gate, but only + # until the bounded startup timeout. + admission_wait_timeout_seconds=0.01, + ) + account_id = await _import_account( + async_client, + "acc_http_bridge_queued_capacity", + "http-bridge-queued@example.com", + ) + service = get_proxy_service_for_app(app_instance) + account = await _get_account(account_id) + hanging_upstream = _SilentUpstreamWebSocket() - async def fake_create_http_bridge_session( + async def fake_select_account_with_budget( self, - key, + deadline, *, - headers, - affinity, - api_key, - request_model, - request_service_tier=None, - idle_ttl_seconds, + request_id, + kind, request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, preferred_account_id=None, - require_preferred_account=False, - fallback_on_preferred_account_unavailable=True, ): + del preferred_account_id del ( self, - headers, - affinity, - api_key, - idle_ttl_seconds, + deadline, + request_id, + kind, request_stage, - preferred_account_id, - require_preferred_account, - fallback_on_preferred_account_unavailable, - ) - create_service_tiers.append(request_service_tier) - if len(create_service_tiers) == 1: - create_started_event.set() - await _wait_for_event(release_first_create) - session = _make_dummy_bridge_session(key) - session.account = cast( - Account, - SimpleNamespace(id=account_id, status=AccountStatus.ACTIVE, plan_type="pro"), - ) - session.request_model = request_model - session.request_service_tier = request_service_tier - session.catalog_omission_quota_admission = CatalogOmissionQuotaAdmission( - normalized_model=request_model, - canonical_quota_key="codex_spark", - normalized_effective_service_tier=request_service_tier, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, ) - return session + return AccountSelection(account=account, error_message=None, error_code=None) - monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target - key = proxy_module._HTTPBridgeSessionKey("request", "bridge-inflight-tier", None) + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + return hanging_upstream - try: - first = asyncio.create_task( - service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.3-codex-spark", - request_service_tier=None, - idle_ttl_seconds=120.0, - max_sessions=8, - ) - ) - await _wait_for_event(create_started_event) + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - second = asyncio.create_task( - service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.3-codex-spark", - request_service_tier="priority", - idle_ttl_seconds=120.0, - max_sessions=8, - ) + first_payload = proxy_module.ResponsesRequest( + model="gpt-5.1", + instructions="Return exactly OK.", + input="queued-session", + prompt_cache_key="queued-session-a", + ) + first_affinity = proxy_module._sticky_key_for_responses_request( + first_payload, + {}, + codex_session_affinity=False, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + first_key = proxy_module._make_http_bridge_session_key( + first_payload, + headers={}, + affinity=first_affinity, + api_key=None, + request_id="req_queue_a", + ) + first_session = await service._get_or_create_http_bridge_session( + first_key, + headers={}, + affinity=first_affinity, + api_key=None, + request_model="gpt-5.1", + idle_ttl_seconds=120.0, + max_sessions=1, + ) + + await first_session.response_create_gate.acquire() + request_state, text_data = service._prepare_http_bridge_request( + first_payload, + {}, + api_key=None, + api_key_reservation=None, + ) + request_state.transport = "http" + submit_task = asyncio.create_task( + service._submit_http_bridge_request( + first_session, + request_state=request_state, + text_data=text_data, + queue_limit=8, ) - await asyncio.sleep(0) - assert not second.done() + ) + await asyncio.sleep(0) + await asyncio.sleep(0.05) + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await asyncio.wait_for(submit_task, timeout=0.2) + exc = exc_info.value + assert exc.status_code == 429 + assert exc.payload["error"]["code"] == "response_create_gate_timeout" + assert exc.payload["error"]["type"] == "rate_limit_error" - release_first_create.set() - first_session, second_session = await asyncio.gather(first, second) + assert await service._http_bridge_pending_count(first_session) == 0 + async with first_session.pending_lock: + assert list(first_session.pending_requests) == [] + assert first_session.queued_request_count == 0 - assert create_service_tiers == [None, "priority"] - assert first_session is not second_session - assert first_session.request_service_tier is None - assert second_session.request_service_tier == "priority" - assert first_session.closed is False - assert service._http_bridge_sessions[key] is first_session - assert second_session.key.affinity_kind == "internal_request_parallel" - assert service._http_bridge_sessions[second_session.key] is second_session - finally: - release_first_create.set() - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() + first_session.response_create_gate.release() + await service._close_http_bridge_session(first_session) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_waits_for_inflight_capacity_before_rate_limiting_other_keys( - app_instance, monkeypatch +async def test_v1_responses_http_bridge_gate_wait_is_clamped_to_remaining_budget( + async_client, + app_instance, + monkeypatch, ): - service = get_proxy_service_for_app(app_instance) - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() - - _install_proxy_settings( + _install_bridge_settings_with_limits( monkeypatch, - app_settings=_make_app_settings( - enabled=True, - max_sessions=1, - codex_idle_ttl_seconds=120.0, - instance_id="instance-a", - instance_ring=[], - ), - dashboard_settings=_make_dashboard_settings(), + enabled=True, + max_sessions=1, + admission_wait_timeout_seconds=5.0, + ) + account_id = await _import_account( + async_client, + "acc_http_bridge_budget_clamp", + "http-bridge-budget-clamp@example.com", ) + service = get_proxy_service_for_app(app_instance) + account = await _get_account(account_id) + hanging_upstream = _SilentUpstreamWebSocket() - first_create_started = asyncio.Event() - release_first_create = asyncio.Event() - create_attempts: list[str] = [] - - async def fake_create_http_bridge_session( + async def fake_select_account_with_budget( self, - key, + deadline, *, - headers, - affinity, - api_key, - request_model, - idle_ttl_seconds, + request_id, + kind, request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, preferred_account_id=None, - require_preferred_account=False, - fallback_on_preferred_account_unavailable=True, ): + del preferred_account_id del ( self, - headers, - affinity, - request_model, - idle_ttl_seconds, + deadline, + request_id, + kind, request_stage, - preferred_account_id, - require_preferred_account, - fallback_on_preferred_account_unavailable, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, ) - create_attempts.append(key.affinity_key) - if key.affinity_key == "bridge-capacity-a": - first_create_started.set() - await _wait_for_event(release_first_create) - raise RuntimeError("first create failed") - return _make_dummy_bridge_session(key) + return AccountSelection(account=account, error_message=None, error_code=None) - monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target - key_one = proxy_module._HTTPBridgeSessionKey("request", "bridge-capacity-a", None) - key_two = proxy_module._HTTPBridgeSessionKey("request", "bridge-capacity-b", None) + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + return hanging_upstream - first = asyncio.create_task( - service._get_or_create_http_bridge_session( - key_one, - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=1, - ) + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + payload = proxy_module.ResponsesRequest( + model="gpt-5.1", + instructions="Return exactly OK.", + input="budget-clamp-session", + prompt_cache_key="budget-clamp-session-a", + ) + affinity = proxy_module._sticky_key_for_responses_request( + payload, + {}, + codex_session_affinity=False, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + key = proxy_module._make_http_bridge_session_key( + payload, + headers={}, + affinity=affinity, + api_key=None, + request_id="req_budget_clamp_a", + ) + session = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=affinity, + api_key=None, + request_model="gpt-5.1", + idle_ttl_seconds=120.0, + max_sessions=1, ) - await _wait_for_event(first_create_started) - second = asyncio.create_task( - service._get_or_create_http_bridge_session( - key_two, - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=1, - ) + await session.response_create_gate.acquire() + request_state, text_data = service._prepare_http_bridge_request( + payload, + {}, + api_key=None, + api_key_reservation=None, ) - await asyncio.sleep(0.01) - assert not second.done() + request_state.transport = "http" + # Age the request so only ~0.2s of the bridge request budget remains: + # the gate wait must be clamped to that tail, not run the full 5s + # admission timeout past the budget. + budget_seconds = proxy_module._http_bridge_request_budget_seconds(proxy_module.get_settings()) + request_state.started_at = time.monotonic() - (budget_seconds - 0.2) - release_first_create.set() + started = time.monotonic() + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=text_data, + queue_limit=8, + ) + elapsed = time.monotonic() - started - with pytest.raises(RuntimeError, match="first create failed"): - await first - created_session = await asyncio.wait_for(second, timeout=1.0) + assert exc_info.value.status_code == 429 + assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" + assert elapsed < 2.0, f"gate wait ran past the remaining budget: {elapsed:.2f}s" - assert create_attempts == ["bridge-capacity-a", "bridge-capacity-b"] - assert service._http_bridge_sessions[key_two] is created_session - assert key_one not in service._http_bridge_inflight_sessions - assert key_two not in service._http_bridge_inflight_sessions + session.response_create_gate.release() + await service._close_http_bridge_session(session) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_forks_parallel_unanchored_session_requests( +async def test_v1_responses_http_bridge_gate_contention_waits_and_completes_after_release( + async_client, app_instance, monkeypatch, ): - service = get_proxy_service_for_app(app_instance) - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() - - _install_proxy_settings( + _install_bridge_settings_with_limits( monkeypatch, - app_settings=_make_app_settings( - enabled=True, - max_sessions=8, - admission_wait_timeout_seconds=1.0, - codex_idle_ttl_seconds=120.0, - instance_id="instance-a", - instance_ring=[], - ), - dashboard_settings=_make_dashboard_settings(), + enabled=True, + max_sessions=1, + admission_wait_timeout_seconds=0.05, ) + monkeypatch.setattr(http_bridge_streaming_module, "_RESPONSE_CREATE_GATE_RETRY_SLEEP_SECONDS", 0.02) + monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 0.02) + account_id = await _import_account( + async_client, + "acc_http_bridge_gate_wait", + "http-bridge-gate-wait@example.com", + ) + service = get_proxy_service_for_app(app_instance) + account = await _get_account(account_id) + hanging_upstream = _SilentUpstreamWebSocket() - created_keys: list[proxy_module._HTTPBridgeSessionKey] = [] - - async def fake_create_http_bridge_session( + async def fake_select_account_with_budget( self, - key, + deadline, *, - headers, - affinity, - api_key, - request_model, - idle_ttl_seconds, + request_id, + kind, request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, preferred_account_id=None, - require_preferred_account=False, - fallback_on_preferred_account_unavailable=True, ): + del preferred_account_id del ( self, - headers, - affinity, - api_key, - idle_ttl_seconds, + deadline, + request_id, + kind, request_stage, - preferred_account_id, - require_preferred_account, - fallback_on_preferred_account_unavailable, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, ) - created_keys.append(key) - session = _make_dummy_bridge_session(key) - session.request_model = request_model - return session + return AccountSelection(account=account, error_message=None, error_code=None) - async def fake_claim_durable_http_bridge_session( - self, - session, + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, *, - allow_takeover, - force_owner_epoch_advance=False, - expected_takeover_owner_instance_id=None, - expected_takeover_owner_process_epoch=None, + base_url=None, + session=None, ): - del ( - self, - session, - allow_takeover, - force_owner_epoch_advance, - expected_takeover_owner_instance_id, - expected_takeover_owner_process_epoch, - ) + del headers, access_token, account_id_header, base_url, session + return hanging_upstream - monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - monkeypatch.setattr( - proxy_module.ProxyService, - "_claim_durable_http_bridge_session", - fake_claim_durable_http_bridge_session, + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + payload = proxy_module.ResponsesRequest( + model="gpt-5.1", + instructions="Return exactly OK.", + input="gate-wait-session", + prompt_cache_key="gate-wait-session-a", + ) + affinity = proxy_module._sticky_key_for_responses_request( + payload, + {}, + codex_session_affinity=False, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + key = proxy_module._make_http_bridge_session_key( + payload, + headers={}, + affinity=affinity, + api_key=None, + request_id="req_gate_wait_a", + ) + session = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=affinity, + api_key=None, + request_model="gpt-5.1", + idle_ttl_seconds=120.0, + max_sessions=1, ) - shared_key = proxy_module._HTTPBridgeSessionKey("session_header", "shared-codex-process", None) - foreground = _make_dummy_bridge_session(shared_key) - foreground.request_model = "gpt-5.6-sol" - foreground.queued_request_count = 1 - service._http_bridge_sessions[shared_key] = foreground + # Simulate a legitimate in-flight turn holding the per-session gate. + await session.response_create_gate.acquire() + request_state, text_data = service._prepare_http_bridge_request( + payload, + {}, + api_key=None, + api_key_reservation=None, + ) + request_state.transport = "http" + assert request_state.event_queue is not None - async def get_memory_session(request_scope_id: str) -> proxy_module._HTTPBridgeSession: - request_id_token = set_request_id("duplicate-client-request-id") - request_scope_token = set_request_scope_id(request_scope_id) - try: - return await service._get_or_create_http_bridge_session( - shared_key, - headers={"session_id": "shared-codex-process"}, - affinity=proxy_module._AffinityPolicy( - key="shared-codex-process", - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ), - api_key=None, - request_model="gpt-5.4-mini", - idle_ttl_seconds=120.0, - max_sessions=8, + async def consume() -> list[str]: + return [ + chunk + async for chunk in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data=text_data, + queue_limit=8, + propagate_http_errors=False, + downstream_turn_state=None, ) - finally: - reset_request_scope_id(request_scope_token) - reset_request_id(request_id_token) + ] - try: - first_memory, second_memory = await asyncio.gather( - get_memory_session("memory-request-a"), - get_memory_session("memory-request-b"), - ) + consume_task = asyncio.create_task(consume()) + # Let at least one bounded gate acquisition attempt expire while held. + await asyncio.sleep(0.15) + assert not consume_task.done() + session.response_create_gate.release() - assert first_memory is not foreground - assert second_memory is not foreground - assert first_memory is not second_memory - assert foreground.request_model == "gpt-5.6-sol" - assert {key.affinity_kind for key in created_keys} == {"internal_unanchored_parallel"} - assert len({key.affinity_key for key in created_keys}) == 2 - assert all(key.strength == "hard" for key in created_keys) - finally: - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() + enqueued = False + for _ in range(200): + async with session.pending_lock: + enqueued = request_state in session.pending_requests + if enqueued: + break + await asyncio.sleep(0.01) + assert enqueued, "queued request should submit after the gate frees" + + request_state.event_queue.put_nowait('data: {"type":"response.completed","response":{"id":"resp_gate_wait"}}\n\n') + request_state.event_queue.put_nowait(None) + chunks = await asyncio.wait_for(consume_task, timeout=5.0) + + event_payloads = [cast(dict[str, object], proxy_module.parse_sse_data_json(chunk)) for chunk in chunks] + event_types = [event["type"] for event in event_payloads] + assert "response.completed" in event_types + keepalives = [event for event in event_payloads if event["type"] == "codex.keepalive"] + assert keepalives, "gate contention should emit capacity-wait keepalives" + assert any(event.get("status") == "waiting_for_account_capacity" for event in keepalives) + + await service._close_http_bridge_session(session) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_reserved_handoff_forks_before_submit( +async def test_http_bridge_stale_gate_retires_after_leading_rate_limit_telemetry( app_instance, monkeypatch, ): - service = get_proxy_service_for_app(app_instance) - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() - + app_settings = _make_app_settings( + enabled=True, + admission_wait_timeout_seconds=0.001, + ) + app_settings.http_responses_session_bridge_stuck_gate_retire_after_seconds = 0.01 _install_proxy_settings( monkeypatch, - app_settings=_make_app_settings( - enabled=True, - max_sessions=8, - admission_wait_timeout_seconds=1.0, - codex_idle_ttl_seconds=120.0, - instance_id="instance-a", - instance_ring=[], - ), + app_settings=app_settings, dashboard_settings=_make_dashboard_settings(), ) + service = get_proxy_service_for_app(app_instance) + upstream = _SilentUpstreamWebSocket() + key = proxy_module._HTTPBridgeSessionKey("session_header", "stale-after-rate-limits", None) + gate = asyncio.Semaphore(1) + await gate.acquire() + request_state = proxy_module._WebSocketRequestState( + request_id="req-stale-after-rate-limits", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic() - 1.0, + transport="http", + response_create_gate=gate, + response_create_gate_acquired=True, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + ) + session = proxy_module._HTTPBridgeSession( + key=key, + headers={}, + affinity=proxy_module._AffinityPolicy(key="stale-after-rate-limits"), + request_model="gpt-5.6-sol", + account=cast(Account, SimpleNamespace(id="acct-stale-rate-limits", status=AccountStatus.ACTIVE)), + upstream=cast(proxy_module.UpstreamWebSocket, upstream), + upstream_control=proxy_module._WebSocketUpstreamControl(), + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + response_create_gate=gate, + queued_request_count=1, + last_used_at=time.monotonic(), + idle_ttl_seconds=120.0, + ) + service._http_bridge_sessions[key] = session - shared_key = proxy_module._HTTPBridgeSessionKey("session_header", "shared-codex-process", None) - - async def fake_create_http_bridge_session(self, key, **kwargs): - del self - session = _make_dummy_bridge_session(key) - session.request_model = kwargs["request_model"] - return session - - monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - monkeypatch.setattr(proxy_module.ProxyService, "_claim_durable_http_bridge_session", AsyncMock()) + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "codex.rate_limits", + "plan_type": "pro", + "rate_limits": {"allowed": True, "limit_reached": False}, + }, + separators=(",", ":"), + ), + ) - async def get_session(request_scope_id: str) -> proxy_module._HTTPBridgeSession: - request_id_token = set_request_id("duplicate-client-request-id") - request_scope_token = set_request_scope_id(request_scope_id) - try: - return await service._get_or_create_http_bridge_session( - shared_key, - headers={"session_id": "shared-codex-process"}, - affinity=proxy_module._AffinityPolicy( - key="shared-codex-process", - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ), - api_key=None, - request_model="gpt-5.6-sol", - idle_ttl_seconds=120.0, - max_sessions=8, - ) - finally: - reset_request_scope_id(request_scope_token) - reset_request_id(request_id_token) + assert request_state.latency_first_upstream_event_ms is not None + assert request_state.latency_response_created_ms is None + assert request_state.response_id is None + assert request_state.awaiting_response_created is True + assert gate.locked() is True - first = await get_session("scope-before-submit-a") - _reserve_http_bridge_unanchored_handoff(first, request_scope_id="scope-before-submit-a") - first.last_used_at = time.monotonic() - 300.0 - first.idle_ttl_seconds = 1.0 + waiter = proxy_module._WebSocketRequestState( + request_id="req-waiting-after-rate-limits", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + downstream_visible=True, + ) try: - second = await get_session("scope-before-submit-b") + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._acquire_request_state_response_create_admission( + waiter, + response_create_gate=gate, + bridge_session=session, + ) finally: - _release_http_bridge_unanchored_handoff(first, request_scope_id="scope-before-submit-a") + if gate.locked(): + gate.release() - assert first.key == shared_key - assert service._http_bridge_sessions[shared_key] is first - assert first.closed is False - assert first.queued_request_count == 0 - assert first.unanchored_reservation_id is None - assert second is not first - assert second.key.affinity_kind == "internal_unanchored_parallel" - assert second.key.strength == "hard" + assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" + assert session.closed is True + assert key not in service._http_bridge_sessions + assert upstream.closed is True @pytest.mark.asyncio -async def test_v1_responses_http_bridge_reused_unanchored_refresh_reserves_canonical_handoff( +async def test_http_bridge_stale_gate_direct_retirement_quarantines_wedged_reattach( app_instance, monkeypatch, ): - service = get_proxy_service_for_app(app_instance) - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() - + """Regression for the #1534 quarantine bypass: when the silent reattach is + the ONLY stale pending request, the stuck-gate watchdog retires the whole + session directly (no partial cleanup, no reader-failure funnel). That + direct retirement must still quarantine the key, or the next request + rebuilds the identical anchored wedge.""" + app_settings = _make_app_settings( + enabled=True, + admission_wait_timeout_seconds=0.001, + ) + app_settings.http_responses_session_bridge_stuck_gate_retire_after_seconds = 0.01 _install_proxy_settings( monkeypatch, - app_settings=_make_app_settings( - enabled=True, - max_sessions=8, - admission_wait_timeout_seconds=1.0, - codex_idle_ttl_seconds=120.0, - instance_id="instance-a", - instance_ring=[], - ), + app_settings=app_settings, dashboard_settings=_make_dashboard_settings(), ) + service = get_proxy_service_for_app(app_instance) + http_bridge_quarantine_module._http_bridge_quarantine_registry(service).clear() + upstream = _SilentUpstreamWebSocket() + key = proxy_module._HTTPBridgeSessionKey("session_header", "quarantine-direct-retire-all-stale", None) + gate = asyncio.Semaphore(1) + await gate.acquire() + wedged_reattach = proxy_module._WebSocketRequestState( + request_id="req-wedged-direct-retire", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic() - 1.0, + transport="http", + response_create_gate=gate, + response_create_gate_acquired=True, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + ) + # The #1534 wedge shape: a proxy-injected reattach whose response.create + # was sent and that streamed response events, but whose response.created + # was never assigned. + wedged_reattach.proxy_injected_previous_response_id = True + wedged_reattach.previous_response_id = "resp_wedged_direct_retire" + wedged_reattach.response_create_sent_at = time.monotonic() - 1.0 + wedged_reattach.response_event_count = 3 + wedged_reattach.last_upstream_activity_at = time.monotonic() - 1.0 + session = proxy_module._HTTPBridgeSession( + key=key, + headers={}, + affinity=proxy_module._AffinityPolicy(key="quarantine-direct-retire-all-stale"), + request_model="gpt-5.6-sol", + account=cast(Account, SimpleNamespace(id="acct-quarantine-direct-retire", status=AccountStatus.ACTIVE)), + upstream=cast(proxy_module.UpstreamWebSocket, upstream), + upstream_control=proxy_module._WebSocketUpstreamControl(), + pending_requests=deque([wedged_reattach]), + pending_lock=anyio.Lock(), + response_create_gate=gate, + queued_request_count=1, + last_used_at=time.monotonic(), + idle_ttl_seconds=120.0, + ) + service._http_bridge_sessions[key] = session - shared_key = proxy_module._HTTPBridgeSessionKey("session_header", "shared-codex-process", None) - canonical = _make_dummy_bridge_session(shared_key) - canonical.request_model = "gpt-5.6-sol" - canonical.durable_session_id = "durable-shared" - canonical.durable_owner_epoch = 1 - service._http_bridge_sessions[shared_key] = canonical - refresh_started = asyncio.Event() - allow_refresh = asyncio.Event() + waiter = proxy_module._WebSocketRequestState( + request_id="req-waiting-behind-wedged-reattach", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + downstream_visible=True, + ) + try: + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._acquire_request_state_response_create_admission( + waiter, + response_create_gate=gate, + bridge_session=session, + ) + finally: + if gate.locked(): + gate.release() - async def blocked_refresh(session): - assert session is canonical - refresh_started.set() - await allow_refresh.wait() + assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" + assert session.closed is True + assert key not in service._http_bridge_sessions + assert upstream.closed is True + # The direct all-stale retirement must record the quarantine so the next + # request takes the fresh no-anchor path instead of re-attaching. + assert session.quarantined is True + assert http_bridge_quarantine_module._http_bridge_session_key_quarantined(service, key) is True + entry = http_bridge_quarantine_module._http_bridge_quarantine_registry(service)[key] + assert entry.reason == "reattach_missing_response_created" + http_bridge_quarantine_module._http_bridge_quarantine_registry(service).clear() - async def fake_create_http_bridge_session(self, key, **kwargs): - del self - session = _make_dummy_bridge_session(key) - session.request_model = kwargs["request_model"] - return session - monkeypatch.setattr(service, "_refresh_durable_http_bridge_session", blocked_refresh) +@pytest.mark.asyncio +async def test_codex_responses_http_bridge_replaces_retired_gate_without_client_retry( + async_client, + app_instance, + monkeypatch, +): + _install_bridge_settings_with_limits( + monkeypatch, + enabled=True, + admission_wait_timeout_seconds=0.001, + ) + proxy_module.get_settings().http_responses_session_bridge_stuck_gate_retire_after_seconds = 0.01 + account_id = await _import_account( + async_client, + "acc-http-bridge-retired-gate-replace", + "http-bridge-retired-gate-replace@example.com", + ) + account = await _get_account(account_id) + service = get_proxy_service_for_app(app_instance) + replacement_upstream = _FakeBridgeUpstreamWebSocket() monkeypatch.setattr( - proxy_module.ProxyService, - "_create_http_bridge_session", - fake_create_http_bridge_session, + service, + "_select_account_with_budget", + AsyncMock(return_value=AccountSelection(account=account, error_message=None, error_code=None)), ) - monkeypatch.setattr(proxy_module.ProxyService, "_claim_durable_http_bridge_session", AsyncMock()) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) - async def get_session(request_scope_id: str) -> proxy_module._HTTPBridgeSession: - request_id_token = set_request_id("duplicate-client-request-id") - request_scope_token = set_request_scope_id(request_scope_id) - try: - return await service._get_or_create_http_bridge_session( - shared_key, - headers={"session_id": "shared-codex-process"}, - affinity=proxy_module._AffinityPolicy( - key="shared-codex-process", - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ), - api_key=None, - request_model="gpt-5.6-sol", - idle_ttl_seconds=120.0, - max_sessions=8, - ) - finally: - reset_request_scope_id(request_scope_token) - reset_request_id(request_id_token) + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + return replacement_upstream - first_task = asyncio.create_task(get_session("refresh-request-a")) - await _wait_for_event(refresh_started) - assert canonical.unanchored_reservation_id == "refresh-request-a" + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - second_task = asyncio.create_task(get_session("refresh-request-b")) - await asyncio.sleep(0) - assert not second_task.done() - allow_refresh.set() + request_headers = { + "session_id": "retired-gate-replace", + "x-codex-turn-state": "http_turn_retired_gate_replace", + "user-agent": "codex_cli_rs/0.145.0", + } + payload = proxy_module.ResponsesRequest( + model="gpt-5.6-sol", + instructions="Return exactly OK.", + input="continue after stale gate", + ) + affinity = proxy_module._sticky_key_for_responses_request( + payload, + request_headers, + codex_session_affinity=True, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + key = proxy_module._HTTPBridgeSessionKey("session_header", "retired-gate-replace", None) + stale_upstream = _SilentUpstreamWebSocket() + gate = asyncio.Semaphore(1) + await gate.acquire() + stale_request = proxy_module._WebSocketRequestState( + request_id="req-stale-gate-owner", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic() - 1.0, + transport="http", + response_create_gate=gate, + response_create_gate_acquired=True, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.6-sol"}', + ) + stale_session = proxy_module._HTTPBridgeSession( + key=key, + headers=request_headers, + affinity=affinity, + request_model="gpt-5.6-sol", + account=account, + upstream=cast(proxy_module.UpstreamWebSocket, stale_upstream), + upstream_control=proxy_module._WebSocketUpstreamControl(), + pending_requests=deque([stale_request]), + pending_lock=anyio.Lock(), + response_create_gate=gate, + queued_request_count=1, + last_used_at=time.monotonic(), + idle_ttl_seconds=120.0, + codex_session=True, + ) + stale_session.downstream_turn_state_aliases.add("http_turn_retired_gate_replace") + service._http_bridge_sessions[key] = stale_session + service._http_bridge_turn_state_index[ + proxy_module._http_bridge_turn_state_alias_key("http_turn_retired_gate_replace", None) + ] = key - first, second = await asyncio.gather(first_task, second_task) try: - assert first is canonical - assert second is not canonical - assert second.key.affinity_kind == "internal_unanchored_parallel" - assert second.unanchored_reservation_id == "refresh-request-b" + response = await asyncio.wait_for( + async_client.post( + "/backend-api/codex/responses", + json=payload.to_payload(), + headers=request_headers, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, + ) finally: - _release_http_bridge_unanchored_handoff(first, request_scope_id="refresh-request-a") - _release_http_bridge_unanchored_handoff(second, request_scope_id="refresh-request-b") - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() + if gate.locked(): + gate.release() + + assert response.status_code == 200 + assert stale_session.closed is True + assert stale_upstream.closed is True + assert len(replacement_upstream.sent_text) == 1 + assert any( + current_session is not stale_session and current_session.upstream is replacement_upstream + for current_session in service._http_bridge_sessions.values() + ) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_cancellation_during_durable_refresh_releases_reservation( +async def test_v1_responses_http_bridge_enforces_queue_limit_atomically_for_same_session( + async_client, app_instance, monkeypatch, ): + _install_bridge_settings_with_limits(monkeypatch, enabled=True, queue_limit=1) + account_id = await _import_account(async_client, "acc_http_bridge_queue", "http-bridge-queue@example.com") service = get_proxy_service_for_app(app_instance) - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() + account = await _get_account(account_id) + hanging_upstream = _SilentUpstreamWebSocket() - _install_proxy_settings( - monkeypatch, - app_settings=_make_app_settings( - enabled=True, - max_sessions=8, - admission_wait_timeout_seconds=1.0, - codex_idle_ttl_seconds=120.0, - instance_id="instance-a", - instance_ring=[], - ), - dashboard_settings=_make_dashboard_settings(), - ) + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + ) + return AccountSelection(account=account, error_message=None, error_code=None) - shared_key = proxy_module._HTTPBridgeSessionKey("session_header", "shared-codex-process", None) - canonical = _make_dummy_bridge_session(shared_key) - canonical.request_model = "gpt-5.6-sol" - canonical.durable_session_id = "durable-shared" - canonical.durable_owner_epoch = 1 - service._http_bridge_sessions[shared_key] = canonical - refresh_started = asyncio.Event() + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target - async def stuck_refresh(_session): - refresh_started.set() - await asyncio.Event().wait() + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + return hanging_upstream - monkeypatch.setattr(service, "_refresh_durable_http_bridge_session", stuck_refresh) - request_id_token = set_request_id("cancelled-client-request-id") - request_scope_token = set_request_scope_id("cancelled-request-scope") + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + payload = proxy_module.ResponsesRequest( + model="gpt-5.1", + instructions="Return exactly OK.", + input="same-session", + prompt_cache_key="same-session-key", + ) + affinity = proxy_module._sticky_key_for_responses_request( + payload, + {}, + codex_session_affinity=False, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + key = proxy_module._make_http_bridge_session_key( + payload, + headers={}, + affinity=affinity, + api_key=None, + request_id="req_queue", + ) + session = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=affinity, + api_key=None, + request_model="gpt-5.1", + idle_ttl_seconds=120.0, + max_sessions=128, + ) + + first_state, first_text = service._prepare_http_bridge_request(payload, {}, api_key=None, api_key_reservation=None) + first_state.transport = "http" + session.unanchored_reservation_id = "scope-submit" + request_scope_token = set_request_scope_id("scope-submit") try: - lookup_task = asyncio.create_task( - service._get_or_create_http_bridge_session( - shared_key, - headers={"session_id": "shared-codex-process"}, - affinity=proxy_module._AffinityPolicy( - key="shared-codex-process", - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ), - api_key=None, - request_model="gpt-5.6-sol", - idle_ttl_seconds=120.0, - max_sessions=8, - ) + await service._submit_http_bridge_request( + session, + request_state=first_state, + text_data=first_text, + queue_limit=1, ) - await _wait_for_event(refresh_started) - assert canonical.unanchored_reservation_id == "cancelled-request-scope" - lookup_task.cancel() - with pytest.raises(asyncio.CancelledError): - await lookup_task finally: reset_request_scope_id(request_scope_token) - reset_request_id(request_id_token) - assert getattr(canonical, "unanchored_reservation_id", None) is None + assert session.unanchored_reservation_id is None + + second_state, second_text = service._prepare_http_bridge_request( + payload, {}, api_key=None, api_key_reservation=None + ) + second_state.transport = "http" + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._submit_http_bridge_request( + session, + request_state=second_state, + text_data=second_text, + queue_limit=1, + ) + + exc = exc_info.value + assert exc.status_code == 429 + assert session.queued_request_count == 1 + await service._close_http_bridge_session(session) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_request_key_follower_isolates_different_model(app_instance, monkeypatch): +async def test_v1_responses_http_bridge_creates_different_session_keys_in_parallel(app_instance, monkeypatch): service = get_proxy_service_for_app(app_instance) service._http_bridge_sessions.clear() service._http_bridge_inflight_sessions.clear() @@ -14136,7 +10816,6 @@ async def test_v1_responses_http_bridge_request_key_follower_isolates_different_ app_settings=_make_app_settings( enabled=True, max_sessions=8, - admission_wait_timeout_seconds=1.0, codex_idle_ttl_seconds=120.0, instance_id="instance-a", instance_ring=[], @@ -14144,7 +10823,11 @@ async def test_v1_responses_http_bridge_request_key_follower_isolates_different_ dashboard_settings=_make_dashboard_settings(), ) - create_started = asyncio.Event() + create_started: list[str] = [] + create_started_events = { + "bridge-a": asyncio.Event(), + "bridge-b": asyncio.Event(), + } release_create = asyncio.Event() async def fake_create_http_bridge_session( @@ -14160,63 +10843,65 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, headers, affinity, + request_model, idle_ttl_seconds, request_stage, preferred_account_id, require_preferred_account, fallback_on_preferred_account_unavailable, ) - create_started.set() + create_started.append(key.affinity_key) + create_started_events[key.affinity_key].set() await _wait_for_event(release_create) - session = _make_dummy_bridge_session(key) - session.request_model = request_model - return session + return _make_dummy_bridge_session(key) monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - key = proxy_module._HTTPBridgeSessionKey("session_header", "shared-request", None) + key_one = proxy_module._HTTPBridgeSessionKey("request", "bridge-a", None) + key_two = proxy_module._HTTPBridgeSessionKey("request", "bridge-b", None) try: - creator = asyncio.create_task( + first = asyncio.create_task( service._get_or_create_http_bridge_session( - key, - headers={"x-codex-session-id": "shared-request"}, - affinity=proxy_module._AffinityPolicy( - key="shared-request", kind=proxy_module.StickySessionKind.CODEX_SESSION - ), + key_one, + headers={}, + affinity=proxy_module._AffinityPolicy(), api_key=None, - request_model="gpt-5.1", + request_model="gpt-5.4", idle_ttl_seconds=120.0, max_sessions=8, ) ) - await _wait_for_event(create_started) - follower = asyncio.create_task( + second = asyncio.create_task( service._get_or_create_http_bridge_session( - key, - headers={"x-codex-session-id": "shared-request"}, - affinity=proxy_module._AffinityPolicy( - key="shared-request", kind=proxy_module.StickySessionKind.CODEX_SESSION - ), + key_two, + headers={}, + affinity=proxy_module._AffinityPolicy(), api_key=None, request_model="gpt-5.4", idle_ttl_seconds=120.0, max_sessions=8, ) ) + await _wait_for_event(create_started_events["bridge-a"]) + await _wait_for_event(create_started_events["bridge-b"]) + assert key_one in service._http_bridge_inflight_sessions + assert key_two in service._http_bridge_inflight_sessions + release_create.set() - created_session, follower_session = await asyncio.gather(creator, follower) + session_one, session_two = await asyncio.gather(first, second) - assert created_session is not follower_session - assert created_session.request_model == "gpt-5.1" - assert follower_session.request_model == "gpt-5.4" - assert created_session.closed is False - assert follower_session.key.affinity_kind == "internal_unanchored_parallel" + assert sorted(create_started) == ["bridge-a", "bridge-b"] + assert session_one.key == key_one + assert session_two.key == key_two + assert service._http_bridge_sessions[key_one] is session_one + assert service._http_bridge_sessions[key_two] is session_two finally: service._http_bridge_sessions.clear() service._http_bridge_inflight_sessions.clear() @@ -14224,9 +10909,7 @@ async def fake_create_http_bridge_session( @pytest.mark.asyncio -async def test_v1_responses_http_bridge_forks_follower_when_account_assignment_changes_during_creation( - async_client, app_instance, monkeypatch -): +async def test_v1_responses_http_bridge_singleflights_same_session_key_during_creation(app_instance, monkeypatch): service = get_proxy_service_for_app(app_instance) service._http_bridge_sessions.clear() service._http_bridge_inflight_sessions.clear() @@ -14245,20 +10928,9 @@ async def test_v1_responses_http_bridge_forks_follower_when_account_assignment_c dashboard_settings=_make_dashboard_settings(), ) - create_started = asyncio.Event() + create_started: list[str] = [] + create_started_event = asyncio.Event() release_create = asyncio.Event() - create_calls: list[list[str]] = [] - durable_claims: list[tuple[str, bool]] = [] - stale_account_id = await _import_account( - async_client, - "acc_http_bridge_stale", - "http-bridge-stale@example.com", - ) - fresh_account_id = await _import_account( - async_client, - "acc_http_bridge_fresh", - "http-bridge-fresh@example.com", - ) async def fake_create_http_bridge_session( self, @@ -14273,6 +10945,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -14285,92 +10958,52 @@ async def fake_create_http_bridge_session( require_preferred_account, fallback_on_preferred_account_unavailable, ) - create_calls.append(list(api_key.assigned_account_ids if api_key is not None else [])) - if len(create_calls) == 1: - create_started.set() - await _wait_for_event(release_create) - session = _make_dummy_bridge_session(key) - cast(Any, session).account = SimpleNamespace(id=stale_account_id, status=AccountStatus.ACTIVE) - session.queued_request_count = 1 - session.upstream_control.retire_after_drain = True - return session - session = _make_dummy_bridge_session(key) - cast(Any, session).account = SimpleNamespace(id=fresh_account_id, status=AccountStatus.ACTIVE) - return session + create_started.append(key.affinity_key) + create_started_event.set() + await _wait_for_event(release_create) + return _make_dummy_bridge_session(key) monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - async def fake_claim_durable_http_bridge_session( - self, - session, - *, - allow_takeover, - force_owner_epoch_advance=False, - expected_takeover_owner_instance_id=None, - expected_takeover_owner_process_epoch=None, - ): - del ( - self, - allow_takeover, - expected_takeover_owner_instance_id, - expected_takeover_owner_process_epoch, - ) - durable_claims.append((session.account.id, force_owner_epoch_advance)) - session.durable_session_id = "durable-session" - session.durable_owner_epoch = 2 if force_owner_epoch_advance else 1 - - monkeypatch.setattr( - proxy_module.ProxyService, - "_claim_durable_http_bridge_session", - fake_claim_durable_http_bridge_session, - ) - - session_header = f"shared-session-{stale_account_id}" - key = proxy_module._HTTPBridgeSessionKey("session_header", session_header, "key-assignments") - stale_api_key = _make_api_key_data(key_id="key-assignments", assigned_account_ids=[stale_account_id]) - refreshed_api_key = _make_api_key_data(key_id="key-assignments", assigned_account_ids=[fresh_account_id]) + key = proxy_module._HTTPBridgeSessionKey("request", "bridge-singleflight", None) try: - creator = asyncio.create_task( + first = asyncio.create_task( service._get_or_create_http_bridge_session( key, - headers={"session_id": session_header}, - affinity=proxy_module._AffinityPolicy( - key=session_header, - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ), - api_key=stale_api_key, - request_model="gpt-5.1", + headers={}, + affinity=proxy_module._AffinityPolicy(), + api_key=None, + request_model="gpt-5.4", idle_ttl_seconds=120.0, max_sessions=8, ) ) - await _wait_for_event(create_started) - follower = asyncio.create_task( + await _wait_for_event(create_started_event) + assert create_started == ["bridge-singleflight"] + assert key in service._http_bridge_inflight_sessions + + second = asyncio.create_task( service._get_or_create_http_bridge_session( key, - headers={"session_id": session_header}, - affinity=proxy_module._AffinityPolicy( - key=session_header, - kind=proxy_module.StickySessionKind.CODEX_SESSION, - ), - api_key=refreshed_api_key, + headers={}, + affinity=proxy_module._AffinityPolicy(), + api_key=None, request_model="gpt-5.4", idle_ttl_seconds=120.0, max_sessions=8, ) ) + await asyncio.sleep(0) + assert create_started == ["bridge-singleflight"] + assert not second.done() + release_create.set() - created_session, follower_session = await asyncio.gather(creator, follower) + session_one, session_two = await asyncio.gather(first, second) - assert created_session is not follower_session - assert created_session.account.id == stale_account_id - assert follower_session.account.id == fresh_account_id - assert service._http_bridge_sessions[key] is created_session - assert follower_session.key.affinity_kind == "internal_unanchored_parallel" - assert service._http_bridge_sessions[follower_session.key] is follower_session - assert create_calls == [[stale_account_id], [fresh_account_id]] - assert durable_claims == [(stale_account_id, False), (fresh_account_id, False)] + assert create_started == ["bridge-singleflight"] + assert session_one is session_two + assert service._http_bridge_sessions[key] is session_one finally: service._http_bridge_sessions.clear() service._http_bridge_inflight_sessions.clear() @@ -14378,7 +11011,10 @@ async def fake_claim_durable_http_bridge_session( @pytest.mark.asyncio -async def test_v1_responses_http_bridge_singleflights_stale_session_replacement(app_instance, monkeypatch): +async def test_v1_responses_http_bridge_inflight_waiter_rejects_service_tier_provenance_mismatch( + app_instance, + monkeypatch, +): service = get_proxy_service_for_app(app_instance) service._http_bridge_sessions.clear() service._http_bridge_inflight_sessions.clear() @@ -14397,7 +11033,33 @@ async def test_v1_responses_http_bridge_singleflights_stale_session_replacement( dashboard_settings=_make_dashboard_settings(), ) - create_started: list[str] = [] + account_id = None + + class Registry: + def account_ids_for_model(self, model: str) -> set[str]: + assert model == "gpt-5.3-codex-spark" + return set() + + def plan_types_for_model(self, model: str) -> set[str]: + assert model == "gpt-5.3-codex-spark" + return {"pro"} + + def account_ids_for_model_service_tier(self, model: str, service_tier: str) -> set[str]: + assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") + return set() + + def plan_types_for_model_service_tier(self, model: str, service_tier: str) -> set[str]: + assert (model, service_tier) == ("gpt-5.3-codex-spark", "priority") + return {"pro"} + + def get_snapshot(self): + return SimpleNamespace(account_plans={account_id: "pro"}) + + monkeypatch.setattr(proxy_support, "get_model_registry", lambda: Registry()) + + create_service_tiers: list[str | None] = [] + create_started_event = asyncio.Event() + release_first_create = asyncio.Event() async def fake_create_http_bridge_session( self, @@ -14407,33 +11069,46 @@ async def fake_create_http_bridge_session( affinity, api_key, request_model, + request_service_tier=None, idle_ttl_seconds, request_stage="first_turn", preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, headers, affinity, - request_model, + api_key, idle_ttl_seconds, request_stage, preferred_account_id, require_preferred_account, fallback_on_preferred_account_unavailable, ) - create_started.append(key.affinity_key) - await asyncio.sleep(0.2) - return _make_dummy_bridge_session(key) + create_service_tiers.append(request_service_tier) + if len(create_service_tiers) == 1: + create_started_event.set() + await _wait_for_event(release_first_create) + session = _make_dummy_bridge_session(key) + session.account = cast( + Account, + SimpleNamespace(id=account_id, status=AccountStatus.ACTIVE, plan_type="pro"), + ) + session.request_model = request_model + session.request_service_tier = request_service_tier + session.catalog_omission_quota_admission = CatalogOmissionQuotaAdmission( + normalized_model=request_model, + canonical_quota_key="codex_spark", + normalized_effective_service_tier=request_service_tier, + ) + return session monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - key = proxy_module._HTTPBridgeSessionKey("request", "bridge-stale-replace", None) - stale_session = _make_dummy_bridge_session(key) - stale_session.closed = True - service._http_bridge_sessions[key] = stale_session + key = proxy_module._HTTPBridgeSessionKey("request", "bridge-inflight-tier", None) try: first = asyncio.create_task( @@ -14442,35 +11117,51 @@ async def fake_create_http_bridge_session( headers={}, affinity=proxy_module._AffinityPolicy(), api_key=None, - request_model="gpt-5.4", + request_model="gpt-5.3-codex-spark", + request_service_tier=None, idle_ttl_seconds=120.0, max_sessions=8, ) ) + await _wait_for_event(create_started_event) + second = asyncio.create_task( service._get_or_create_http_bridge_session( key, headers={}, affinity=proxy_module._AffinityPolicy(), api_key=None, - request_model="gpt-5.4", + request_model="gpt-5.3-codex-spark", + request_service_tier="priority", idle_ttl_seconds=120.0, max_sessions=8, ) ) - session_one, session_two = await asyncio.gather(first, second) + await asyncio.sleep(0) + assert not second.done() - assert create_started == ["bridge-stale-replace"] - assert session_one is session_two - assert service._http_bridge_sessions[key] is session_one + release_first_create.set() + first_session, second_session = await asyncio.gather(first, second) + + assert create_service_tiers == [None, "priority"] + assert first_session is not second_session + assert first_session.request_service_tier is None + assert second_session.request_service_tier == "priority" + assert first_session.closed is False + assert service._http_bridge_sessions[key] is first_session + assert second_session.key.affinity_kind == "internal_request_parallel" + assert service._http_bridge_sessions[second_session.key] is second_session finally: + release_first_create.set() service._http_bridge_sessions.clear() service._http_bridge_inflight_sessions.clear() service._http_bridge_turn_state_index.clear() @pytest.mark.asyncio -async def test_v1_responses_http_bridge_cleans_up_cancelled_singleflight_creator(app_instance, monkeypatch): +async def test_v1_responses_http_bridge_waits_for_inflight_capacity_before_rate_limiting_other_keys( + app_instance, monkeypatch +): service = get_proxy_service_for_app(app_instance) service._http_bridge_sessions.clear() service._http_bridge_inflight_sessions.clear() @@ -14480,7 +11171,7 @@ async def test_v1_responses_http_bridge_cleans_up_cancelled_singleflight_creator monkeypatch, app_settings=_make_app_settings( enabled=True, - max_sessions=8, + max_sessions=1, codex_idle_ttl_seconds=120.0, instance_id="instance-a", instance_ring=[], @@ -14489,7 +11180,8 @@ async def test_v1_responses_http_bridge_cleans_up_cancelled_singleflight_creator ) first_create_started = asyncio.Event() - create_attempts = 0 + release_first_create = asyncio.Event() + create_attempts: list[str] = [] async def fake_create_http_bridge_session( self, @@ -14504,6 +11196,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -14516,54 +11209,61 @@ async def fake_create_http_bridge_session( require_preferred_account, fallback_on_preferred_account_unavailable, ) - nonlocal create_attempts - create_attempts += 1 - if create_attempts == 1: + create_attempts.append(key.affinity_key) + if key.affinity_key == "bridge-capacity-a": first_create_started.set() - await _wait_for_event(asyncio.Event()) + await _wait_for_event(release_first_create) + raise RuntimeError("first create failed") return _make_dummy_bridge_session(key) monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - key = proxy_module._HTTPBridgeSessionKey("request", "bridge-cancelled-create", None) + key_one = proxy_module._HTTPBridgeSessionKey("request", "bridge-capacity-a", None) + key_two = proxy_module._HTTPBridgeSessionKey("request", "bridge-capacity-b", None) - creator = asyncio.create_task( + first = asyncio.create_task( service._get_or_create_http_bridge_session( - key, + key_one, headers={}, affinity=proxy_module._AffinityPolicy(), api_key=None, request_model="gpt-5.4", idle_ttl_seconds=120.0, - max_sessions=8, + max_sessions=1, ) ) await _wait_for_event(first_create_started) - creator.cancel() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(creator, timeout=_TEST_SYNC_TIMEOUT_SECONDS) - replacement = await asyncio.wait_for( + second = asyncio.create_task( service._get_or_create_http_bridge_session( - key, + key_two, headers={}, affinity=proxy_module._AffinityPolicy(), api_key=None, request_model="gpt-5.4", idle_ttl_seconds=120.0, - max_sessions=8, - ), - timeout=1.0, + max_sessions=1, + ) ) + await asyncio.sleep(0.01) + assert not second.done() - assert create_attempts == 2 - assert service._http_bridge_sessions[key] is replacement - assert key not in service._http_bridge_inflight_sessions + release_first_create.set() + + with pytest.raises(RuntimeError, match="first create failed"): + await first + created_session = await asyncio.wait_for(second, timeout=1.0) + + assert create_attempts == ["bridge-capacity-a", "bridge-capacity-b"] + assert service._http_bridge_sessions[key_two] is created_session + assert key_one not in service._http_bridge_inflight_sessions + assert key_two not in service._http_bridge_inflight_sessions @pytest.mark.asyncio -async def test_v1_responses_http_bridge_cleans_up_cancelled_singleflight_creator_after_create( - app_instance, monkeypatch +async def test_v1_responses_http_bridge_forks_parallel_unanchored_session_requests( + app_instance, + monkeypatch, ): service = get_proxy_service_for_app(app_instance) service._http_bridge_sessions.clear() @@ -14575,6 +11275,7 @@ async def test_v1_responses_http_bridge_cleans_up_cancelled_singleflight_creator app_settings=_make_app_settings( enabled=True, max_sessions=8, + admission_wait_timeout_seconds=1.0, codex_idle_ttl_seconds=120.0, instance_id="instance-a", instance_ring=[], @@ -14582,9 +11283,7 @@ async def test_v1_responses_http_bridge_cleans_up_cancelled_singleflight_creator dashboard_settings=_make_dashboard_settings(), ) - create_finished = asyncio.Event() - allow_return = asyncio.Event() - create_attempts = 0 + created_keys: list[proxy_module._HTTPBridgeSessionKey] = [] async def fake_create_http_bridge_session( self, @@ -14599,68 +11298,164 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, headers, affinity, - request_model, + api_key, idle_ttl_seconds, request_stage, preferred_account_id, require_preferred_account, fallback_on_preferred_account_unavailable, ) - nonlocal create_attempts - create_attempts += 1 - if create_attempts == 1: - create_finished.set() - await _wait_for_event(allow_return) - return _make_dummy_bridge_session(key) + created_keys.append(key) + session = _make_dummy_bridge_session(key) + session.request_model = request_model + return session + + async def fake_claim_durable_http_bridge_session( + self, + session, + *, + allow_takeover, + force_owner_epoch_advance=False, + record_restart_takeover=False, + ): + del self, session, allow_takeover, force_owner_epoch_advance monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) + monkeypatch.setattr( + proxy_module.ProxyService, + "_claim_durable_http_bridge_session", + fake_claim_durable_http_bridge_session, + ) - key = proxy_module._HTTPBridgeSessionKey("request", "bridge-cancelled-after-create", None) - creator = asyncio.create_task( - service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, + shared_key = proxy_module._HTTPBridgeSessionKey("session_header", "shared-codex-process", None) + foreground = _make_dummy_bridge_session(shared_key) + foreground.request_model = "gpt-5.6-sol" + foreground.queued_request_count = 1 + service._http_bridge_sessions[shared_key] = foreground + + async def get_memory_session(request_scope_id: str) -> proxy_module._HTTPBridgeSession: + request_id_token = set_request_id("duplicate-client-request-id") + request_scope_token = set_request_scope_id(request_scope_id) + try: + return await service._get_or_create_http_bridge_session( + shared_key, + headers={"session_id": "shared-codex-process"}, + affinity=proxy_module._AffinityPolicy( + key="shared-codex-process", + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ), + api_key=None, + request_model="gpt-5.4-mini", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + finally: + reset_request_scope_id(request_scope_token) + reset_request_id(request_id_token) + + try: + first_memory, second_memory = await asyncio.gather( + get_memory_session("memory-request-a"), + get_memory_session("memory-request-b"), ) - ) - await _wait_for_event(create_finished) - async with service._http_bridge_lock: - allow_return.set() - await asyncio.sleep(0) - creator.cancel() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(creator, timeout=_TEST_SYNC_TIMEOUT_SECONDS) + assert first_memory is not foreground + assert second_memory is not foreground + assert first_memory is not second_memory + assert foreground.request_model == "gpt-5.6-sol" + assert {key.affinity_kind for key in created_keys} == {"internal_unanchored_parallel"} + assert len({key.affinity_key for key in created_keys}) == 2 + assert all(key.strength == "hard" for key in created_keys) + finally: + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() - replacement = await asyncio.wait_for( - service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_reserved_handoff_forks_before_submit( + app_instance, + monkeypatch, +): + service = get_proxy_service_for_app(app_instance) + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() + + _install_proxy_settings( + monkeypatch, + app_settings=_make_app_settings( + enabled=True, max_sessions=8, + admission_wait_timeout_seconds=1.0, + codex_idle_ttl_seconds=120.0, + instance_id="instance-a", + instance_ring=[], ), - timeout=1.0, + dashboard_settings=_make_dashboard_settings(), ) - assert create_attempts == 2 - assert service._http_bridge_sessions[key] is replacement - assert key not in service._http_bridge_inflight_sessions + shared_key = proxy_module._HTTPBridgeSessionKey("session_header", "shared-codex-process", None) + + async def fake_create_http_bridge_session(self, key, **kwargs): + del self + session = _make_dummy_bridge_session(key) + session.request_model = kwargs["request_model"] + return session + + monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) + monkeypatch.setattr(proxy_module.ProxyService, "_claim_durable_http_bridge_session", AsyncMock()) + + async def get_session(request_scope_id: str) -> proxy_module._HTTPBridgeSession: + request_id_token = set_request_id("duplicate-client-request-id") + request_scope_token = set_request_scope_id(request_scope_id) + try: + return await service._get_or_create_http_bridge_session( + shared_key, + headers={"session_id": "shared-codex-process"}, + affinity=proxy_module._AffinityPolicy( + key="shared-codex-process", + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ), + api_key=None, + request_model="gpt-5.6-sol", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + finally: + reset_request_scope_id(request_scope_token) + reset_request_id(request_id_token) + + first = await get_session("scope-before-submit-a") + _reserve_http_bridge_unanchored_handoff(first, request_scope_id="scope-before-submit-a") + first.last_used_at = time.monotonic() - 300.0 + first.idle_ttl_seconds = 1.0 + try: + second = await get_session("scope-before-submit-b") + finally: + _release_http_bridge_unanchored_handoff(first, request_scope_id="scope-before-submit-a") + + assert first.key == shared_key + assert service._http_bridge_sessions[shared_key] is first + assert first.closed is False + assert first.queued_request_count == 0 + assert first.unanchored_reservation_id is None + assert second is not first + assert second.key.affinity_kind == "internal_unanchored_parallel" + assert second.key.strength == "hard" @pytest.mark.asyncio -async def test_v1_responses_http_bridge_waits_for_inflight_session_before_continuity_error(app_instance, monkeypatch): +async def test_v1_responses_http_bridge_reused_unanchored_refresh_reserves_canonical_handoff( + app_instance, + monkeypatch, +): service = get_proxy_service_for_app(app_instance) service._http_bridge_sessions.clear() service._http_bridge_inflight_sessions.clear() @@ -14671,6 +11466,7 @@ async def test_v1_responses_http_bridge_waits_for_inflight_session_before_contin app_settings=_make_app_settings( enabled=True, max_sessions=8, + admission_wait_timeout_seconds=1.0, codex_idle_ttl_seconds=120.0, instance_id="instance-a", instance_ring=[], @@ -14678,87 +11474,82 @@ async def test_v1_responses_http_bridge_waits_for_inflight_session_before_contin dashboard_settings=_make_dashboard_settings(), ) - create_started = asyncio.Event() - release_create = asyncio.Event() - - async def fake_create_http_bridge_session( - self, - key, - *, - headers, - affinity, - api_key, - request_model, - idle_ttl_seconds, - request_stage="first_turn", - preferred_account_id=None, - require_preferred_account=False, - fallback_on_preferred_account_unavailable=True, - ): - del ( - self, - headers, - affinity, - request_model, - idle_ttl_seconds, - request_stage, - preferred_account_id, - require_preferred_account, - fallback_on_preferred_account_unavailable, - ) - create_started.set() - await _wait_for_event(release_create) - return _make_dummy_bridge_session(key) + shared_key = proxy_module._HTTPBridgeSessionKey("session_header", "shared-codex-process", None) + canonical = _make_dummy_bridge_session(shared_key) + canonical.request_model = "gpt-5.6-sol" + canonical.durable_session_id = "durable-shared" + canonical.durable_owner_epoch = 1 + service._http_bridge_sessions[shared_key] = canonical + refresh_started = asyncio.Event() + allow_refresh = asyncio.Event() - monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) + async def blocked_refresh(session): + assert session is canonical + refresh_started.set() + await allow_refresh.wait() - key = proxy_module._HTTPBridgeSessionKey("request", "bridge-waits-for-inflight", None) + async def fake_create_http_bridge_session(self, key, **kwargs): + del self + session = _make_dummy_bridge_session(key) + session.request_model = kwargs["request_model"] + return session - creator = asyncio.create_task( - service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - ) + monkeypatch.setattr(service, "_refresh_durable_http_bridge_session", blocked_refresh) + monkeypatch.setattr( + proxy_module.ProxyService, + "_create_http_bridge_session", + fake_create_http_bridge_session, ) - await _wait_for_event(create_started) + monkeypatch.setattr(proxy_module.ProxyService, "_claim_durable_http_bridge_session", AsyncMock()) + + async def get_session(request_scope_id: str) -> proxy_module._HTTPBridgeSession: + request_id_token = set_request_id("duplicate-client-request-id") + request_scope_token = set_request_scope_id(request_scope_id) + try: + return await service._get_or_create_http_bridge_session( + shared_key, + headers={"session_id": "shared-codex-process"}, + affinity=proxy_module._AffinityPolicy( + key="shared-codex-process", + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ), + api_key=None, + request_model="gpt-5.6-sol", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + finally: + reset_request_scope_id(request_scope_token) + reset_request_id(request_id_token) - follower = asyncio.create_task( - service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - previous_response_id="resp_inflight", - ) - ) - await asyncio.sleep(0.01) - assert follower.done() + first_task = asyncio.create_task(get_session("refresh-request-a")) + await _wait_for_event(refresh_started) + assert canonical.unanchored_reservation_id == "refresh-request-a" - release_create.set() - created_session = await creator - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await follower + second_task = asyncio.create_task(get_session("refresh-request-b")) + await asyncio.sleep(0) + assert not second_task.done() + allow_refresh.set() - assert service._http_bridge_sessions[key] is created_session - exc = exc_info.value - assert exc.status_code == 502 - assert exc.payload["error"] == { - "message": "Upstream websocket closed before response.completed", - "type": "server_error", - "code": "stream_incomplete", - } + first, second = await asyncio.gather(first_task, second_task) + try: + assert first is canonical + assert second is not canonical + assert second.key.affinity_kind == "internal_unanchored_parallel" + assert second.unanchored_reservation_id == "refresh-request-b" + finally: + _release_http_bridge_unanchored_handoff(first, request_scope_id="refresh-request-a") + _release_http_bridge_unanchored_handoff(second, request_scope_id="refresh-request-b") + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() @pytest.mark.asyncio -async def test_v1_responses_http_bridge_prunes_idle_session_before_reuse(app_instance, monkeypatch): +async def test_v1_responses_http_bridge_cancellation_during_durable_refresh_releases_reservation( + app_instance, + monkeypatch, +): service = get_proxy_service_for_app(app_instance) service._http_bridge_sessions.clear() service._http_bridge_inflight_sessions.clear() @@ -14769,6 +11560,7 @@ async def test_v1_responses_http_bridge_prunes_idle_session_before_reuse(app_ins app_settings=_make_app_settings( enabled=True, max_sessions=8, + admission_wait_timeout_seconds=1.0, codex_idle_ttl_seconds=120.0, instance_id="instance-a", instance_ring=[], @@ -14776,922 +11568,812 @@ async def test_v1_responses_http_bridge_prunes_idle_session_before_reuse(app_ins dashboard_settings=_make_dashboard_settings(), ) - create_started: list[str] = [] - - async def fake_create_http_bridge_session( - self, - key, - *, - headers, - affinity, - api_key, - request_model, - idle_ttl_seconds, - request_stage="first_turn", - preferred_account_id=None, - require_preferred_account=False, - fallback_on_preferred_account_unavailable=True, - ): - del ( - self, - headers, - affinity, - request_model, - idle_ttl_seconds, - request_stage, - preferred_account_id, - require_preferred_account, - fallback_on_preferred_account_unavailable, - ) - create_started.append(key.affinity_key) - return _make_dummy_bridge_session(key) - - monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) + shared_key = proxy_module._HTTPBridgeSessionKey("session_header", "shared-codex-process", None) + canonical = _make_dummy_bridge_session(shared_key) + canonical.request_model = "gpt-5.6-sol" + canonical.durable_session_id = "durable-shared" + canonical.durable_owner_epoch = 1 + service._http_bridge_sessions[shared_key] = canonical + refresh_started = asyncio.Event() - key = proxy_module._HTTPBridgeSessionKey("request", "bridge-idle-prune", None) - stale_session = _make_dummy_bridge_session(key) - stale_session.last_used_at = time.monotonic() - 300.0 - stale_session.idle_ttl_seconds = 120.0 - service._http_bridge_sessions[key] = stale_session + async def stuck_refresh(_session): + refresh_started.set() + await asyncio.Event().wait() + monkeypatch.setattr(service, "_refresh_durable_http_bridge_session", stuck_refresh) + request_id_token = set_request_id("cancelled-client-request-id") + request_scope_token = set_request_scope_id("cancelled-request-scope") try: - replacement = await service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_module._AffinityPolicy(), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - ) - - assert create_started == ["bridge-idle-prune"] - assert replacement is not stale_session - assert service._http_bridge_sessions[key] is replacement - finally: - service._http_bridge_sessions.clear() - service._http_bridge_inflight_sessions.clear() - service._http_bridge_turn_state_index.clear() - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("upstream_type", "prime_reused_session", "expected_event_types", "expected_failure_sequence"), - [ - ( - _CreatedThenCloseUpstreamWebSocket, - False, - ["response.created", "response.failed"], - 0, - ), - ( - _ReasoningThenAbruptCloseUpstreamWebSocket, - False, - [ - "response.created", - "response.output_item.added", - "response.reasoning_summary_part.added", - "response.reasoning_summary_text.delta", - "response.failed", - ], - 4, - ), - ( - _CompleteThenReasoningAbruptCloseUpstreamWebSocket, - True, - [ - "response.created", - "response.output_item.added", - "response.reasoning_summary_part.added", - "response.reasoning_summary_text.delta", - "response.failed", - ], - 4, - ), - ], - ids=["created-then-close", "reasoning-then-abrupt-close", "reused-reasoning-then-abrupt-close"], -) -async def test_v1_responses_http_bridge_stream_failure_remains_valid_sse( - async_client, - monkeypatch, - upstream_type, - prime_reused_session, - expected_event_types, - expected_failure_sequence, -): - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_sse_failure", - "http-bridge-sse-failure@example.com", - ) - account = await _get_account(account_id) - upstream = upstream_type() - - async def fake_select_account_with_budget( - self, - deadline, - *, - request_id, - kind, - request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, - preferred_account_id=None, - ): - del preferred_account_id - del ( - self, - deadline, - request_id, - kind, - request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, - ) - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - return upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - headers = {"x-codex-turn-state": "turn-sse-failure"} if prime_reused_session else {} - if prime_reused_session: - prime = await async_client.post( - "/v1/responses", - headers=headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "prime-sse-session", - "prompt_cache_key": "sse-failure-key", - }, + lookup_task = asyncio.create_task( + service._get_or_create_http_bridge_session( + shared_key, + headers={"session_id": "shared-codex-process"}, + affinity=proxy_module._AffinityPolicy( + key="shared-codex-process", + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ), + api_key=None, + request_model="gpt-5.6-sol", + idle_ttl_seconds=120.0, + max_sessions=8, + ) ) - assert prime.status_code == 200 - - async with async_client.stream( - "POST", - "/v1/responses", - headers=headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "trigger-sse-failure", - "prompt_cache_key": "sse-failure-key", - "stream": True, - }, - ) as response: - assert response.status_code == 200 - lines = [line async for line in response.aiter_lines() if line.startswith("data: ")] + await _wait_for_event(refresh_started) + assert canonical.unanchored_reservation_id == "cancelled-request-scope" + lookup_task.cancel() + with pytest.raises(asyncio.CancelledError): + await lookup_task + finally: + reset_request_scope_id(request_scope_token) + reset_request_id(request_id_token) - events = [json.loads(line[6:]) for line in lines if line[6:] != "[DONE]"] - assert [event["type"] for event in events] == expected_event_types - assert events[0]["response"]["id"] == events[-1]["response"]["id"] - assert events[-1]["sequence_number"] == expected_failure_sequence - assert events[-1]["response"]["error"]["code"] == "stream_incomplete" + assert getattr(canonical, "unanchored_reservation_id", None) is None @pytest.mark.asyncio -async def test_v1_responses_http_bridge_upstream_failure_attributes_api_key_in_request_log( - async_client, - app_instance, - monkeypatch, -): - """An authenticated bridge request that fails through the session failure - fan-out (upstream send failure) must persist its request-log error row - with the request's api_key_id — the fan-out has no session-level key.""" - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_key_attribution", - "http-bridge-key-attribution@example.com", +async def test_v1_responses_http_bridge_request_key_follower_isolates_different_model(app_instance, monkeypatch): + service = get_proxy_service_for_app(app_instance) + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() + + _install_proxy_settings( + monkeypatch, + app_settings=_make_app_settings( + enabled=True, + max_sessions=8, + admission_wait_timeout_seconds=1.0, + codex_idle_ttl_seconds=120.0, + instance_id="instance-a", + instance_ring=[], + ), + dashboard_settings=_make_dashboard_settings(), ) - account = await _get_account(account_id) - upstream = _FakeBridgeUpstreamWebSocket() - failing_upstream = _FailingSendThenCloseUpstreamWebSocket() - response = await async_client.put("/api/settings", json={"apiKeyAuthEnabled": True}) - assert response.status_code == 200 - response = await async_client.post("/api/api-keys/", json={"name": "bridge-key-attribution"}) - assert response.status_code == 200 - api_key_id = response.json()["id"] - api_key_token = response.json()["key"] + create_started = asyncio.Event() + release_create = asyncio.Event() - async def fake_select_account_with_budget( + async def fake_create_http_bridge_session( self, - deadline, + key, *, - request_id, - kind, + headers, + affinity, + api_key, + request_model, + idle_ttl_seconds, request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, preferred_account_id=None, + require_preferred_account=False, + fallback_on_preferred_account_unavailable=True, + **_kwargs, ): - del preferred_account_id del ( self, - deadline, - request_id, - kind, + headers, + affinity, + idle_ttl_seconds, request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, + preferred_account_id, + require_preferred_account, + fallback_on_preferred_account_unavailable, ) - return AccountSelection(account=account, error_message=None, error_code=None) + create_started.set() + await _wait_for_event(release_create) + session = _make_dummy_bridge_session(key) + session.request_model = request_model + return session - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target + monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - return upstream + key = proxy_module._HTTPBridgeSessionKey("session_header", "shared-request", None) - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + try: + creator = asyncio.create_task( + service._get_or_create_http_bridge_session( + key, + headers={"x-codex-session-id": "shared-request"}, + affinity=proxy_module._AffinityPolicy( + key="shared-request", kind=proxy_module.StickySessionKind.CODEX_SESSION + ), + api_key=None, + request_model="gpt-5.1", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + ) + await _wait_for_event(create_started) + follower = asyncio.create_task( + service._get_or_create_http_bridge_session( + key, + headers={"x-codex-session-id": "shared-request"}, + affinity=proxy_module._AffinityPolicy( + key="shared-request", kind=proxy_module.StickySessionKind.CODEX_SESSION + ), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + ) + release_create.set() + created_session, follower_session = await asyncio.gather(creator, follower) + + assert created_session is not follower_session + assert created_session.request_model == "gpt-5.1" + assert follower_session.request_model == "gpt-5.4" + assert created_session.closed is False + assert follower_session.key.affinity_kind == "internal_unanchored_parallel" + finally: + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() - auth_headers = {"Authorization": f"Bearer {api_key_token}"} - first = await async_client.post( - "/v1/responses", - headers=auth_headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "bridge-key-attribution", - }, - ) - assert first.status_code == 200 - first_body = first.json() +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_forks_follower_when_account_assignment_changes_during_creation( + async_client, app_instance, monkeypatch +): service = get_proxy_service_for_app(app_instance) - async with service._http_bridge_lock: - session = next(iter(service._http_bridge_sessions.values())) - await _replace_http_bridge_upstream_reader( - service, - session, - cast(proxy_module.UpstreamWebSocket, failing_upstream), - ) + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() - second = await async_client.post( - "/v1/responses", - headers=auth_headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello-again", - "prompt_cache_key": "bridge-key-attribution", - "previous_response_id": first_body["id"], - }, + _install_proxy_settings( + monkeypatch, + app_settings=_make_app_settings( + enabled=True, + max_sessions=8, + admission_wait_timeout_seconds=1.0, + codex_idle_ttl_seconds=120.0, + instance_id="instance-a", + instance_ring=[], + ), + dashboard_settings=_make_dashboard_settings(), ) - assert second.status_code == 502 - - # The failure fan-out schedules the log write from detached cleanup, which - # can register after a single drain call returns, so poll until it lands. - rows: list[RequestLog] = [] - deadline = time.monotonic() + _TEST_SYNC_TIMEOUT_SECONDS - while time.monotonic() < deadline: - assert await service.drain_persistence_tasks(timeout_seconds=10) - async with SessionLocal() as session: - rows = list((await session.execute(select(RequestLog).where(RequestLog.status == "error"))).scalars().all()) - if rows: - break - await asyncio.sleep(0.05) - assert len(rows) == 1 - assert rows[0].account_id == account_id - assert rows[0].api_key_id == api_key_id - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_surfaces_upstream_error_event_as_http_400(async_client, monkeypatch): - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( + create_started = asyncio.Event() + release_create = asyncio.Event() + create_calls: list[list[str]] = [] + durable_claims: list[tuple[str, bool]] = [] + stale_account_id = await _import_account( async_client, - "acc_http_bridge_error_norm", - "http-bridge-error-norm@example.com", + "acc_http_bridge_stale", + "http-bridge-stale@example.com", + ) + fresh_account_id = await _import_account( + async_client, + "acc_http_bridge_fresh", + "http-bridge-fresh@example.com", ) - account = await _get_account(account_id) - fake_upstream = _ErrorOnlyUpstreamWebSocket() - async def fake_select_account_with_budget( + async def fake_create_http_bridge_session( self, - deadline, + key, *, - request_id, - kind, + headers, + affinity, + api_key, + request_model, + idle_ttl_seconds, request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, preferred_account_id=None, + require_preferred_account=False, + fallback_on_preferred_account_unavailable=True, + **_kwargs, ): - del preferred_account_id del ( self, - deadline, - request_id, - kind, + headers, + affinity, + request_model, + idle_ttl_seconds, request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, + preferred_account_id, + require_preferred_account, + fallback_on_preferred_account_unavailable, ) - return AccountSelection(account=account, error_message=None, error_code=None) + create_calls.append(list(api_key.assigned_account_ids if api_key is not None else [])) + if len(create_calls) == 1: + create_started.set() + await _wait_for_event(release_create) + session = _make_dummy_bridge_session(key) + cast(Any, session).account = SimpleNamespace(id=stale_account_id, status=AccountStatus.ACTIVE) + session.queued_request_count = 1 + session.upstream_control.retire_after_drain = True + return session + session = _make_dummy_bridge_session(key) + cast(Any, session).account = SimpleNamespace(id=fresh_account_id, status=AccountStatus.ACTIVE) + return session - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target + monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, + async def fake_claim_durable_http_bridge_session( + self, + session, *, - base_url=None, - session=None, + allow_takeover, + force_owner_epoch_advance=False, + record_restart_takeover=False, ): - del headers, access_token, account_id_header, base_url, session - return fake_upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + del self, allow_takeover + durable_claims.append((session.account.id, force_owner_epoch_advance)) + session.durable_session_id = "durable-session" + session.durable_owner_epoch = 2 if force_owner_epoch_advance else 1 - response = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.3-codex-spark", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-error-norm-key", - "stream": True, - }, + monkeypatch.setattr( + proxy_module.ProxyService, + "_claim_durable_http_bridge_session", + fake_claim_durable_http_bridge_session, ) - assert response.status_code == 400 - assert response.json() == { - "error": { - "message": "The 'gpt-5.3-codex-spark' model is not supported when using Codex with a ChatGPT account.", - "type": "invalid_request_error", - "code": "invalid_request_error", - } - } + session_header = f"shared-session-{stale_account_id}" + key = proxy_module._HTTPBridgeSessionKey("session_header", session_header, "key-assignments") + stale_api_key = _make_api_key_data(key_id="key-assignments", assigned_account_ids=[stale_account_id]) + refreshed_api_key = _make_api_key_data(key_id="key-assignments", assigned_account_ids=[fresh_account_id]) + + try: + creator = asyncio.create_task( + service._get_or_create_http_bridge_session( + key, + headers={"session_id": session_header}, + affinity=proxy_module._AffinityPolicy( + key=session_header, + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ), + api_key=stale_api_key, + request_model="gpt-5.1", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + ) + await _wait_for_event(create_started) + follower = asyncio.create_task( + service._get_or_create_http_bridge_session( + key, + headers={"session_id": session_header}, + affinity=proxy_module._AffinityPolicy( + key=session_header, + kind=proxy_module.StickySessionKind.CODEX_SESSION, + ), + api_key=refreshed_api_key, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + ) + release_create.set() + created_session, follower_session = await asyncio.gather(creator, follower) + + assert created_session is not follower_session + assert created_session.account.id == stale_account_id + assert follower_session.account.id == fresh_account_id + assert service._http_bridge_sessions[key] is created_session + assert follower_session.key.affinity_kind == "internal_unanchored_parallel" + assert service._http_bridge_sessions[follower_session.key] is follower_session + assert create_calls == [[stale_account_id], [fresh_account_id]] + assert durable_claims == [(stale_account_id, False), (fresh_account_id, False)] + finally: + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() @pytest.mark.asyncio -async def test_v1_responses_http_bridge_retries_stale_account_model_route_on_another_account( - async_client, - monkeypatch, -): - _install_bridge_settings(monkeypatch, enabled=True) - first_account_id = await _import_account( - async_client, - "acc_http_bridge_model_rejected", - "http-bridge-model-rejected@example.com", - ) - second_account_id = await _import_account( - async_client, - "acc_http_bridge_model_supported", - "http-bridge-model-supported@example.com", - ) - first_account = await _get_account(first_account_id) - second_account = await _get_account(second_account_id) - first_upstream = _ErrorOnlyUpstreamWebSocket() - second_upstream = _FakeBridgeUpstreamWebSocket() - connect_calls: list[str | None] = [] - selection_exclusions: list[set[str]] = [] - handle_stream_error = AsyncMock() +async def test_v1_responses_http_bridge_singleflights_stale_session_replacement(app_instance, monkeypatch): + service = get_proxy_service_for_app(app_instance) + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline - excluded = set(cast(set[str], kwargs.get("exclude_account_ids") or set())) - selection_exclusions.append(excluded) - account = second_account if first_account.id in excluded else first_account - return AccountSelection(account=account, error_message=None, error_code=None) + _install_proxy_settings( + monkeypatch, + app_settings=_make_app_settings( + enabled=True, + max_sessions=8, + admission_wait_timeout_seconds=1.0, + codex_idle_ttl_seconds=120.0, + instance_id="instance-a", + instance_ring=[], + ), + dashboard_settings=_make_dashboard_settings(), + ) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target + create_started: list[str] = [] - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, + async def fake_create_http_bridge_session( + self, + key, *, - base_url=None, - session=None, + headers, + affinity, + api_key, + request_model, + idle_ttl_seconds, + request_stage="first_turn", + preferred_account_id=None, + require_preferred_account=False, + fallback_on_preferred_account_unavailable=True, + **_kwargs, ): - del headers, access_token, base_url, session - connect_calls.append(account_id_header) - return first_upstream if len(connect_calls) == 1 else second_upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_handle_stream_error", handle_stream_error) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + del ( + self, + headers, + affinity, + request_model, + idle_ttl_seconds, + request_stage, + preferred_account_id, + require_preferred_account, + fallback_on_preferred_account_unavailable, + ) + create_started.append(key.affinity_key) + await asyncio.sleep(0.2) + return _make_dummy_bridge_session(key) - events = await _collect_sse_events( - async_client, - "/v1/responses", - json_body={ - "model": "gpt-5.3-codex-spark", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-account-model-retry-key", - "stream": True, - }, - ) + monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - _assert_created_text_delta_completed(events) - assert len(connect_calls) == 2 - assert selection_exclusions[0] == set() - assert first_account.id in selection_exclusions[-1] - assert first_upstream.closed is True - assert len(first_upstream.sent_text) == 1 - assert len(second_upstream.sent_text) == 1 - first_payload = json.loads(first_upstream.sent_text[0]) - second_payload = json.loads(second_upstream.sent_text[0]) - first_payload.get("client_metadata", {}).pop("x-codex-installation-id", None) - second_payload.get("client_metadata", {}).pop("x-codex-installation-id", None) - assert first_payload == second_payload - handle_stream_error.assert_not_awaited() + key = proxy_module._HTTPBridgeSessionKey("request", "bridge-stale-replace", None) + stale_session = _make_dummy_bridge_session(key) + stale_session.closed = True + service._http_bridge_sessions[key] = stale_session + + try: + first = asyncio.create_task( + service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_module._AffinityPolicy(), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + ) + second = asyncio.create_task( + service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_module._AffinityPolicy(), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + ) + session_one, session_two = await asyncio.gather(first, second) + + assert create_started == ["bridge-stale-replace"] + assert session_one is session_two + assert service._http_bridge_sessions[key] is session_one + finally: + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() @pytest.mark.asyncio -async def test_v1_responses_http_bridge_surfaces_selected_replacement_failure(async_client, monkeypatch): - _install_bridge_settings(monkeypatch, enabled=True) - first_account_id = await _import_account( - async_client, - "acc_http_bridge_selected_replacement_rejected", - "http-bridge-selected-replacement-rejected@example.com", - ) - second_account_id = await _import_account( - async_client, - "acc_http_bridge_selected_replacement_failed", - "http-bridge-selected-replacement-failed@example.com", - ) - first_account = await _get_account(first_account_id) - second_account = await _get_account(second_account_id) - first_upstream = _ErrorOnlyUpstreamWebSocket() - connect_calls: list[str | None] = [] +async def test_v1_responses_http_bridge_cleans_up_cancelled_singleflight_creator(app_instance, monkeypatch): + service = get_proxy_service_for_app(app_instance) + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline - excluded = set(cast(set[str], kwargs.get("exclude_account_ids") or set())) - account = second_account if first_account.id in excluded else first_account - return AccountSelection(account=account, error_message=None, error_code=None) + _install_proxy_settings( + monkeypatch, + app_settings=_make_app_settings( + enabled=True, + max_sessions=8, + codex_idle_ttl_seconds=120.0, + instance_id="instance-a", + instance_ring=[], + ), + dashboard_settings=_make_dashboard_settings(), + ) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target + first_create_started = asyncio.Event() + create_attempts = 0 - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, + async def fake_create_http_bridge_session( + self, + key, *, - base_url=None, - session=None, + headers, + affinity, + api_key, + request_model, + idle_ttl_seconds, + request_stage="first_turn", + preferred_account_id=None, + require_preferred_account=False, + fallback_on_preferred_account_unavailable=True, + **_kwargs, ): - del headers, access_token, base_url, session - connect_calls.append(account_id_header) - if len(connect_calls) == 1: - return first_upstream - raise proxy_module.ProxyResponseError( - 503, - proxy_module.openai_error( - "replacement_unavailable", - "Selected replacement connection failed", - error_type="server_error", - ), + del ( + self, + headers, + affinity, + request_model, + idle_ttl_seconds, + request_stage, + preferred_account_id, + require_preferred_account, + fallback_on_preferred_account_unavailable, ) + nonlocal create_attempts + create_attempts += 1 + if create_attempts == 1: + first_create_started.set() + await _wait_for_event(asyncio.Event()) + return _make_dummy_bridge_session(key) - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - response = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.3-codex-spark", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-selected-replacement-failure-key", - "stream": True, - }, + key = proxy_module._HTTPBridgeSessionKey("request", "bridge-cancelled-create", None) + + creator = asyncio.create_task( + service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_module._AffinityPolicy(), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) ) + await _wait_for_event(first_create_started) + creator.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(creator, timeout=_TEST_SYNC_TIMEOUT_SECONDS) - assert response.status_code == 503 - assert response.json() == { - "error": { - "message": "Selected replacement connection failed", - "type": "server_error", - "code": "replacement_unavailable", - } - } - assert len(connect_calls) == 2 + replacement = await asyncio.wait_for( + service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_module._AffinityPolicy(), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ), + timeout=1.0, + ) + + assert create_attempts == 2 + assert service._http_bridge_sessions[key] is replacement + assert key not in service._http_bridge_inflight_sessions @pytest.mark.asyncio -async def test_v1_responses_http_bridge_preserves_rate_limit_metadata_in_429(async_client, monkeypatch): - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_ratelimit", - "http-bridge-ratelimit@example.com", +async def test_v1_responses_http_bridge_cleans_up_cancelled_singleflight_creator_after_create( + app_instance, monkeypatch +): + service = get_proxy_service_for_app(app_instance) + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() + + _install_proxy_settings( + monkeypatch, + app_settings=_make_app_settings( + enabled=True, + max_sessions=8, + codex_idle_ttl_seconds=120.0, + instance_id="instance-a", + instance_ring=[], + ), + dashboard_settings=_make_dashboard_settings(), ) - account = await _get_account(account_id) - fake_upstream = _RateLimitErrorUpstreamWebSocket() - async def fake_select_account_with_budget( + create_finished = asyncio.Event() + allow_return = asyncio.Event() + create_attempts = 0 + + async def fake_create_http_bridge_session( self, - deadline, + key, *, - request_id, - kind, + headers, + affinity, + api_key, + request_model, + idle_ttl_seconds, request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, preferred_account_id=None, + require_preferred_account=False, + fallback_on_preferred_account_unavailable=True, + **_kwargs, ): - del preferred_account_id - return AccountSelection(account=account, error_message=None, error_code=None) + del ( + self, + headers, + affinity, + request_model, + idle_ttl_seconds, + request_stage, + preferred_account_id, + require_preferred_account, + fallback_on_preferred_account_unavailable, + ) + nonlocal create_attempts + create_attempts += 1 + if create_attempts == 1: + create_finished.set() + await _wait_for_event(allow_return) + return _make_dummy_bridge_session(key) - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - return target + monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - return fake_upstream + key = proxy_module._HTTPBridgeSessionKey("request", "bridge-cancelled-after-create", None) + creator = asyncio.create_task( + service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_module._AffinityPolicy(), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + ) + await _wait_for_event(create_finished) + async with service._http_bridge_lock: + allow_return.set() + await asyncio.sleep(0) + creator.cancel() - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(creator, timeout=_TEST_SYNC_TIMEOUT_SECONDS) - response = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-4o", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "http-bridge-ratelimit-key", - "stream": True, - }, + replacement = await asyncio.wait_for( + service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_module._AffinityPolicy(), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ), + timeout=1.0, ) - assert response.status_code == 429 - body = response.json() - assert body["error"]["code"] == "rate_limit_exceeded" - assert body["error"]["plan_type"] == "team" - assert body["error"]["resets_at"] == 1700000000 - assert body["error"]["resets_in_seconds"] == 3600 + assert create_attempts == 2 + assert service._http_bridge_sessions[key] is replacement + assert key not in service._http_bridge_inflight_sessions @pytest.mark.asyncio -async def test_v1_responses_http_bridge_cancellation_releases_queued_slot(async_client, app_instance, monkeypatch): - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account(async_client, "acc_http_bridge_cancel", "http-bridge-cancel@example.com") +async def test_v1_responses_http_bridge_waits_for_inflight_session_before_continuity_error(app_instance, monkeypatch): service = get_proxy_service_for_app(app_instance) - account = await _get_account(account_id) - upstream = _SilentUpstreamWebSocket() + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() - async def fake_select_account_with_budget( + _install_proxy_settings( + monkeypatch, + app_settings=_make_app_settings( + enabled=True, + max_sessions=8, + codex_idle_ttl_seconds=120.0, + instance_id="instance-a", + instance_ring=[], + ), + dashboard_settings=_make_dashboard_settings(), + ) + + create_started = asyncio.Event() + release_create = asyncio.Event() + + async def fake_create_http_bridge_session( self, - deadline, + key, *, - request_id, - kind, + headers, + affinity, + api_key, + request_model, + idle_ttl_seconds, request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, preferred_account_id=None, + require_preferred_account=False, + fallback_on_preferred_account_unavailable=True, + **_kwargs, ): - del preferred_account_id del ( self, - deadline, - request_id, - kind, + headers, + affinity, + request_model, + idle_ttl_seconds, request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, + preferred_account_id, + require_preferred_account, + fallback_on_preferred_account_unavailable, ) - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target + create_started.set() + await _wait_for_event(release_create) + return _make_dummy_bridge_session(key) - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - return upstream + monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + key = proxy_module._HTTPBridgeSessionKey("request", "bridge-waits-for-inflight", None) - payload = proxy_module.ResponsesRequest( - model="gpt-5.1", - instructions="Return exactly OK.", - input="cancel-me", - prompt_cache_key="cancel-key", - ) - affinity = proxy_module._sticky_key_for_responses_request( - payload, - {}, - codex_session_affinity=False, - openai_cache_affinity=True, - openai_cache_affinity_max_age_seconds=300, - sticky_threads_enabled=False, - api_key=None, - ) - key = proxy_module._make_http_bridge_session_key( - payload, - headers={}, - affinity=affinity, - api_key=None, - request_id="req_cancel", - ) - session = await service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=affinity, - api_key=None, - request_model="gpt-5.1", - idle_ttl_seconds=120.0, - max_sessions=128, + creator = asyncio.create_task( + service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_module._AffinityPolicy(), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) ) + await _wait_for_event(create_started) - await session.response_create_gate.acquire() - request_state, text_data = service._prepare_http_bridge_request(payload, {}, api_key=None, api_key_reservation=None) - request_state.transport = "http" - task = asyncio.create_task( - service._submit_http_bridge_request( - session, - request_state=request_state, - text_data=text_data, - queue_limit=8, + follower = asyncio.create_task( + service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_module._AffinityPolicy(), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + previous_response_id="resp_inflight", ) ) - await asyncio.sleep(0) - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task + await asyncio.sleep(0.01) + assert follower.done() - assert session.queued_request_count == 0 - async with session.pending_lock: - assert list(session.pending_requests) == [] - session.response_create_gate.release() - await service._close_http_bridge_session(session) + release_create.set() + created_session = await creator + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await follower + + assert service._http_bridge_sessions[key] is created_session + exc = exc_info.value + assert exc.status_code == 502 + assert exc.payload["error"] == { + "message": "Upstream websocket closed before response.completed", + "type": "server_error", + "code": "stream_incomplete", + } @pytest.mark.asyncio -async def test_v1_responses_http_bridge_ambiguous_send_failure_does_not_restart_reader( - async_client, - monkeypatch, -): - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_send_retry", - "http-bridge-send-retry@example.com", +async def test_v1_responses_http_bridge_prunes_idle_session_before_reuse(app_instance, monkeypatch): + service = get_proxy_service_for_app(app_instance) + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() + + _install_proxy_settings( + monkeypatch, + app_settings=_make_app_settings( + enabled=True, + max_sessions=8, + codex_idle_ttl_seconds=120.0, + instance_id="instance-a", + instance_ring=[], + ), + dashboard_settings=_make_dashboard_settings(), ) - account = await _get_account(account_id) - upstreams = [_FailingSendThenCloseUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] - connect_count = 0 - async def fake_select_account_with_budget( + create_started: list[str] = [] + + async def fake_create_http_bridge_session( self, - deadline, + key, *, - request_id, - kind, + headers, + affinity, + api_key, + request_model, + idle_ttl_seconds, request_stage="first_turn", - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids=None, - additional_limit_name=None, - api_key=None, preferred_account_id=None, + require_preferred_account=False, + fallback_on_preferred_account_unavailable=True, + **_kwargs, ): - del preferred_account_id del ( self, - deadline, - request_id, - kind, + headers, + affinity, + request_model, + idle_ttl_seconds, request_stage, - sticky_key, - sticky_kind, - reallocate_sticky, - sticky_max_age_seconds, - prefer_earlier_reset_accounts, - routing_strategy, - model, - exclude_account_ids, - additional_limit_name, + preferred_account_id, + require_preferred_account, + fallback_on_preferred_account_unavailable, ) - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target + create_started.append(key.affinity_key) + return _make_dummy_bridge_session(key) - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - del headers, access_token, account_id_header, base_url, session - nonlocal connect_count - upstream = upstreams[connect_count] - connect_count += 1 - if isinstance(upstream, _FakeBridgeUpstreamWebSocket) and not upstream._messages.qsize(): - await upstream._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.created", - "response": {"id": "resp_retry_send", "object": "response", "status": "in_progress"}, - }, - separators=(",", ":"), - ), - ) - ) - await upstream._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - { - "type": "response.completed", - "response": { - "id": "resp_retry_send", - "object": "response", - "status": "completed", - "usage": { - "input_tokens": 24, - "output_tokens": 2, - "total_tokens": 26, - "input_tokens_details": {"cached_tokens": 20}, - "output_tokens_details": {"reasoning_tokens": 0}, - }, - }, - }, - separators=(",", ":"), - ), - ) - ) - return upstream + monkeypatch.setattr(proxy_module.ProxyService, "_create_http_bridge_session", fake_create_http_bridge_session) - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + key = proxy_module._HTTPBridgeSessionKey("request", "bridge-idle-prune", None) + stale_session = _make_dummy_bridge_session(key) + stale_session.last_used_at = time.monotonic() - 300.0 + stale_session.idle_ttl_seconds = 120.0 + service._http_bridge_sessions[key] = stale_session - response = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "retry-send", - "prompt_cache_key": "retry-send-key", - }, - ) + try: + replacement = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_module._AffinityPolicy(), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) - assert response.status_code == 502 - assert response.json()["error"]["code"] == "stream_incomplete" - assert connect_count == 1 + assert create_started == ["bridge-idle-prune"] + assert replacement is not stale_session + assert service._http_bridge_sessions[key] is replacement + finally: + service._http_bridge_sessions.clear() + service._http_bridge_inflight_sessions.clear() + service._http_bridge_turn_state_index.clear() @pytest.mark.asyncio -async def test_v1_responses_http_bridge_ambiguous_receive_after_accept_is_not_replayed_and_next_request_is_fresh( +@pytest.mark.parametrize( + ("upstream_type", "prime_reused_session", "expected_event_types", "expected_failure_sequence"), + [ + ( + _CreatedThenCloseUpstreamWebSocket, + False, + ["response.created", "response.failed"], + 0, + ), + ( + _ReasoningThenAbruptCloseUpstreamWebSocket, + False, + [ + "response.created", + "response.output_item.added", + "response.reasoning_summary_part.added", + "response.reasoning_summary_text.delta", + "response.failed", + ], + 4, + ), + ( + _CompleteThenReasoningAbruptCloseUpstreamWebSocket, + True, + [ + "response.created", + "response.output_item.added", + "response.reasoning_summary_part.added", + "response.reasoning_summary_text.delta", + "response.failed", + ], + 4, + ), + ], + ids=["created-then-close", "reasoning-then-abrupt-close", "reused-reasoning-then-abrupt-close"], +) +async def test_v1_responses_http_bridge_stream_failure_remains_valid_sse( async_client, - app_instance, - caplog, monkeypatch, + upstream_type, + prime_reused_session, + expected_event_types, + expected_failure_sequence, ): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_ambiguous_receive", - "http-bridge-ambiguous-receive@example.com", + "acc_http_bridge_sse_failure", + "http-bridge-sse-failure@example.com", ) account = await _get_account(account_id) - ambiguous_upstream = _AmbiguousAcceptedReceiveErrorUpstreamWebSocket() - recovered_upstream = _FakeBridgeUpstreamWebSocket(response_id_prefix="resp_after_ambiguous_receive") - upstreams = [ambiguous_upstream, recovered_upstream] - connect_count = 0 + upstream = upstream_type() async def fake_select_account_with_budget( self, @@ -15711,8 +12393,8 @@ async def fake_select_account_with_budget( additional_limit_name=None, api_key=None, preferred_account_id=None, - **_kwargs, ): + del preferred_account_id del ( self, deadline, @@ -15728,8 +12410,6 @@ async def fake_select_account_with_budget( model, exclude_account_ids, additional_limit_name, - api_key, - preferred_account_id, ) return AccountSelection(account=account, error_message=None, error_code=None) @@ -15746,101 +12426,73 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - nonlocal connect_count - upstream = upstreams[connect_count] - connect_count += 1 return upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - service = get_proxy_service_for_app(app_instance) - retry_precreated = AsyncMock(wraps=service._retry_http_bridge_precreated_request) - fail_pending = AsyncMock(wraps=service._fail_pending_websocket_requests) - record_retry_circuit_failure = AsyncMock(wraps=service._record_http_bridge_retry_circuit_failure) - monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) - monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_retry_circuit_failure) - - first = await async_client.post( - "/v1/responses", - headers={"x-codex-session-id": "ambiguous-receive-hard-key"}, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "ambiguous-accept", - "prompt_cache_key": "ambiguous-receive-key", - }, - ) - - assert first.status_code == 502 - assert first.json()["error"]["code"] == "stream_incomplete" - assert len(ambiguous_upstream.sent_text) == 1 - assert ambiguous_upstream.irreversible_effect_count == 1 - assert connect_count == 1 - retry_precreated.assert_not_awaited() - fail_pending.assert_awaited_once() - fail_pending_call = fail_pending.await_args - assert fail_pending_call is not None - assert fail_pending_call.kwargs["penalize_account"] is True - record_retry_circuit_failure.assert_awaited_once() - circuit_call = record_retry_circuit_failure.await_args - assert circuit_call is not None - assert circuit_call.args[0].key.affinity_kind == "session_header" - assert circuit_call.kwargs == {"detail": "stream_incomplete"} - assert "admission_waiters=0" in caplog.text - assert "retry_action=suppressed_ambiguous_accept" in caplog.text - assert "circuit_action=record_stream_incomplete" in caplog.text - for _ in range(100): - async with service._http_bridge_lock: - retired = not service._http_bridge_sessions - if retired and ambiguous_upstream.closed: - break - await asyncio.sleep(0.01) - assert retired is True - assert ambiguous_upstream.closed is True + headers = {"x-codex-turn-state": "turn-sse-failure"} if prime_reused_session else {} + if prime_reused_session: + prime = await async_client.post( + "/v1/responses", + headers=headers, + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "prime-sse-session", + "prompt_cache_key": "sse-failure-key", + }, + ) + assert prime.status_code == 200 - second = await async_client.post( + async with async_client.stream( + "POST", "/v1/responses", - headers={"x-codex-session-id": "ambiguous-receive-hard-key"}, + headers=headers, json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": "fresh-after-ambiguous-accept", - "prompt_cache_key": "ambiguous-receive-key", + "input": "trigger-sse-failure", + "prompt_cache_key": "sse-failure-key", + "stream": True, }, - ) + ) as response: + assert response.status_code == 200 + lines = [line async for line in response.aiter_lines() if line.startswith("data: ")] - assert second.status_code == 200 - assert second.json()["output"][0]["content"][0]["text"] == "OK" - assert connect_count == 2 - assert len(recovered_upstream.sent_text) == 1 + events = [json.loads(line[6:]) for line in lines if line[6:] != "[DONE]"] + assert [event["type"] for event in events] == expected_event_types + assert events[0]["response"]["id"] == events[-1]["response"]["id"] + assert events[-1]["sequence_number"] == expected_failure_sequence + assert events[-1]["response"]["error"]["code"] == "stream_incomplete" @pytest.mark.asyncio -async def test_v1_responses_http_bridge_idle_recovery_hands_reader_to_replacement( +async def test_v1_responses_http_bridge_upstream_failure_attributes_api_key_in_request_log( async_client, app_instance, monkeypatch, ): - app_settings = _make_app_settings(enabled=True) - app_settings.sse_keepalive_interval_seconds = 0.01 - _install_proxy_settings( - monkeypatch, - app_settings=app_settings, - dashboard_settings=_make_dashboard_settings(), - ) - monkeypatch.setattr(proxy_module, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.01) - monkeypatch.setattr(proxy_module, "_STREAM_KEEPALIVE_MAX_COUNT", 1) + """An authenticated bridge request that fails through the session failure + fan-out (upstream send failure) must persist its request-log error row + with the request's api_key_id — the fan-out has no session-level key.""" + _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_reader_handoff", - "http-bridge-reader-handoff@example.com", + "acc_http_bridge_key_attribution", + "http-bridge-key-attribution@example.com", ) account = await _get_account(account_id) - upstreams = [_SilentUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] - connect_count = 0 + upstream = _FakeBridgeUpstreamWebSocket() + failing_upstream = _FailingSendThenCloseUpstreamWebSocket() + + response = await async_client.put("/api/settings", json={"apiKeyAuthEnabled": True}) + assert response.status_code == 200 + response = await async_client.post("/api/api-keys/", json={"name": "bridge-key-attribution"}) + assert response.status_code == 200 + api_key_id = response.json()["id"] + api_key_token = response.json()["key"] async def fake_select_account_with_budget( self, @@ -15860,8 +12512,8 @@ async def fake_select_account_with_budget( additional_limit_name=None, api_key=None, preferred_account_id=None, - **_kwargs, ): + del preferred_account_id del ( self, deadline, @@ -15877,8 +12529,6 @@ async def fake_select_account_with_budget( model, exclude_account_ids, additional_limit_name, - api_key, - preferred_account_id, ) return AccountSelection(account=account, error_message=None, error_code=None) @@ -15895,222 +12545,74 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - nonlocal connect_count - upstream = upstreams[connect_count] - connect_count += 1 return upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - service = get_proxy_service_for_app(app_instance) - record_retry_circuit_failure = AsyncMock(wraps=service._record_http_bridge_retry_circuit_failure) - monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_retry_circuit_failure) - response = await async_client.post( + auth_headers = {"Authorization": f"Bearer {api_key_token}"} + first = await async_client.post( "/v1/responses", + headers=auth_headers, json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": "recover-reader-handoff", - "prompt_cache_key": "reader-handoff-key", + "input": "hello", + "prompt_cache_key": "bridge-key-attribution", }, ) + assert first.status_code == 200 + first_body = first.json() - assert response.status_code == 200 - assert response.json()["output"][0]["content"][0]["text"] == "OK" - assert connect_count == 2 - assert upstreams[0].closed is True - record_retry_circuit_failure.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_retire_empty_terminal_session_does_not_double_count_retry_circuit(app_instance, monkeypatch): - service = get_proxy_service_for_app(app_instance) - session = proxy_module._HTTPBridgeSession( - key=proxy_module._HTTPBridgeSessionKey("session_header", "terminal-close-key", None), - headers={}, - affinity=proxy_module._AffinityPolicy( - key="terminal-close-key", - kind=proxy_module.StickySessionKind.CODEX_SESSION, - max_age_seconds=300, - ), - request_model="gpt-5.1", - account=cast(Account, SimpleNamespace(id="acct-terminal-close", status=AccountStatus.ACTIVE)), - upstream=cast(proxy_module.UpstreamWebSocket, _FakeBridgeUpstreamWebSocket()), - upstream_control=proxy_module._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=time.monotonic(), - idle_ttl_seconds=120.0, - ) - record_retry_circuit_failure = AsyncMock(wraps=service._record_http_bridge_retry_circuit_failure) - monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_retry_circuit_failure) - - await service._retire_stale_pending_http_bridge_session( - session, - detail="stream_incomplete", - response_events_seen=0, - ) - - record_retry_circuit_failure.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_retry_http_bridge_precreated_request_releases_pending_lock_before_reconnect(app_instance, monkeypatch): service = get_proxy_service_for_app(app_instance) - session = proxy_module._HTTPBridgeSession( - key=proxy_module._HTTPBridgeSessionKey("prompt_cache", "retry-lock-key", None), - headers={}, - affinity=proxy_module._AffinityPolicy( - key="retry-lock-key", - kind=proxy_module.StickySessionKind.PROMPT_CACHE, - max_age_seconds=300, - ), - request_model="gpt-5.1", - account=cast(Account, SimpleNamespace(id="acct-retry", status=AccountStatus.ACTIVE)), - upstream=cast(proxy_module.UpstreamWebSocket, _SilentUpstreamWebSocket()), - upstream_control=proxy_module._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=1, - last_used_at=time.monotonic(), - idle_ttl_seconds=120.0, - ) - request_state = proxy_module._WebSocketRequestState( - request_id="req-precreated-retry", - model="gpt-5.1", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, - transport="http", - response_create_gate_acquired=True, - request_text=json.dumps({"type": "response.create", "model": "gpt-5.1", "input": []}), - ) - session.pending_requests.append(request_state) - reconnect_started = asyncio.Event() - allow_reconnect_finish = asyncio.Event() - lock_reacquired = asyncio.Event() - replacement_upstream = _RecordingUpstreamWebSocket() - - async def fake_reconnect( - self, - target_session, - *, - request_state, - restart_reader=False, - require_same_account=False, - require_preferred_account=False, - ): - del self, request_state, restart_reader, require_same_account, require_preferred_account - reconnect_started.set() - await _wait_for_event(allow_reconnect_finish) - target_session.upstream = replacement_upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_reconnect_http_bridge_session", fake_reconnect) - - retry_task = asyncio.create_task(service._retry_http_bridge_precreated_request(session)) - await _wait_for_event(reconnect_started) - - async def acquire_pending_lock() -> None: - async with session.pending_lock: - lock_reacquired.set() - - lock_task = asyncio.create_task(acquire_pending_lock()) - await asyncio.wait_for(lock_reacquired.wait(), timeout=1.0) - allow_reconnect_finish.set() - - assert await retry_task is True - await lock_task - assert replacement_upstream.sent_text == [request_state.request_text] - + async with service._http_bridge_lock: + session = next(iter(service._http_bridge_sessions.values())) + await _replace_http_bridge_upstream_reader( + service, + session, + cast(proxy_module.UpstreamWebSocket, failing_upstream), + ) -@pytest.mark.asyncio -async def test_retry_http_bridge_precreated_request_ignores_existing_response_id_entries(app_instance, monkeypatch): - service = get_proxy_service_for_app(app_instance) - session = proxy_module._HTTPBridgeSession( - key=proxy_module._HTTPBridgeSessionKey("prompt_cache", "retry-race-key", None), - headers={}, - affinity=proxy_module._AffinityPolicy( - key="retry-race-key", - kind=proxy_module.StickySessionKind.PROMPT_CACHE, - max_age_seconds=300, - ), - request_model="gpt-5.1", - account=cast(Account, SimpleNamespace(id="acct-race", status=AccountStatus.ACTIVE)), - upstream=cast(proxy_module.UpstreamWebSocket, _SilentUpstreamWebSocket()), - upstream_control=proxy_module._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=2, - last_used_at=time.monotonic(), - idle_ttl_seconds=120.0, - ) - existing_request = proxy_module._WebSocketRequestState( - request_id="req-existing", - model="gpt-5.1", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - response_id="resp-existing", - awaiting_response_created=False, - transport="http", - ) - retry_request = proxy_module._WebSocketRequestState( - request_id="req-precreated-race", - model="gpt-5.1", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, - transport="http", - request_text=json.dumps({"type": "response.create", "model": "gpt-5.1", "input": ["retry"]}), + second = await async_client.post( + "/v1/responses", + headers=auth_headers, + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello-again", + "prompt_cache_key": "bridge-key-attribution", + "previous_response_id": first_body["id"], + }, ) - session.pending_requests.extend([existing_request, retry_request]) - replacement_upstream = _RecordingUpstreamWebSocket() - - async def fake_reconnect( - self, - target_session, - *, - request_state, - restart_reader=False, - require_same_account=False, - require_preferred_account=False, - ): - del self, request_state, restart_reader, require_same_account, require_preferred_account - target_session.upstream = replacement_upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_reconnect_http_bridge_session", fake_reconnect) + assert second.status_code == 502 - assert await service._retry_http_bridge_precreated_request(session) is True - assert replacement_upstream.sent_text == [retry_request.request_text] + # The failure fan-out schedules the log write from detached cleanup, which + # can register after a single drain call returns, so poll until it lands. + rows: list[RequestLog] = [] + deadline = time.monotonic() + _TEST_SYNC_TIMEOUT_SECONDS + while time.monotonic() < deadline: + assert await service.drain_persistence_tasks(timeout_seconds=10) + async with SessionLocal() as session: + rows = list((await session.execute(select(RequestLog).where(RequestLog.status == "error"))).scalars().all()) + if rows: + break + await asyncio.sleep(0.05) + assert len(rows) == 1 + assert rows[0].account_id == account_id + assert rows[0].api_key_id == api_key_id @pytest.mark.asyncio -async def test_v1_responses_http_bridge_send_failure_returns_upstream_unavailable( - async_client, - app_instance, - monkeypatch, -): +async def test_v1_responses_http_bridge_surfaces_upstream_error_event_as_http_400(async_client, monkeypatch): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_send_failure_previous_response", - "http-bridge-send-failure-previous-response@example.com", + "acc_http_bridge_error_norm", + "http-bridge-error-norm@example.com", ) account = await _get_account(account_id) - fake_upstream = _FakeBridgeUpstreamWebSocket() - failing_upstream = _FailingSendThenCloseUpstreamWebSocket() - connect_count = 0 + fake_upstream = _ErrorOnlyUpstreamWebSocket() async def fake_select_account_with_budget( self, @@ -16163,68 +12665,263 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - nonlocal connect_count - connect_count += 1 - return fake_upstream if connect_count == 1 else failing_upstream + return fake_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - first = await async_client.post( + response = await async_client.post( "/v1/responses", json={ - "model": "gpt-5.1", + "model": "gpt-5.3-codex-spark", "instructions": "Return exactly OK.", "input": "hello", - "prompt_cache_key": "send-failure-previous-response", + "prompt_cache_key": "http-bridge-error-norm-key", + "stream": True, }, ) - assert first.status_code == 200 - first_body = first.json() - service = get_proxy_service_for_app(app_instance) - async with service._http_bridge_lock: - session = next(iter(service._http_bridge_sessions.values())) - await _replace_http_bridge_upstream_reader( - service, - session, - cast(proxy_module.UpstreamWebSocket, failing_upstream), + assert response.status_code == 400 + assert response.json() == { + "error": { + "message": "The 'gpt-5.3-codex-spark' model is not supported when using Codex with a ChatGPT account.", + "type": "invalid_request_error", + "code": "invalid_request_error", + } + } + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_retries_stale_account_model_route_on_another_account( + async_client, + monkeypatch, +): + _install_bridge_settings(monkeypatch, enabled=True) + first_account_id = await _import_account( + async_client, + "acc_http_bridge_model_rejected", + "http-bridge-model-rejected@example.com", + ) + second_account_id = await _import_account( + async_client, + "acc_http_bridge_model_supported", + "http-bridge-model-supported@example.com", + ) + first_account = await _get_account(first_account_id) + second_account = await _get_account(second_account_id) + first_upstream = _ErrorOnlyUpstreamWebSocket() + second_upstream = _FakeBridgeUpstreamWebSocket() + connect_calls: list[str | None] = [] + selection_exclusions: list[set[str]] = [] + handle_stream_error = AsyncMock() + + async def fake_select_account_with_budget(self, deadline, **kwargs): + del self, deadline + excluded = set(cast(set[str], kwargs.get("exclude_account_ids") or set())) + selection_exclusions.append(excluded) + account = second_account if first_account.id in excluded else first_account + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, base_url, session + connect_calls.append(account_id_header) + return first_upstream if len(connect_calls) == 1 else second_upstream + + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_handle_stream_error", handle_stream_error) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + events = await _collect_sse_events( + async_client, + "/v1/responses", + json_body={ + "model": "gpt-5.3-codex-spark", + "instructions": "Return exactly OK.", + "input": "hello", + "prompt_cache_key": "http-bridge-account-model-retry-key", + "stream": True, + }, + ) + + _assert_created_text_delta_completed(events) + assert len(connect_calls) == 2 + assert selection_exclusions[0] == set() + assert first_account.id in selection_exclusions[-1] + assert first_upstream.closed is True + assert len(first_upstream.sent_text) == 1 + assert len(second_upstream.sent_text) == 1 + first_payload = json.loads(first_upstream.sent_text[0]) + second_payload = json.loads(second_upstream.sent_text[0]) + first_payload.get("client_metadata", {}).pop("x-codex-installation-id", None) + second_payload.get("client_metadata", {}).pop("x-codex-installation-id", None) + assert first_payload == second_payload + handle_stream_error.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_surfaces_selected_replacement_failure(async_client, monkeypatch): + _install_bridge_settings(monkeypatch, enabled=True) + first_account_id = await _import_account( + async_client, + "acc_http_bridge_selected_replacement_rejected", + "http-bridge-selected-replacement-rejected@example.com", + ) + second_account_id = await _import_account( + async_client, + "acc_http_bridge_selected_replacement_failed", + "http-bridge-selected-replacement-failed@example.com", + ) + first_account = await _get_account(first_account_id) + second_account = await _get_account(second_account_id) + first_upstream = _ErrorOnlyUpstreamWebSocket() + connect_calls: list[str | None] = [] + + async def fake_select_account_with_budget(self, deadline, **kwargs): + del self, deadline + excluded = set(cast(set[str], kwargs.get("exclude_account_ids") or set())) + account = second_account if first_account.id in excluded else first_account + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, base_url, session + connect_calls.append(account_id_header) + if len(connect_calls) == 1: + return first_upstream + raise proxy_module.ProxyResponseError( + 503, + proxy_module.openai_error( + "replacement_unavailable", + "Selected replacement connection failed", + error_type="server_error", + ), ) - second = await async_client.post( + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + response = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.3-codex-spark", + "instructions": "Return exactly OK.", + "input": "hello", + "prompt_cache_key": "http-bridge-selected-replacement-failure-key", + "stream": True, + }, + ) + + assert response.status_code == 503 + assert response.json() == { + "error": { + "message": "Selected replacement connection failed", + "type": "server_error", + "code": "replacement_unavailable", + } + } + assert len(connect_calls) == 2 + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_preserves_rate_limit_metadata_in_429(async_client, monkeypatch): + _install_bridge_settings(monkeypatch, enabled=True) + account_id = await _import_account( + async_client, + "acc_http_bridge_ratelimit", + "http-bridge-ratelimit@example.com", + ) + account = await _get_account(account_id) + fake_upstream = _RateLimitErrorUpstreamWebSocket() + + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + return fake_upstream + + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + response = await async_client.post( "/v1/responses", json={ - "model": "gpt-5.1", + "model": "gpt-4o", "instructions": "Return exactly OK.", - "input": "hello-again", - "prompt_cache_key": "send-failure-previous-response", - "previous_response_id": first_body["id"], + "input": "hello", + "prompt_cache_key": "http-bridge-ratelimit-key", + "stream": True, }, ) - assert second.status_code == 502 - assert second.json()["error"]["code"] in ("upstream_unavailable", "stream_incomplete", "bridge_owner_unreachable") - assert "previous_response_not_found" not in second.json()["error"].get("code", "") - assert connect_count == 1 + assert response.status_code == 429 + body = response.json() + assert body["error"]["code"] == "rate_limit_exceeded" + assert body["error"]["plan_type"] == "team" + assert body["error"]["resets_at"] == 1700000000 + assert body["error"]["resets_in_seconds"] == 3600 @pytest.mark.asyncio -async def test_v1_responses_http_bridge_precreated_disconnect_returns_upstream_unavailable( - async_client, - app_instance, - monkeypatch, -): +async def test_v1_responses_http_bridge_cancellation_releases_queued_slot(async_client, app_instance, monkeypatch): _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_precreated_previous_response", - "http-bridge-precreated-previous-response@example.com", - ) + account_id = await _import_account(async_client, "acc_http_bridge_cancel", "http-bridge-cancel@example.com") + service = get_proxy_service_for_app(app_instance) account = await _get_account(account_id) - fake_upstream = _FakeBridgeUpstreamWebSocket() - precreated_close_upstream = _PrecreatedCloseUpstreamWebSocket() - connect_count = 0 + upstream = _SilentUpstreamWebSocket() async def fake_select_account_with_budget( self, @@ -16277,67 +12974,80 @@ async def fake_connect_responses_websocket( session=None, ): del headers, access_token, account_id_header, base_url, session - nonlocal connect_count - connect_count += 1 - return fake_upstream if connect_count == 1 else precreated_close_upstream + return upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - first = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "precreated-previous-response", - }, + payload = proxy_module.ResponsesRequest( + model="gpt-5.1", + instructions="Return exactly OK.", + input="cancel-me", + prompt_cache_key="cancel-key", + ) + affinity = proxy_module._sticky_key_for_responses_request( + payload, + {}, + codex_session_affinity=False, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + api_key=None, + ) + key = proxy_module._make_http_bridge_session_key( + payload, + headers={}, + affinity=affinity, + api_key=None, + request_id="req_cancel", + ) + session = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=affinity, + api_key=None, + request_model="gpt-5.1", + idle_ttl_seconds=120.0, + max_sessions=128, ) - assert first.status_code == 200 - first_body = first.json() - service = get_proxy_service_for_app(app_instance) - async with service._http_bridge_lock: - session = next(iter(service._http_bridge_sessions.values())) - await _replace_http_bridge_upstream_reader( - service, + await session.response_create_gate.acquire() + request_state, text_data = service._prepare_http_bridge_request(payload, {}, api_key=None, api_key_reservation=None) + request_state.transport = "http" + task = asyncio.create_task( + service._submit_http_bridge_request( session, - cast(proxy_module.UpstreamWebSocket, precreated_close_upstream), + request_state=request_state, + text_data=text_data, + queue_limit=8, ) - - second = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello-again", - "prompt_cache_key": "precreated-previous-response", - "previous_response_id": first_body["id"], - }, ) + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task - assert second.status_code == 502 - assert second.json()["error"]["code"] in ("upstream_unavailable", "stream_incomplete", "upstream_request_timeout") - assert "previous_response_not_found" not in second.json()["error"].get("code", "") - assert connect_count == 1 + assert session.queued_request_count == 0 + async with session.pending_lock: + assert list(session.pending_requests) == [] + session.response_create_gate.release() + await service._close_http_bridge_session(session) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_rebinds_after_upstream_previous_response_not_found( +async def test_v1_responses_http_bridge_ambiguous_send_failure_does_not_restart_reader( async_client, - app_instance, monkeypatch, ): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_previous_response_rebind", - "http-bridge-previous-response-rebind@example.com", + "acc_http_bridge_send_retry", + "http-bridge-send-retry@example.com", ) account = await _get_account(account_id) - first_upstream = _FakeBridgeUpstreamWebSocket() - recovered_upstream = _FakeBridgeUpstreamWebSocket() + upstreams = [_FailingSendThenCloseUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] connect_count = 0 async def fake_select_account_with_budget( @@ -16375,7 +13085,6 @@ async def fake_select_account_with_budget( model, exclude_account_ids, additional_limit_name, - api_key, ) return AccountSelection(account=account, error_message=None, error_code=None) @@ -16393,100 +13102,127 @@ async def fake_connect_responses_websocket( ): del headers, access_token, account_id_header, base_url, session nonlocal connect_count + upstream = upstreams[connect_count] connect_count += 1 - if connect_count == 1: - return first_upstream - return recovered_upstream + if isinstance(upstream, _FakeBridgeUpstreamWebSocket) and not upstream._messages.qsize(): + await upstream._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": {"id": "resp_retry_send", "object": "response", "status": "in_progress"}, + }, + separators=(",", ":"), + ), + ) + ) + await upstream._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_retry_send", + "object": "response", + "status": "completed", + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) + ) + return upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - first = await async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "previous-response-rebind", - }, - ) - assert first.status_code == 200 - first_body = first.json() - - service = get_proxy_service_for_app(app_instance) - async with service._http_bridge_lock: - session = next(iter(service._http_bridge_sessions.values())) - await _replace_http_bridge_upstream_reader( - service, - session, - cast(proxy_module.UpstreamWebSocket, _PreviousResponseNotFoundUpstreamWebSocket()), - ) - - second = await async_client.post( + response = await async_client.post( "/v1/responses", json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": "hello-again", - "prompt_cache_key": "previous-response-rebind", - "previous_response_id": first_body["id"], + "input": "retry-send", + "prompt_cache_key": "retry-send-key", }, ) - assert second.status_code == 200 - assert second.json()["output"][0]["content"][0]["text"] == "OK" - assert connect_count == 2 + assert response.status_code == 502 + assert response.json()["error"]["code"] == "stream_incomplete" + assert connect_count == 1 -@pytest.mark.parametrize( - "retained_output_shape", - ["assistant_message", "agent_message"], -) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_recovers_store_context_trim_after_proxy_anchor_rejection( +async def test_v1_responses_http_bridge_idle_recovery_hands_reader_to_replacement( async_client, app_instance, monkeypatch, - retained_output_shape, ): - """An exact live-session trim can replay its preserved full request once.""" - _install_bridge_settings(monkeypatch, enabled=True) + app_settings = _make_app_settings(enabled=True) + app_settings.sse_keepalive_interval_seconds = 0.01 + _install_proxy_settings( + monkeypatch, + app_settings=app_settings, + dashboard_settings=_make_dashboard_settings(), + ) + monkeypatch.setattr(proxy_module, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.01) + monkeypatch.setattr(proxy_module, "_STREAM_KEEPALIVE_MAX_COUNT", 1) account_id = await _import_account( async_client, - "acc_http_bridge_store_context_recovery", - "http-bridge-store-context-recovery@example.com", + "acc_http_bridge_reader_handoff", + "http-bridge-reader-handoff@example.com", ) account = await _get_account(account_id) - agent_retained_output = [ - { - "type": "reasoning", - "id": "rs_previous", - "encrypted_content": "opaque", - "summary": [], - }, - { - "type": "agent_message", - "id": "amsg_01a02b33-3b30-7742-bdb3-091f07cf2ea0", - "author": "/root/episode_identity_final_audit", - "recipient": "/root", - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - "create_time": 1787431172.912141, - }, - "content": [{"type": "input_text", "text": "verified inter-agent result"}], - }, - ] - stale_upstream = _CompleteThenRejectStalePreviousResponseUpstreamWebSocket( - "resp_store_context", - completed_output=agent_retained_output if retained_output_shape == "agent_message" else None, - ) - recovered_upstream = _FakeBridgeUpstreamWebSocket("resp_store_context_recovered") - upstreams = [stale_upstream, recovered_upstream] + upstreams = [_SilentUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] connect_count = 0 - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline, kwargs + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + **_kwargs, + ): + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + api_key, + preferred_account_id, + ) return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): @@ -16501,8 +13237,8 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - nonlocal connect_count del headers, access_token, account_id_header, base_url, session + nonlocal connect_count upstream = upstreams[connect_count] connect_count += 1 return upstream @@ -16510,200 +13246,41 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - session_id = "store-context-stale-anchor" - session_headers = {"x-codex-session-id": session_id} - historical_input = [ - {"role": "user", "content": "hello"}, - # Long-running Codex sessions contain later per-turn developer - # controls inside the exact stored prefix. The same-session prefix - # fingerprint, not cross-account replay classification, owns them. - {"type": "message", "role": "developer", "content": "stored per-turn control"}, - ] - first = await async_client.post( - "/v1/responses", - headers=session_headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": historical_input, - "prompt_cache_key": session_id, - }, - ) - assert first.status_code == 200, first.text - first_body = first.json() - assert first_body["id"] == "resp_store_context_1" - - # Reproduce the production shape: the live bridge still owns the stored - # prefix, but durable full-resend evidence is unavailable for this exact - # request. The bridge itself must bind the complete request before it - # trims that prefix and injects the response anchor. service = get_proxy_service_for_app(app_instance) - async with service._http_bridge_lock: - live_session = next(iter(service._http_bridge_sessions.values())) - live_session.codex_session = True - monkeypatch.setattr( - service._durable_bridge, - "lookup_request_targets", - AsyncMock(return_value=None), - ) - if retained_output_shape == "assistant_message": - retained_output = first_body["output"] - follow_up_messages = [{"role": "user", "content": "continue from the complete context"}] - else: - # Exact production shape from the stranded long-running Codex task: - # response-owned reasoning, one completed inter-agent delivery, and - # two later user retries after the stale anchor had already failed. - retained_output = first_body["output"] - follow_up_messages = [ - { - "type": "message", - "id": "msg_01a02c31-2f60-7dd2-9f22-d7ef316596b1", - "role": "user", - "content": [{"type": "input_text", "text": "first retry after the completed inter-agent result"}], - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02c31-2f60-7dd2-9f22-d7ef316596b1", - "create_time": 1787433300.0, - }, - }, - { - "type": "message", - "id": "msg_01a02c43-4980-7afb-97f5-2e2d30aa73de", - "role": "user", - "content": [{"type": "input_text", "text": "second retry after the stale anchor error"}], - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02c43-4980-7afb-97f5-2e2d30aa73de", - "create_time": 1787433402.605, - }, - }, - ] - complete_follow_up = [*historical_input, *retained_output, *follow_up_messages] - assert live_session.last_response_transition_manifest is not None - proof_payload = proxy_module.ResponsesRequest.model_validate( - { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": complete_follow_up, - "prompt_cache_key": session_id, - } - ) - assert ( - http_bridge_streaming_module._verify_store_context_full_resend( - proof_payload, - live_session, - ) - is not None - ) - second = await async_client.post( + record_retry_circuit_failure = AsyncMock(wraps=service._record_http_bridge_retry_circuit_failure) + monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_retry_circuit_failure) + + response = await async_client.post( "/v1/responses", - headers=session_headers, json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": complete_follow_up, - "prompt_cache_key": session_id, + "input": "recover-reader-handoff", + "prompt_cache_key": "reader-handoff-key", }, ) - assert second.status_code == 200, second.text - assert second.json()["id"] == "resp_store_context_recovered_1" + assert response.status_code == 200 + assert response.json()["output"][0]["content"][0]["text"] == "OK" assert connect_count == 2 - assert len(stale_upstream.sent_text) == 2 - anchored_payload = json.loads(stale_upstream.sent_text[1]) - assert anchored_payload["previous_response_id"] == first_body["id"] - assert anchored_payload["input"] == complete_follow_up[len(historical_input) :] - assert len(recovered_upstream.sent_text) == 1 - recovered_payload = json.loads(recovered_upstream.sent_text[0]) - assert "previous_response_id" not in recovered_payload - if retained_output_shape == "assistant_message": - assert recovered_payload["input"] == [ - historical_input[0], - *first_body["output"], - complete_follow_up[-1], - ] - else: - recovered_agent_messages = [ - item - for item in recovered_payload["input"] - if item.get("type") == "message" and item.get("id", "").startswith("amsg_") - ] - assert recovered_agent_messages == [retained_output[-1]] - assert [item.get("content") for item in recovered_payload["input"] if item.get("role") == "user"] == [ - historical_input[0]["content"], - *[item["content"] for item in follow_up_messages], - ] - assert recovered_payload["instructions"].endswith("stored per-turn control") + assert upstreams[0].closed is True + record_retry_circuit_failure.assert_not_awaited() -@pytest.mark.parametrize( - "malformed_suffix", - [ - pytest.param( - [{"type": ["unhashable", "type"], "content": "not replay authority"}], - id="unhashable-type", - ), - pytest.param( - [{"type": "message", "role": {"unhashable": "role"}, "content": "not replay authority"}], - id="unhashable-role", - ), - pytest.param( - [ - { - "type": "agent_message", - "id": "amsg_01a02b33-3b30-7742-bdb3-091f07cf2ea0", - "author": "/root/episode_identity_final_audit", - "recipient": "/root", - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - "create_time": 10**400, - }, - "content": [{"type": "input_text", "text": "not replay authority"}], - }, - ], - id="oversized-agent-message-create-time", - ), - *[ - pytest.param( - [ - { - "type": "agent_message", - "id": "amsg_01a02b33-3b30-7742-bdb3-091f07cf2ea0", - "author": author, - "recipient": "/root", - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - "create_time": 1787431172.912141, - }, - "content": [{"type": "input_text", "text": "not replay authority"}], - }, - {"role": "user", "content": "fresh retry"}, - ], - id=test_id, - ) - for test_id, author in ( - ("agent-message-non-root-path", "/foo"), - ("agent-message-path-traversal", "/root/.."), - ("agent-message-uppercase-task", "/root/UPPER"), - ) - ], - ], -) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_malformed_full_resend_diagnostic_stays_fail_closed( +async def test_backend_responses_http_bridge_idle_retirement_does_not_open_retry_circuit_on_next_failure( async_client, app_instance, monkeypatch, - malformed_suffix, ): - """Malformed diagnostics must not turn an anchored rejection into HTTP 500.""" _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_malformed_full_resend", - "http-bridge-malformed-full-resend@example.com", + "acc_backend_idle_retirement_circuit", + "backend-idle-retirement-circuit@example.com", ) account = await _get_account(account_id) - upstream = _FakeBridgeUpstreamWebSocket("resp_malformed_full_resend") + upstream = _FakeBridgeUpstreamWebSocket("resp_idle_retirement_circuit") async def fake_select_account_with_budget(self, deadline, **kwargs): del self, deadline, kwargs @@ -16728,70 +13305,203 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - stored_input: list[proxy_module.JsonValue] = [{"role": "user", "content": "stored question"}] - durable_lookup = proxy_module.DurableBridgeLookup( - session_id="durable-malformed-full-resend", - canonical_kind="prompt_cache", - canonical_key="malformed-full-resend", - api_key_scope="__anonymous__", - account_id=account.id, - owner_instance_id=None, - owner_epoch=1, - lease_expires_at=None, - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state=None, - latest_response_id="resp_malformed_anchor", - latest_input_item_count=len(stored_input), - latest_input_full_fingerprint=proxy_module._fingerprint_input_items(stored_input), - latest_pending_tool_calls={}, - model="gpt-5.1", + session_id = "backend-idle-retirement-circuit-session" + prompt_cache_key = "backend-idle-retirement-circuit-thread" + headers = {"session_id": session_id} + bridge_key = _make_http_bridge_session_header_fallback_key( + headers=headers, + api_key=None, + explicit_prompt_cache_key=prompt_cache_key, ) + assert bridge_key is not None service = get_proxy_service_for_app(app_instance) - monkeypatch.setattr( - service._durable_bridge, - "lookup_request_targets", - AsyncMock(return_value=durable_lookup), + + # Reproduce the live ordering without waiting for production-scale + # watchdogs: an idle no-pending retirement, then one genuine pre-response + # request failure on the same hard key. Only the latter may be a strike. + idle_session = _make_dummy_bridge_session(bridge_key) + await service._retire_stale_pending_http_bridge_session( + idle_session, + detail="stream_incomplete", + response_events_seen=0, ) + failed_request_session = _make_dummy_bridge_session(bridge_key) + failures = await service._record_http_bridge_retry_circuit_failure( + failed_request_session, + detail="missing_response_created_timeout", + ) + assert failures == 1 + assert await service._http_bridge_precreated_retry_allowed(failed_request_session) is True - response = await async_client.post( - "/v1/responses", - json={ + events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": [*stored_input, *malformed_suffix], - "prompt_cache_key": "malformed-full-resend", + "input": "continue after one real timeout", + "prompt_cache_key": prompt_cache_key, + "stream": True, }, + headers=headers, ) - assert response.status_code == 200, response.text - assert len(upstream.sent_text) == 1 - forwarded = json.loads(upstream.sent_text[0]) - assert "previous_response_id" not in forwarded - assert forwarded["input"] == [*stored_input, *malformed_suffix] + _assert_created_text_delta_completed(events) + assert events[-1]["response"]["id"] == "resp_idle_retirement_circuit_1" + + +@pytest.mark.asyncio +async def test_retry_http_bridge_precreated_request_releases_pending_lock_before_reconnect(app_instance, monkeypatch): + service = get_proxy_service_for_app(app_instance) + session = proxy_module._HTTPBridgeSession( + key=proxy_module._HTTPBridgeSessionKey("prompt_cache", "retry-lock-key", None), + headers={}, + affinity=proxy_module._AffinityPolicy( + key="retry-lock-key", + kind=proxy_module.StickySessionKind.PROMPT_CACHE, + max_age_seconds=300, + ), + request_model="gpt-5.1", + account=cast(Account, SimpleNamespace(id="acct-retry", status=AccountStatus.ACTIVE)), + upstream=cast(proxy_module.UpstreamWebSocket, _SilentUpstreamWebSocket()), + upstream_control=proxy_module._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=time.monotonic(), + idle_ttl_seconds=120.0, + ) + request_state = proxy_module._WebSocketRequestState( + request_id="req-precreated-retry", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + transport="http", + response_create_gate_acquired=True, + request_text=json.dumps({"type": "response.create", "model": "gpt-5.1", "input": []}), + ) + session.pending_requests.append(request_state) + reconnect_started = asyncio.Event() + allow_reconnect_finish = asyncio.Event() + lock_reacquired = asyncio.Event() + replacement_upstream = _RecordingUpstreamWebSocket() + + async def fake_reconnect( + self, + target_session, + *, + request_state, + restart_reader=False, + require_same_account=False, + require_preferred_account=False, + ): + del self, request_state, restart_reader, require_same_account, require_preferred_account + reconnect_started.set() + await _wait_for_event(allow_reconnect_finish) + target_session.upstream = replacement_upstream + + monkeypatch.setattr(proxy_module.ProxyService, "_reconnect_http_bridge_session", fake_reconnect) + + retry_task = asyncio.create_task(service._retry_http_bridge_precreated_request(session)) + await _wait_for_event(reconnect_started) + + async def acquire_pending_lock() -> None: + async with session.pending_lock: + lock_reacquired.set() + + lock_task = asyncio.create_task(acquire_pending_lock()) + await asyncio.wait_for(lock_reacquired.wait(), timeout=1.0) + allow_reconnect_finish.set() + + assert await retry_task is True + await lock_task + assert replacement_upstream.sent_text == [request_state.request_text] + + +@pytest.mark.asyncio +async def test_retry_http_bridge_precreated_request_ignores_existing_response_id_entries(app_instance, monkeypatch): + service = get_proxy_service_for_app(app_instance) + session = proxy_module._HTTPBridgeSession( + key=proxy_module._HTTPBridgeSessionKey("prompt_cache", "retry-race-key", None), + headers={}, + affinity=proxy_module._AffinityPolicy( + key="retry-race-key", + kind=proxy_module.StickySessionKind.PROMPT_CACHE, + max_age_seconds=300, + ), + request_model="gpt-5.1", + account=cast(Account, SimpleNamespace(id="acct-race", status=AccountStatus.ACTIVE)), + upstream=cast(proxy_module.UpstreamWebSocket, _SilentUpstreamWebSocket()), + upstream_control=proxy_module._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=2, + last_used_at=time.monotonic(), + idle_ttl_seconds=120.0, + ) + existing_request = proxy_module._WebSocketRequestState( + request_id="req-existing", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + response_id="resp-existing", + awaiting_response_created=False, + transport="http", + ) + retry_request = proxy_module._WebSocketRequestState( + request_id="req-precreated-race", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + transport="http", + request_text=json.dumps({"type": "response.create", "model": "gpt-5.1", "input": ["retry"]}), + ) + session.pending_requests.extend([existing_request, retry_request]) + replacement_upstream = _RecordingUpstreamWebSocket() + + async def fake_reconnect( + self, + target_session, + *, + request_state, + restart_reader=False, + require_same_account=False, + require_preferred_account=False, + ): + del self, request_state, restart_reader, require_same_account, require_preferred_account + target_session.upstream = replacement_upstream + + monkeypatch.setattr(proxy_module.ProxyService, "_reconnect_http_bridge_session", fake_reconnect) + + assert await service._retry_http_bridge_precreated_request(session) is True + assert replacement_upstream.sent_text == [retry_request.request_text] -@pytest.mark.parametrize( - "canonical_invalid_shape", - [False, True], - ids=["previous-response-not-found", "canonical-invalid-previous-response-id"], -) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_quarantines_persistently_stale_proxy_anchor_then_recovers_full_resend( +async def test_v1_responses_http_bridge_send_failure_returns_upstream_unavailable( async_client, app_instance, monkeypatch, - canonical_invalid_shape, ): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_invalid_request_rebind", - "http-bridge-invalid-request-rebind@example.com", + "acc_http_bridge_send_failure_previous_response", + "http-bridge-send-failure-previous-response@example.com", ) account = await _get_account(account_id) - first_upstream = _FakeBridgeUpstreamWebSocket() - stale_upstream: _RejectStalePreviousResponseUpstreamWebSocket | None = None - recovered_upstream: _RejectStalePreviousResponseUpstreamWebSocket | None = None + fake_upstream = _FakeBridgeUpstreamWebSocket() + failing_upstream = _FailingSendThenCloseUpstreamWebSocket() connect_count = 0 async def fake_select_account_with_budget( @@ -16829,7 +13539,6 @@ async def fake_select_account_with_budget( model, exclude_account_ids, additional_limit_name, - api_key, ) return AccountSelection(account=account, error_message=None, error_code=None) @@ -16848,134 +13557,108 @@ async def fake_connect_responses_websocket( del headers, access_token, account_id_header, base_url, session nonlocal connect_count connect_count += 1 - if connect_count == 1: - return first_upstream - if connect_count == 2: - assert stale_upstream is not None - return stale_upstream - assert recovered_upstream is not None - return recovered_upstream + return fake_upstream if connect_count == 1 else failing_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - historical_input = [ - {"role": "user", "content": "hello"}, - { - "type": "message", - "role": "developer", - "content": "stored per-turn control", - }, - ] first = await async_client.post( "/v1/responses", - headers={"x-codex-session-id": "persistent-stale-anchor"}, json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": historical_input, - "prompt_cache_key": "invalid-request-rebind", + "input": "hello", + "prompt_cache_key": "send-failure-previous-response", }, ) assert first.status_code == 200 first_body = first.json() service = get_proxy_service_for_app(app_instance) - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket( - first_body["id"], - canonical_invalid_shape=canonical_invalid_shape, - ) - recovered_upstream = _RejectStalePreviousResponseUpstreamWebSocket( - first_body["id"], - canonical_invalid_shape=canonical_invalid_shape, - ) async with service._http_bridge_lock: session = next(iter(service._http_bridge_sessions.values())) - await service._reset_http_bridge_session_after_local_terminal_error( - session, - error_code="test_transport_replaced", - error_message="Force a fresh physical websocket while preserving durable continuity", - preserve_durable_lease=True, - ) + await _replace_http_bridge_upstream_reader( + service, + session, + cast(proxy_module.UpstreamWebSocket, failing_upstream), + ) - # This carries prior user input but omits the completed assistant item, so - # it is full-resend-shaped without satisfying the immutable completeness - # proof. The proxy must not silently discard the rejected anchor in-place. second = await async_client.post( "/v1/responses", - headers={"x-codex-session-id": "persistent-stale-anchor"}, json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": [ - *historical_input, - {"role": "user", "content": "hello-again"}, - ], - "prompt_cache_key": "invalid-request-rebind", + "input": "hello-again", + "prompt_cache_key": "send-failure-previous-response", + "previous_response_id": first_body["id"], }, ) - assert second.status_code == 400 - assert second.json()["error"]["code"] == "previous_response_complete_context_required" - assert connect_count == 2 - assert stale_upstream is not None - assert json.loads(stale_upstream.sent_text[-1])["previous_response_id"] == first_body["id"] - - # A subsequent complete full resend is safe to send without the rejected - # anchor. Quarantine must suppress re-injection on the fresh websocket. - third = await async_client.post( - "/v1/responses", - headers={"x-codex-session-id": "persistent-stale-anchor"}, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": [ - *historical_input, - *first_body["output"], - {"role": "user", "content": "hello-again"}, - ], - "prompt_cache_key": "invalid-request-rebind", - }, + assert second.status_code == 502 + assert second.json()["error"]["code"] in ( + "upstream_unavailable", + "stream_incomplete", + "bridge_owner_unreachable", + "bridge_continuity_persistence_failed", ) - - assert third.status_code == 200 - assert third.json()["output"][0]["content"][0]["text"] == "OK" - assert connect_count == 3 - assert recovered_upstream is not None - assert len(recovered_upstream.sent_text) == 1 - recovered_payload = json.loads(recovered_upstream.sent_text[0]) - assert "previous_response_id" not in recovered_payload - assert len(recovered_payload["input"]) == 3 - assert recovered_payload["input"][0] == historical_input[0] - assert recovered_payload["instructions"].endswith("stored per-turn control") + assert "previous_response_not_found" not in second.json()["error"].get("code", "") + assert connect_count == 1 -@pytest.mark.parametrize("use_latest_durable_anchor", [True, False], ids=["known-anchor", "unknown-anchor"]) -@pytest.mark.parametrize("include_completed_output", [True, False], ids=["complete-context", "incomplete-context"]) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_recovers_verified_full_resend_with_explicit_stale_anchor( +async def test_v1_responses_http_bridge_precreated_disconnect_returns_upstream_unavailable( async_client, app_instance, monkeypatch, - use_latest_durable_anchor, - include_completed_output, ): - """A complete owner-bound resend may drop its rejected explicit anchor once.""" _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_explicit_stale_anchor", - "http-bridge-explicit-stale-anchor@example.com", + "acc_http_bridge_precreated_previous_response", + "http-bridge-precreated-previous-response@example.com", ) account = await _get_account(account_id) - first_upstream = _FakeBridgeUpstreamWebSocket("resp_explicit_stale") - stale_upstream: _RejectStalePreviousResponseUpstreamWebSocket | None = None - recovered_upstream: _RejectStalePreviousResponseUpstreamWebSocket | None = None + fake_upstream = _FakeBridgeUpstreamWebSocket() + precreated_close_upstream = _PrecreatedCloseUpstreamWebSocket() connect_count = 0 - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline, kwargs + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + ) return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): @@ -16990,118 +13673,107 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - nonlocal connect_count del headers, access_token, account_id_header, base_url, session + nonlocal connect_count connect_count += 1 - if connect_count == 1: - return first_upstream - if connect_count == 2: - assert stale_upstream is not None - return stale_upstream - assert recovered_upstream is not None - return recovered_upstream + return fake_upstream if connect_count == 1 else precreated_close_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - session_id = "explicit-stale-anchor" - session_headers = {"x-codex-session-id": session_id} - historical_input = [{"role": "user", "content": "hello"}] first = await async_client.post( "/v1/responses", - headers=session_headers, json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": historical_input, - "prompt_cache_key": session_id, + "input": "hello", + "prompt_cache_key": "precreated-previous-response", }, ) - assert first.status_code == 200, first.text + assert first.status_code == 200 first_body = first.json() - stale_response_id = ( - first_body["id"] - if use_latest_durable_anchor - else "resp_0000000000000000000000000000000000000000000000000000000000000000" - ) - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket(stale_response_id) - recovered_upstream = _RejectStalePreviousResponseUpstreamWebSocket(stale_response_id) service = get_proxy_service_for_app(app_instance) async with service._http_bridge_lock: session = next(iter(service._http_bridge_sessions.values())) - await service._reset_http_bridge_session_after_local_terminal_error( - session, - error_code="test_transport_replaced", - error_message="Force a fresh socket while preserving the durable anchor", - preserve_durable_lease=True, - ) + await _replace_http_bridge_upstream_reader( + service, + session, + cast(proxy_module.UpstreamWebSocket, precreated_close_upstream), + ) - complete_resend = [ - *historical_input, - *(first_body["output"] if include_completed_output else []), - {"role": "user", "content": "continue from complete context"}, - ] - recovered = await async_client.post( + second = await async_client.post( "/v1/responses", - headers=session_headers, json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": complete_resend, - "prompt_cache_key": session_id, - "previous_response_id": stale_response_id, + "input": "hello-again", + "prompt_cache_key": "precreated-previous-response", + "previous_response_id": first_body["id"], }, ) - if not include_completed_output: - assert recovered.status_code == 502, recovered.text - assert recovered.json()["error"]["code"] == "bridge_previous_response_not_found" - assert connect_count == 3 - assert stale_upstream is not None - assert recovered_upstream is not None - assert all( - json.loads(sent)["previous_response_id"] == stale_response_id - for sent in [*stale_upstream.sent_text, *recovered_upstream.sent_text] - ) - return - - assert recovered.status_code == 200, recovered.text - assert recovered.json()["id"] == "resp_recovered_1" - assert connect_count == 3 - assert stale_upstream is not None - assert json.loads(stale_upstream.sent_text[0])["previous_response_id"] == stale_response_id - assert recovered_upstream is not None - recovered_payload = json.loads(recovered_upstream.sent_text[0]) - assert "previous_response_id" not in recovered_payload - assert recovered_payload["input"] == complete_resend + assert second.status_code == 502 + assert second.json()["error"]["code"] in ("upstream_unavailable", "stream_incomplete", "upstream_request_timeout") + assert "previous_response_not_found" not in second.json()["error"].get("code", "") + assert connect_count == 1 @pytest.mark.asyncio -async def test_v1_responses_http_bridge_recovers_manifest_bound_codex_retry_once( +async def test_v1_responses_http_bridge_rebinds_after_upstream_previous_response_not_found( async_client, app_instance, monkeypatch, -) -> None: +): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_manifest_recovery", - "http-bridge-manifest-recovery@example.com", + "acc_http_bridge_previous_response_rebind", + "http-bridge-previous-response-rebind@example.com", ) account = await _get_account(account_id) - first_upstream = _InterruptedCustomToolUpstreamWebSocket( - "resp_manifest_recovery", - emit_added=True, - empty_terminal_output=True, - ) - stale_upstream: _RejectStalePreviousResponseUpstreamWebSocket | None = None - recovered_upstream: _RejectStalePreviousResponseUpstreamWebSocket | None = None + first_upstream = _FakeBridgeUpstreamWebSocket() + recovered_upstream = _FakeBridgeUpstreamWebSocket() connect_count = 0 - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline, kwargs + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + api_key, + ) return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): @@ -17116,149 +13788,108 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - nonlocal connect_count del headers, access_token, account_id_header, base_url, session + nonlocal connect_count connect_count += 1 if connect_count == 1: return first_upstream - if connect_count == 2: - assert stale_upstream is not None - return stale_upstream - assert recovered_upstream is not None return recovered_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - session_id = "manifest-bound-codex-retry" - session_headers = {"x-codex-session-id": session_id} - historical_input = [{"role": "user", "content": "run the checks"}] first = await async_client.post( "/v1/responses", - headers=session_headers, json={ - "model": "gpt-5.6-sol", - "instructions": "Continue the task.", - "input": historical_input, - "prompt_cache_key": session_id, + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + "prompt_cache_key": "previous-response-rebind", }, ) - assert first.status_code == 200, first.text + assert first.status_code == 200 first_body = first.json() service = get_proxy_service_for_app(app_instance) - durable_lookup = await service._durable_bridge.lookup_request_targets( - session_key_kind="session_header", - session_key_value=session_id, - api_key_id=None, - turn_state=None, - session_header=session_id, - previous_response_id=first_body["id"], - ) - assert durable_lookup is not None - assert durable_lookup.latest_pending_tool_calls == {"call_custom_shell": "custom_tool_call"} - assert durable_lookup.latest_response_transition_manifest is not None - - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket(first_body["id"]) - recovered_upstream = _RejectStalePreviousResponseUpstreamWebSocket(first_body["id"]) async with service._http_bridge_lock: session = next(iter(service._http_bridge_sessions.values())) - await service._reset_http_bridge_session_after_local_terminal_error( - session, - error_code="test_transport_replaced", - error_message="Force a fresh socket while preserving the manifest checkpoint", - preserve_durable_lease=True, - ) + await _replace_http_bridge_upstream_reader( + service, + session, + cast(proxy_module.UpstreamWebSocket, _PreviousResponseNotFoundUpstreamWebSocket()), + ) - full_resend = [ - *historical_input, - *first_body["output"], - { - "type": "custom_tool_call_output", - "id": "ctco_manifest_recovery", - "call_id": "call_custom_shell", - "output": "verified", - "status": "completed", - "internal_chat_message_metadata_passthrough": { - "turn_id": "00000000-0000-4000-8000-000000000501", - "create_time": 1.0, - }, - }, - { - "type": "message", - "id": "msg_00000000-0000-4000-8000-000000000502", - "role": "developer", - "content": [{"type": "input_text", "text": "current app context"}], - "internal_chat_message_metadata_passthrough": {"turn_id": "00000000-0000-4000-8000-000000000503"}, - }, - { - "type": "message", - "id": "msg_00000000-0000-4000-8000-000000000504", - "role": "user", - "content": [{"type": "input_text", "text": "continue"}], - "internal_chat_message_metadata_passthrough": { - "turn_id": "00000000-0000-4000-8000-000000000503", - "create_time": 2.0, - }, - }, - ] - recovered = await async_client.post( + second = await async_client.post( "/v1/responses", - headers=session_headers, json={ - "model": "gpt-5.6-sol", - "instructions": "Continue the task.", - "input": full_resend, - "prompt_cache_key": session_id, + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello-again", + "prompt_cache_key": "previous-response-rebind", "previous_response_id": first_body["id"], }, ) - assert recovered.status_code == 200, recovered.text - assert recovered.json()["id"] == "resp_recovered_1" - assert connect_count == 3 - assert stale_upstream is not None - assert len(stale_upstream.sent_text) == 1 - stale_payload = json.loads(stale_upstream.sent_text[0]) - assert stale_payload.pop("previous_response_id") == first_body["id"] - assert recovered_upstream is not None - assert len(recovered_upstream.sent_text) == 1 - recovered_payload = json.loads(recovered_upstream.sent_text[0]) - assert "previous_response_id" not in recovered_payload - assert recovered_payload == stale_payload + assert second.status_code == 200 + assert second.json()["output"][0]["content"][0]["text"] == "OK" + assert connect_count == 2 @pytest.mark.asyncio -async def test_v1_responses_http_bridge_retains_stale_anchor_until_verified_replay_completes( +async def test_v1_responses_http_bridge_rebinds_after_upstream_invalid_request_previous_response_not_found_param( async_client, app_instance, monkeypatch, ): - """A failed fresh replay never destroys the last durable checkpoint.""" _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, - "acc_http_bridge_stale_anchor_checkpoint", - "http-bridge-stale-anchor-checkpoint@example.com", + "acc_http_bridge_invalid_request_rebind", + "http-bridge-invalid-request-rebind@example.com", ) account = await _get_account(account_id) - source_upstream = _ClosingBridgeUpstreamWebSocket("resp_checkpoint_source") - wedged_upstream = _EventsWithoutCreatedUpstreamWebSocket("resp_checkpoint_wedge") - first_closed_upstream = _ClosedBeforeSendUpstreamWebSocket() - second_closed_upstream = _ClosedBeforeSendUpstreamWebSocket() - recovered_upstream = _FakeBridgeUpstreamWebSocket("resp_checkpoint_recovered") - upstreams = [ - source_upstream, - wedged_upstream, - first_closed_upstream, - second_closed_upstream, - recovered_upstream, - ] + first_upstream = _FakeBridgeUpstreamWebSocket() + recovered_upstream = _FakeBridgeUpstreamWebSocket() connect_count = 0 - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline, kwargs + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + api_key, + ) return AccountSelection(account=account, error_message=None, error_code=None) async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): @@ -17273,108 +13904,52 @@ async def fake_connect_responses_websocket( base_url=None, session=None, ): - nonlocal connect_count del headers, access_token, account_id_header, base_url, session - upstream = upstreams[connect_count] + nonlocal connect_count connect_count += 1 - return upstream + if connect_count == 1: + return first_upstream + return recovered_upstream monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - session_id = "stale-anchor-checkpoint" - session_headers = {"x-codex-session-id": session_id} - historical_input = [{"role": "user", "content": "hello"}] first = await async_client.post( "/v1/responses", - headers=session_headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": historical_input, - }, - ) - assert first.status_code == 200, first.text - stale_response_id = "resp_checkpoint_source_1" - assert first.json()["id"] == stale_response_id - - wedging_resend = [ - *historical_input, - {"role": "user", "content": "follow-up without prior output"}, - ] - wedged = await async_client.post( - "/v1/responses", - headers=session_headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": wedging_resend, - }, - ) - assert wedged.status_code != 200 - assert connect_count == 2 - assert json.loads(wedged_upstream.sent_text[0])["previous_response_id"] == stale_response_id - - verified_resend = [ - *historical_input, - *first.json()["output"], - {"role": "user", "content": "continue safely"}, - ] - failed_replay = await async_client.post( - "/v1/responses", - headers=session_headers, json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": verified_resend, + "input": "hello", + "prompt_cache_key": "invalid-request-rebind", }, ) - assert failed_replay.status_code == 502, failed_replay.text - assert connect_count == 4 - assert first_closed_upstream.sent_text == [] - assert second_closed_upstream.sent_text == [] + assert first.status_code == 200 + first_body = first.json() service = get_proxy_service_for_app(app_instance) - durable_after_failed_replay = await service._durable_bridge.lookup_request_targets( - session_key_kind="session_header", - session_key_value=session_id, - api_key_id=None, - turn_state=None, - session_header=session_id, - previous_response_id=None, - ) - assert durable_after_failed_replay is not None - assert durable_after_failed_replay.latest_response_id == stale_response_id - retry_payload = proxy_module.ResponsesRequest( - model="gpt-5.1", - instructions="Return exactly OK.", - input=verified_resend, - ) - durable_retry_proof = http_bridge_streaming_module._verify_durable_full_resend( - retry_payload, - durable_after_failed_replay, - ) - assert durable_retry_proof is not None - assert durable_retry_proof.matches(retry_payload, durable_after_failed_replay) + async with service._http_bridge_lock: + session = next(iter(service._http_bridge_sessions.values())) + await _replace_http_bridge_upstream_reader( + service, + session, + cast(proxy_module.UpstreamWebSocket, _InvalidRequestPreviousResponseUpstreamWebSocket()), + ) - recovered = await async_client.post( + second = await async_client.post( "/v1/responses", - headers=session_headers, json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": verified_resend, + "input": "hello-again", + "prompt_cache_key": "invalid-request-rebind", + "previous_response_id": first_body["id"], }, ) - assert recovered.status_code == 200, recovered.text - assert recovered.json()["id"] == "resp_checkpoint_recovered_1" - assert connect_count == 5 - assert len(recovered_upstream.sent_text) == 1 - recovered_payload = json.loads(recovered_upstream.sent_text[0]) - assert "previous_response_id" not in recovered_payload - assert recovered_payload["input"] == verified_resend + assert second.status_code == 200 + assert second.json()["output"][0]["content"][0]["text"] == "OK" + assert connect_count == 2 @pytest.mark.asyncio @@ -17383,7 +13958,8 @@ async def test_v1_responses_http_bridge_masks_anonymous_previous_response_not_fo monkeypatch, ): _install_bridge_settings(monkeypatch, enabled=True) - upstream = _AnonymousPreviousResponseNotFoundWithInflightUpstreamWebSocket() + service = get_proxy_service_for_app(app_instance) + upstream = _TwoSameAnchorFollowupsPreviousResponseNotFoundUpstreamWebSocket() connect_count = 0 async def fake_select_account_with_budget( @@ -17451,6 +14027,8 @@ async def fake_connect_responses_websocket( AsyncClient(transport=ASGITransport(app=app_instance), base_url="http://testserver") as admin_client, AsyncClient(transport=ASGITransport(app=app_instance), base_url="http://testserver") as first_client, AsyncClient(transport=ASGITransport(app=app_instance), base_url="http://testserver") as second_client, + AsyncClient(transport=ASGITransport(app=app_instance), base_url="http://testserver") as third_client, + AsyncClient(transport=ASGITransport(app=app_instance), base_url="http://testserver") as fourth_client, ): account_id = await _import_account( admin_client, @@ -17459,28 +14037,39 @@ async def fake_connect_responses_websocket( ) account = await _get_account(account_id) + anchor_response = await first_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello-seed", + "prompt_cache_key": "previous-response-anchor-seed", + }, + ) + first = asyncio.create_task( - first_client.post( + second_client.post( "/v1/responses", json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "previous-response-inflight-mixed", + "input": "hello-a", + "prompt_cache_key": "previous-response-inflight-origin", + "previous_response_id": anchor_response.json()["id"], }, ) ) - await _wait_for_event(upstream.first_request_created) + await _wait_for_event(upstream.first_followup_created) second = asyncio.create_task( - second_client.post( + third_client.post( "/v1/responses", json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": "hello-again", - "prompt_cache_key": "previous-response-inflight-mixed", - "previous_response_id": "resp_bridge_prev_anchor", + "input": "hello-b", + "prompt_cache_key": "previous-response-inflight-anchor", + "previous_response_id": anchor_response.json()["id"], }, ) ) @@ -17490,13 +14079,31 @@ async def fake_connect_responses_websocket( timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - assert first_response.status_code == 200 - assert first_response.json()["output"][0]["content"][0]["text"] == "OK" + third_response = await fourth_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello-on-anchor-again", + "prompt_cache_key": "previous-response-after-error", + }, + ) + + assert not any(not future.done() for future in service._http_bridge_inflight_sessions.values()) + + assert anchor_response.status_code == 200 + assert anchor_response.json()["output"][0]["content"][0]["text"] == "OK" + assert first_response.status_code >= 400 + assert first_response.json()["error"]["code"] == "stream_incomplete" assert second_response.status_code >= 400 assert second_response.json()["error"]["code"] == "stream_incomplete" + assert "previous_response_not_found" not in first_response.json()["error"].get("code", "") + assert "previous_response_not_found" not in first_response.json()["error"].get("message", "") assert "previous_response_not_found" not in second_response.json()["error"].get("code", "") assert "previous_response_not_found" not in second_response.json()["error"].get("message", "") - assert connect_count == 1 + assert third_response.status_code == 200 + assert third_response.json()["output"][0]["content"][0]["text"] == "OK" + assert connect_count == 2 @pytest.mark.asyncio @@ -18507,201 +15114,50 @@ async def test_prepare_http_bridge_request_preserves_existing_client_metadata(ap assert first_request_state.request_id != second_request_state.request_id -class _EventsWithoutCreatedUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): - """Streams response events but never ``response.created``, then closes. - - Models the #1534 production wedge: a reattached HTTP-bridge stream that - delivers upstream response events whose ``response.created`` is never - assigned, so the turn can only end without a completed response. - """ - - async def send_text(self, text: str) -> None: - self.sent_text.append(text) - for delta in ("thinking", " harder"): - await self._messages.put( - _FakeUpstreamMessage( - "text", - text=json.dumps( - {"type": "response.reasoning_summary_text.delta", "delta": delta}, - separators=(",", ":"), - ), - ) - ) - await self._messages.put(_FakeUpstreamMessage("close", close_code=1000)) - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_quarantines_reattach_that_streams_without_response_created( - async_client, app_instance, monkeypatch -): - """Regression for #1534: a reattach that streams events but never gets - ``response.created`` must quarantine the session so the next request does - not rebuild the identical anchored reattach and instead completes on the - fresh no-anchor path.""" - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_quarantine_silent", - "http-bridge-quarantine-silent@example.com", - ) - account = await _get_account(account_id) - service = get_proxy_service_for_app(app_instance) - http_bridge_quarantine_module._http_bridge_quarantine_registry(service).clear() - first_upstream = _ClosingBridgeUpstreamWebSocket("resp_quarantine_source") - wedged_upstream = _EventsWithoutCreatedUpstreamWebSocket("resp_quarantine_wedge") - fresh_upstream = _FakeBridgeUpstreamWebSocket("resp_quarantine_fresh") - upstreams = [first_upstream, wedged_upstream, fresh_upstream] - connect_count = 0 - - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline, kwargs - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - nonlocal connect_count - del headers, access_token, account_id_header, base_url, session - connect_count += 1 - return upstreams[connect_count - 1] - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - - session_headers = {"x-codex-session-id": "quarantine-silent-reattach"} - historical_input = [{"role": "user", "content": "hello"}] - first = await asyncio.wait_for( - async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": historical_input, - }, - headers=session_headers, - ), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, - ) - assert first.status_code == 200, first.text - - # ``response.completed`` resolves the HTTP caller before the reader task - # necessarily consumes the immediately following clean-close frame. Wait - # for that exact physical bridge to retire before issuing the reattach; - # otherwise a loaded runner can race the follow-up into a session that is - # already closed but not yet unregistered, bypassing the wedge this test - # is meant to exercise. - first_bridge_retired = False - deadline = time.monotonic() + _TEST_SYNC_TIMEOUT_SECONDS - while time.monotonic() < deadline: - async with service._http_bridge_lock: - first_bridge_retired = all( - candidate.upstream is not first_upstream for candidate in service._http_bridge_sessions.values() - ) - if first_bridge_retired: - break - await asyncio.sleep(0.01) - assert first_bridge_retired - - wedging_resend = [ - *historical_input, - {"role": "user", "content": "follow-up without prior output"}, - ] - second = await asyncio.wait_for( - async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": wedging_resend, - }, - headers=session_headers, - ), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, - ) - - # The reattach injected the durable anchor and then wedged: events flowed - # but response.created never arrived, so the turn fails terminally. - assert second.status_code != 200 - assert len(wedged_upstream.sent_text) == 1 - wedged_payload = json.loads(wedged_upstream.sent_text[0]) - assert wedged_payload["previous_response_id"] == "resp_quarantine_source_1" - quarantined_entries = [ - entry - for entry in http_bridge_quarantine_module._http_bridge_quarantine_registry(service).values() - if entry.quarantined_until > time.monotonic() - ] - assert len(quarantined_entries) == 1 - assert quarantined_entries[0].reason == "reattach_missing_response_created" - - verified_resend = [ - *historical_input, - *first.json()["output"], - {"role": "user", "content": "continue safely"}, - ] - third = await asyncio.wait_for( - async_client.post( - "/v1/responses", - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": verified_resend, - }, - headers=session_headers, - ), - timeout=_TEST_SYNC_TIMEOUT_SECONDS, - ) - - # The quarantined key must not rebuild the identical anchored reattach: - # the client's own full resend goes upstream unanchored and completes. - assert third.status_code == 200, third.text - assert third.json()["id"] == "resp_quarantine_fresh_1" - assert connect_count == 3 - assert len(fresh_upstream.sent_text) == 1 - fresh_payload = json.loads(fresh_upstream.sent_text[0]) - assert "previous_response_id" not in fresh_payload - assert fresh_payload["input"] == verified_resend - # The completed response on the fresh path clears the quarantine again. - assert not [ - entry - for entry in http_bridge_quarantine_module._http_bridge_quarantine_registry(service).values() - if entry.quarantined_until > time.monotonic() - ] - +class _EventsWithoutCreatedUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + """Streams response events but never ``response.created``, then closes. + + Models the #1534 production wedge: a reattached HTTP-bridge stream that + delivers upstream response events whose ``response.created`` is never + assigned, so the turn can only end without a completed response. + """ + + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + for delta in ("thinking", " harder"): + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + {"type": "response.reasoning_summary_text.delta", "delta": delta}, + separators=(",", ":"), + ), + ) + ) + await self._messages.put(_FakeUpstreamMessage("close", close_code=1000)) + @pytest.mark.asyncio -async def test_v1_responses_http_bridge_quarantined_unproved_full_resend_remains_anchored( +async def test_v1_responses_http_bridge_quarantines_reattach_that_streams_without_response_created( async_client, app_instance, monkeypatch ): - """Quarantine never authorizes an unproved context-dropping replay. - - A multi-item payload can look like a full resend while omitting the prior - assistant/tool output. It must retain the durable anchor and fail closed; - only the exact owner-bound complete resend may then bypass quarantine. - """ - _install_bridge_settings(monkeypatch, enabled=True) + """Regression for #1534: a reattach that streams events but never gets + ``response.created`` must quarantine the session so the next request does + not rebuild the identical anchored reattach and instead completes on the + fresh no-anchor path.""" + _install_bridge_settings_with_limits(monkeypatch, enabled=True, instance_id=socket.gethostname()) account_id = await _import_account( async_client, - "acc_http_bridge_quarantine_unsafe_suffix", - "http-bridge-quarantine-unsafe-suffix@example.com", + "acc_http_bridge_quarantine_silent", + "http-bridge-quarantine-silent@example.com", ) account = await _get_account(account_id) service = get_proxy_service_for_app(app_instance) http_bridge_quarantine_module._http_bridge_quarantine_registry(service).clear() - first_upstream = _ClosingInterruptedCustomToolUpstreamWebSocket("resp_quarantine_unsafe_source") - wedged_upstream = _EventsWithoutCreatedUpstreamWebSocket("resp_quarantine_unsafe_wedge") - stale_anchor_upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_bridge_custom_1") - fresh_upstream = _FakeBridgeUpstreamWebSocket("resp_quarantine_unsafe_fresh") - upstreams = [first_upstream, wedged_upstream, stale_anchor_upstream, fresh_upstream] + first_upstream = _ClosingInterruptedCustomToolUpstreamWebSocket("resp_quarantine_source") + wedged_upstream = _EventsWithoutCreatedUpstreamWebSocket("resp_quarantine_wedge") + fresh_upstream = _FakeBridgeUpstreamWebSocket("resp_quarantine_fresh") + upstreams = [first_upstream, wedged_upstream, fresh_upstream] connect_count = 0 async def fake_select_account_with_budget(self, deadline, **kwargs): @@ -18729,13 +15185,7 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - # The turn-state header makes the bridge session a true Codex continuity - # session (``session.codex_session``), which is what arms the session-level - # anchor injection this regression guards against. - session_headers = { - "x-codex-session-id": "quarantine-unsafe-suffix-reattach", - "x-codex-turn-state": "quarantine-unsafe-suffix-turn", - } + session_headers = {"x-codex-session-id": "quarantine-silent-reattach"} historical_input = [ {"role": "user", "content": [{"type": "input_text", "text": "leading question"}]}, { @@ -18809,435 +15259,77 @@ async def fake_connect_responses_websocket( timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - # The reattach injected the durable anchor and then wedged: the key is now - # quarantined. + # The reattach injected the durable anchor and then wedged: events flowed + # but response.created never arrived, so the turn fails terminally. assert second.status_code != 200 assert len(wedged_upstream.sent_text) == 1 - assert json.loads(wedged_upstream.sent_text[0])["previous_response_id"] == "resp_bridge_custom_1" - assert [ + wedged_payload = json.loads(wedged_upstream.sent_text[0]) + assert wedged_payload["previous_response_id"] == "resp_bridge_custom_1" + quarantined_entries = [ entry for entry in http_bridge_quarantine_module._http_bridge_quarantine_registry(service).values() if entry.quarantined_until > time.monotonic() ] + assert len(quarantined_entries) == 1 + assert quarantined_entries[0].reason == "reattach_missing_response_created" - # Full resend whose durable prefix is trimmable but whose fresh suffix is - # a plain user turn: it neither retains the prior output nor matches the - # pending tool calls, so the safe-fresh-context proof fails. - unsafe_suffix_resend = [ - *historical_input, - {"role": "user", "content": [{"type": "input_text", "text": "follow-up without prior output"}]}, - ] third = await asyncio.wait_for( async_client.post( "/v1/responses", json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": unsafe_suffix_resend, + "input": full_resend, }, headers=session_headers, ), timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - # Merely looking like a full resend is insufficient proof. The rejected - # anchor stays attached, so missing prior output cannot be silently lost. - assert third.status_code == 400, third.text - assert third.json()["error"]["code"] == "previous_response_pending_call_resolution_required" + # The quarantined key must not rebuild the identical anchored reattach: + # the client's own full resend goes upstream unanchored and completes. + assert third.status_code == 200, third.text + assert third.json()["id"] == "resp_quarantine_fresh_1" assert connect_count == 3 - assert len(stale_anchor_upstream.sent_text) == 1 - stale_payload = json.loads(stale_anchor_upstream.sent_text[0]) - assert stale_payload["previous_response_id"] == "resp_bridge_custom_1" - - # No unsafe payload was dispatched unanchored. The separate positive - # quarantine test above proves that an exact owner-bound full resend still - # recovers and clears quarantine. - assert fresh_upstream.sent_text == [] + assert len(fresh_upstream.sent_text) == 1 + fresh_payload = json.loads(fresh_upstream.sent_text[0]) + assert "previous_response_id" not in fresh_payload + assert fresh_payload["input"] == full_resend + # The completed response on the fresh path clears the quarantine again. + assert not [ + entry + for entry in http_bridge_quarantine_module._http_bridge_quarantine_registry(service).values() + if entry.quarantined_until > time.monotonic() + ] -@pytest.mark.parametrize( - ("case_id", "stored_count", "pending_type", "followup_kind"), - [ - ("stored-85-custom-user", 85, "custom_tool_call", "user"), - ("stored-288-function-agent-user", 288, "function_call", "agent-user"), - ("stored-194-custom-terminal", 194, "custom_tool_call", "none"), - ], -) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_persists_pending_resolution_and_blocks_scheduled_delta_before_connect( - async_client, - app_instance, - monkeypatch, - case_id, - stored_count, - pending_type, - followup_kind, +async def test_v1_responses_http_bridge_quarantined_unsafe_full_resend_dispatches_unanchored( + async_client, app_instance, monkeypatch ): - """A known stale pending-call anchor is not rediscovered every wake-up.""" - - _install_bridge_settings(monkeypatch, enabled=True) + """Regression for the #1534 session-state side door: a quarantined + full-resend whose durable prefix is trimmable but whose fresh suffix does + NOT retain the prior output must go upstream genuinely unanchored. Before + the fix, the early durable-anchor injection was suppressed but session + hydration restored ``last_completed_response_id`` and the session-level + injection re-added the same anchor and trimmed the prefix — rebuilding the + wedge despite the ``fresh_reattach_anchor_skipped_quarantined`` log.""" + _install_bridge_settings_with_limits(monkeypatch, enabled=True, instance_id=socket.gethostname()) account_id = await _import_account( async_client, - f"acc_http_bridge_durable_recovery_required_{case_id}", - f"http-bridge-durable-recovery-required-{case_id}@example.com", + "acc_http_bridge_quarantine_unsafe_suffix", + "http-bridge-quarantine-unsafe-suffix@example.com", ) account = await _get_account(account_id) - source_upstream = _ClosingInterruptedCustomToolUpstreamWebSocket( - "resp_recovery_required_source", - tool_call_type=pending_type, - ) - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_bridge_custom_1") - replacement_upstream = _FakeBridgeUpstreamWebSocket("resp_recovery_required_replacement") - upstreams = [source_upstream, stale_upstream, replacement_upstream] - connect_count = 0 - - async def fake_select_account_with_budget(self, deadline, **kwargs): - del self, deadline, kwargs - return AccountSelection(account=account, error_message=None, error_code=None) - - async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): - del self, force, timeout_seconds - return target - - async def fake_connect_responses_websocket( - headers, - access_token, - account_id_header, - *, - base_url=None, - session=None, - ): - nonlocal connect_count - del headers, access_token, account_id_header, base_url, session - upstream = upstreams[connect_count] - connect_count += 1 - return upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) - monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) - monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - service = get_proxy_service_for_app(app_instance) - headers = { - "x-codex-session-id": f"durable-recovery-required-{case_id}", - "x-codex-turn-state": f"durable-recovery-required-turn-{case_id}", - } - stored_input = [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": f"sanitized stored item {index}"}], - } - for index in range(stored_count) - ] - first = await async_client.post( - "/v1/responses", - headers=headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": stored_input, - }, - ) - assert first.status_code == 200, first.text - - # ``response.completed`` can resolve the caller before the reader consumes - # the immediately queued clean-close frame. Wait for the exact source - # bridge to retire so the follow-up deterministically exercises fresh - # stale-anchor reattach instead of racing a closed-but-registered socket. - source_bridge_retired = False - deadline = time.monotonic() + _TEST_SYNC_TIMEOUT_SECONDS - while time.monotonic() < deadline: - async with service._http_bridge_lock: - source_bridge_retired = all( - candidate.upstream is not source_upstream for candidate in service._http_bridge_sessions.values() - ) - if source_bridge_retired: - break - await asyncio.sleep(0.01) - assert source_bridge_retired - - first_delta = await async_client.post( - "/v1/responses", - headers=headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": [{"role": "user", "content": "scheduled continue"}], - }, - ) - assert first_delta.status_code == 400, first_delta.text - assert first_delta.json()["error"]["code"] == "previous_response_pending_call_resolution_required" - assert connect_count == 2 - assert json.loads(stale_upstream.sent_text[0])["previous_response_id"] == "resp_bridge_custom_1" - - durable_marker = await service._durable_bridge.lookup_request_targets( - session_key_kind="turn_state_header", - session_key_value=headers["x-codex-turn-state"], - api_key_id=None, - turn_state=headers["x-codex-turn-state"], - session_header=headers["x-codex-session-id"], - previous_response_id="resp_bridge_custom_1", - ) - assert durable_marker is not None - assert durable_marker.latest_pending_tool_calls == {"call_custom_shell": pending_type} - assert durable_marker.recovery_is_required_for_latest_anchor() is True - - # Simulate process-local quarantine loss. The durable marker remains the - # admission authority and the bad tool output cannot satisfy either - # existing complete-resend proof. http_bridge_quarantine_module._http_bridge_quarantine_registry(service).clear() - mismatched_full_resend = [ - *stored_input, - { - "type": "custom_tool_call", - "call_id": "call_different_tool", - "name": "shell", - "input": "pwd", - }, - { - "type": "custom_tool_call_output", - "call_id": "call_different_tool", - "output": "not the pending result", - }, - ] - repeated = await async_client.post( - "/v1/responses", - headers=headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": mismatched_full_resend, - }, - ) - assert repeated.status_code == 400, repeated.text - assert repeated.json()["error"]["code"] == "previous_response_pending_call_resolution_required" - assert connect_count == 2 - assert replacement_upstream.sent_text == [] - - call_item = { - "type": pending_type, - "call_id": "call_custom_shell", - "name": "shell", - ("input" if pending_type == "custom_tool_call" else "arguments"): ( - "pwd" if pending_type == "custom_tool_call" else "{}" - ), - } - output_item = { - "type": f"{pending_type}_output", - "call_id": "call_custom_shell", - "output": "interrupted before client execution", - } - reasoning_item = { - "type": "reasoning", - "id": "rs_08639659bba14680016a8a0902e9a887d090946ff27b898f05", - "content": None, - "encrypted_content": "opaque", - "summary": [], - } - bounded_followups = [] - if followup_kind == "agent-user": - bounded_followups.append( - { - "type": "agent_message", - "id": "amsg_01a02b33-3b30-7742-bdb3-091f07cf2ea0", - "author": "/root/fixture_worker", - "recipient": "/root", - "content": [{"type": "input_text", "text": "sanitized agent output"}], - } - ) - if followup_kind in {"user", "agent-user"}: - bounded_followups.append( - { - "type": "message", - "id": "msg_01a02c43-4980-7afb-97f5-2e2d30aa73de", - "role": "user", - "content": [{"type": "input_text", "text": "sanitized followup"}], - } - ) - verified_full_resend = [ - *stored_input, - reasoning_item, - call_item, - output_item, - *bounded_followups, - ] - task_id = f"durable-recovery-required-{case_id}" - official_headers, official_client_metadata = _official_codex_turn_carriers( - task_id, - f"turn-{case_id}", - ) - recovery_headers = {**headers, **official_headers} - peer_verified_full_resend = copy.deepcopy(verified_full_resend) - peer_output = next(item for item in peer_verified_full_resend if item.get("type") == f"{pending_type}_output") - peer_output["output"] = "different but valid pending-call resolution" - for candidate in (verified_full_resend, peer_verified_full_resend): - candidate_payload = proxy_module.ResponsesRequest.model_validate( - { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": candidate, - } - ) - candidate_proof = http_bridge_streaming_module._verify_durable_full_resend( - candidate_payload, - durable_marker, - ) - assert candidate_proof is not None - assert candidate_proof.matches(candidate_payload, durable_marker) - - # Both bodies independently satisfy the exact pending-call proof, but one - # marker generation may bind only one projected wire fingerprint. Hold the - # winner at response.create so the loser must fail before submit rather - # than dispatching after the winner clears the marker. - async def recover_once(): - return await async_client.post( - "/v1/responses", - headers=recovery_headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "prompt_cache_key": task_id, - "client_metadata": official_client_metadata, - "input": verified_full_resend, - }, - ) - - async def recover_peer(): - return await async_client.post( - "/v1/responses", - headers=recovery_headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "prompt_cache_key": task_id, - "client_metadata": official_client_metadata, - "input": peer_verified_full_resend, - }, - ) - - async with SessionLocal() as db_session: - assert await db_session.scalar(select(HttpBridgeRowlessRecoveryAuthority.id)) is None - - ( - concurrent_results, - marker_submit_count, - record_observations, - ) = await _run_concurrent_marker_recoveries_at_response_create_gate( - service=service, - monkeypatch=monkeypatch, - recover_once=recover_once, - recover_peer=recover_peer, - ) - assert sorted(result.status_code for result in concurrent_results) == [200, 502], record_observations - recovered = next(result for result in concurrent_results if result.status_code == 200) - rejected = next(result for result in concurrent_results if result.status_code != 200) - assert recovered.status_code == 200, recovered.text - assert rejected.status_code == 502, rejected.text - assert rejected.json()["error"]["code"] == "bridge_continuity_persistence_failed" - assert marker_submit_count == 1 - assert connect_count == 3 - assert len(replacement_upstream.sent_text) == 1 - replacement_payload = json.loads(replacement_upstream.sent_text[0]) - assert "previous_response_id" not in replacement_payload - normalized_verified_payloads = [ - proxy_module.ResponsesRequest.model_validate( - { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": candidate, - } - ) - for candidate in (verified_full_resend, peer_verified_full_resend) - ] - assert replacement_payload["instructions"] == normalized_verified_payloads[0].instructions - assert replacement_payload["input"] in [candidate.input for candidate in normalized_verified_payloads] - winning_normalized_payload = next( - candidate for candidate in normalized_verified_payloads if candidate.input == replacement_payload["input"] - ) - assert isinstance(winning_normalized_payload.input, list) - expected_complete_input_fingerprint = proxy_module._fingerprint_input_items( - cast(list[proxy_module.JsonValue], winning_normalized_payload.input) - ) - replacement_call_ids = [ - item.get("call_id") for item in replacement_payload["input"] if item.get("type") == pending_type - ] - replacement_output_ids = [ - item.get("call_id") for item in replacement_payload["input"] if item.get("type") == f"{pending_type}_output" - ] - assert replacement_call_ids == ["call_custom_shell"] - assert replacement_output_ids == ["call_custom_shell"] - - durable_replacement = await service._durable_bridge.lookup_request_targets( - session_key_kind="turn_state_header", - session_key_value=headers["x-codex-turn-state"], - api_key_id=None, - turn_state=headers["x-codex-turn-state"], - session_header=headers["x-codex-session-id"], - previous_response_id=recovered.json()["id"], - ) - assert durable_replacement is not None - assert durable_replacement.account_id == account.id - assert durable_replacement.latest_response_id == recovered.json()["id"] - assert durable_replacement.latest_input_item_count == len(verified_full_resend) - assert durable_replacement.latest_input_full_fingerprint == expected_complete_input_fingerprint - assert durable_replacement.recovery_is_required_for_latest_anchor() is False - assert durable_replacement.recovery_required_attempt_fingerprint is None - - async with SessionLocal() as db_session: - attempts = list( - ( - await db_session.scalars( - select(HttpBridgeRecoveryAttemptRecord).where( - HttpBridgeRecoveryAttemptRecord.session_id == durable_marker.session_id - ) - ) - ).all() - ) - assert len(attempts) == 1 - assert attempts[0].state == HttpBridgeRecoveryAttemptState.REPLAYED - assert attempts[0].response_id == recovered.json()["id"] - assert len(source_upstream.sent_text) == 1 - - followup = await async_client.post( - "/v1/responses", - headers=headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": [{"role": "user", "content": "continue after replacement"}], - }, - ) - assert followup.status_code == 200, followup.text - assert connect_count == 3 - followup_payload = json.loads(replacement_upstream.sent_text[1]) - assert "previous_response_id" not in followup_payload - assert followup_payload["input"] == [{"role": "user", "content": "continue after replacement"}] - - -@pytest.mark.asyncio -async def test_v1_responses_http_bridge_recovers_abandoned_pending_call_after_anchor_rejection( - async_client, - app_instance, - monkeypatch, -): - """A rejected anchor may discard an unaccepted call without losing client continuity.""" - - _install_bridge_settings(monkeypatch, enabled=True) - account_id = await _import_account( - async_client, - "acc_http_bridge_abandoned_pending", - "http-bridge-abandoned-pending@example.com", - ) - account = await _get_account(account_id) - source_upstream = _ClosingInterruptedCustomToolUpstreamWebSocket("resp_abandoned_pending_source") - stale_upstream = _RejectStalePreviousResponseUpstreamWebSocket("resp_bridge_custom_1") - recovered_upstream = _FakeBridgeUpstreamWebSocket("resp_abandoned_pending_recovered") - upstreams = [source_upstream, stale_upstream, recovered_upstream] + first_upstream = _ClosingInterruptedCustomToolUpstreamWebSocket("resp_quarantine_unsafe_source") + wedged_upstream = _EventsWithoutCreatedUpstreamWebSocket("resp_quarantine_unsafe_wedge") + fresh_upstream = _FakeBridgeUpstreamWebSocket("resp_quarantine_unsafe_fresh") + upstreams = [first_upstream, wedged_upstream, fresh_upstream] connect_count = 0 async def fake_select_account_with_budget(self, deadline, **kwargs): - nonlocal connect_count del self, deadline, kwargs return AccountSelection(account=account, error_message=None, error_code=None) @@ -19255,437 +15347,212 @@ async def fake_connect_responses_websocket( ): nonlocal connect_count del headers, access_token, account_id_header, base_url, session - upstream = upstreams[connect_count] connect_count += 1 - return upstream + return upstreams[connect_count - 1] monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + # The turn-state header makes the bridge session a true Codex continuity + # session (``session.codex_session``), which is what arms the session-level + # anchor injection this regression guards against. session_headers = { - "x-codex-session-id": "abandoned-pending-agent-boundary", - "x-codex-turn-state": "abandoned-pending-agent-boundary-turn", - } - - def response_owned_user_message(*, message_id: str, text: str, create_time: float) -> dict[str, Any]: - return { - "type": "message", - "id": f"msg_{message_id}", - "role": "user", - "content": [{"type": "input_text", "text": text}], - "internal_chat_message_metadata_passthrough": { - "turn_id": message_id, - "create_time": create_time, - }, - } - - historical_agent_message = { - "type": "agent_message", - "id": "amsg_01a02b33-3b30-7742-bdb3-091f07cf2ea1", - "author": "/root/historical_worker", - "recipient": "/root", - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2daa", - "create_time": 1787431100.0, - }, - "content": [{"type": "input_text", "text": "historical inter-agent result"}], - } - historical_assistant_message = { - "type": "message", - "id": "msg_01a02b31-bc02-70b0-a09e-0dedbc2e2dac", - "role": "assistant", - "phase": "final_answer", - "content": [{"type": "output_text", "text": "historical task completed"}], - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2dac", - "create_time": 1787431110.0, - }, - } - historical_developer_message = { - "type": "message", - "id": "msg_01a02a78-d2b5-71e3-a33e-fab25a40b322", - "role": "developer", - "content": [ - {"type": "input_text", "text": "stored permissions"}, - {"type": "input_text", "text": "stored app context"}, - {"type": "input_text", "text": "stored collaboration mode"}, - {"type": "input_text", "text": "stored skills"}, - ], - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02a74-35a4-7bd3-bc2e-ba95279d6c04", - }, + "x-codex-session-id": "quarantine-unsafe-suffix-reattach", + "x-codex-turn-state": "quarantine-unsafe-suffix-turn", } - - def http_transport_normalize(item: dict[str, Any]) -> dict[str, Any]: - normalized = copy.deepcopy(item) - normalized.pop("internal_chat_message_metadata_passthrough", None) - return normalized - historical_input = [ + {"role": "user", "content": [{"type": "input_text", "text": "leading question"}]}, { "type": "additional_tools", "role": "developer", - "tools": [ - { - "type": "custom", - "name": "shell", - "description": "execute a bounded shell command", - "format": {"type": "text"}, - } - ], + "tools": [{"type": "custom", "name": "shell"}], }, { "type": "message", "role": "developer", - "content": [{"type": "input_text", "text": "canonical Responses Lite instructions"}], - }, - http_transport_normalize( - response_owned_user_message( - message_id="01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - text="first question", - create_time=1787431100.0, - ) - ), - http_transport_normalize(historical_developer_message), - http_transport_normalize(historical_agent_message), - http_transport_normalize(historical_assistant_message), - ] - first = await async_client.post( - "/v1/responses", - headers=session_headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": historical_input, + "content": [{"type": "input_text", "text": "canonical Lite instructions"}], }, - ) - assert first.status_code == 200, first.text - - first_delta = await async_client.post( - "/v1/responses", - headers=session_headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": [{"role": "user", "content": "scheduled continue"}], + { + "role": "user", + "content": [{"type": "input_text", "text": "first question"}], }, - ) - assert first_delta.status_code == 400, first_delta.text - assert first_delta.json()["error"]["code"] == "previous_response_pending_call_resolution_required" - assert connect_count == 2 - - service = get_proxy_service_for_app(app_instance) - - agent_message = http_transport_normalize( { - "type": "agent_message", - "id": "amsg_01a02b33-3b30-7742-bdb3-091f07cf2ea0", - "author": "/root/episode_identity_final_audit", - "recipient": "/root", - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - "create_time": 1787431172.912141, - }, - "content": [{"type": "input_text", "text": "verified inter-agent result"}], - } - ) - developer_followup = http_transport_normalize( + "type": "custom_tool_call", + "call_id": "call_historical_shell", + "name": "shell", + "input": "printf historical", + }, { - "type": "message", - "id": "msg_01a02d3a-0319-76f1-9fd0-b28e9b9bc2d7", "role": "developer", - "content": [ - {"type": "input_text", "text": "updated permissions"}, - {"type": "input_text", "text": "updated app context"}, - {"type": "input_text", "text": "updated skills"}, - ], - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02d3a-0319-76f1-9fd0-b28e9b9bc2d7", - "create_time": 1787433450.0, + "content": [{"type": "input_text", "text": "historical control"}], + }, + { + "type": "custom_tool_call_output", + "call_id": "call_historical_shell", + "output": "historical", + }, + ] + first = await asyncio.wait_for( + async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": historical_input, }, - } + headers=session_headers, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) + assert first.status_code == 200, first.text + full_resend = [ *historical_input, { - "type": "reasoning", - "id": "rs_08639659bba14680016a8a0904462c87d08ae1115f2aef2ccc", - "content": None, - "encrypted_content": "opaque", - "summary": [], + "type": "custom_tool_call", + "call_id": "call_custom_shell", + "name": "shell", + "input": "pwd", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_custom_shell", + "output": "/workspace", }, - agent_message, - http_transport_normalize( - response_owned_user_message( - message_id="01a02c31-2f60-7dd2-9f22-d7ef316596b1", - text="first retry", - create_time=1787433300.0, - ) - ), - developer_followup, - http_transport_normalize( - response_owned_user_message( - message_id="01a02c43-4980-7afb-97f5-2e2d30aa73de", - text="second retry", - create_time=1787433402.605, - ) - ), ] - peer_full_resend = copy.deepcopy(full_resend) - peer_full_resend[-1] = http_transport_normalize( - response_owned_user_message( - message_id="01a02c43-4980-7afb-97f5-2e2d30aa73df", - text="different second retry", - create_time=1787433403.0, - ) - ) - durable_lookup = await service._durable_bridge.lookup_request_targets( - session_key_kind="turn_state_header", - session_key_value=session_headers["x-codex-turn-state"], - api_key_id=None, - turn_state=session_headers["x-codex-turn-state"], - session_header=session_headers["x-codex-session-id"], - previous_response_id=None, - ) - assert durable_lookup is not None - assert durable_lookup.latest_pending_tool_calls == {"call_custom_shell": "custom_tool_call"} - assert durable_lookup.recovery_is_required_for_latest_anchor() is True - proof_payload = proxy_module.ResponsesRequest.model_validate( - {"model": "gpt-5.1", "instructions": "Return exactly OK.", "input": full_resend} - ) - assert isinstance(proof_payload.input, list) - proof_input = cast(list[proxy_module.JsonValue], proof_payload.input) - assert durable_lookup.latest_input_item_count == len(historical_input) - assert http_bridge_streaming_module._input_prefix_matches_stored_context( - proof_input, - stored_count=len(historical_input), - stored_fingerprint=durable_lookup.latest_input_full_fingerprint, - ) - parsed_suffix = cast(list[dict[str, proxy_module.JsonValue]], proof_input[len(historical_input) :]) - assert replay_safety_module._is_response_owned_reasoning_boundary_item(parsed_suffix[0]) - assert replay_safety_module._is_retained_agent_message(parsed_suffix[1]) - assert replay_safety_module._is_response_owned_user_message(parsed_suffix[2]) - assert replay_safety_module._is_response_owned_developer_message(parsed_suffix[3]) - assert replay_safety_module._is_response_owned_user_message(parsed_suffix[4]) - proof_projection = replay_safety_module.project_responses_input_for_abandoned_pending_fresh_replay( - proof_input, - stored_count=len(historical_input), - pending_tool_calls=durable_lookup.latest_pending_tool_calls, - ) - assert proof_projection is not None - assert replay_safety_module.responses_input_suffix_proves_abandoned_pending_agent_boundary( - proof_input, - stored_count=len(historical_input), - pending_tool_calls=durable_lookup.latest_pending_tool_calls, - ) - assert ( - http_bridge_streaming_module._verify_durable_abandoned_pending_full_resend( - proof_payload, - durable_lookup, - ) - is not None - ) - peer_proof_payload = proxy_module.ResponsesRequest.model_validate( - {"model": "gpt-5.1", "instructions": "Return exactly OK.", "input": peer_full_resend} - ) - assert ( - http_bridge_streaming_module._verify_durable_abandoned_pending_full_resend( - peer_proof_payload, - durable_lookup, - ) - is not None - ) - - async def recover_once(): - return await async_client.post( + second = await asyncio.wait_for( + async_client.post( "/v1/responses", - headers=session_headers, json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", "input": full_resend, }, - ) + headers=session_headers, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, + ) + + # The reattach injected the durable anchor and then wedged: the key is now + # quarantined. + assert second.status_code != 200 + assert len(wedged_upstream.sent_text) == 1 + assert json.loads(wedged_upstream.sent_text[0])["previous_response_id"] == "resp_bridge_custom_1" + assert [ + entry + for entry in http_bridge_quarantine_module._http_bridge_quarantine_registry(service).values() + if entry.quarantined_until > time.monotonic() + ] - async def recover_peer(): - return await async_client.post( + # Full resend whose durable prefix is trimmable but whose fresh suffix is + # a plain user turn: it neither retains the prior output nor matches the + # pending tool calls, so the safe-fresh-context proof fails. + unsafe_suffix_resend = [ + *historical_input, + {"role": "user", "content": [{"type": "input_text", "text": "follow-up without prior output"}]}, + ] + third = await asyncio.wait_for( + async_client.post( "/v1/responses", - headers=session_headers, json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": peer_full_resend, + "input": unsafe_suffix_resend, }, - ) + headers=session_headers, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, + ) - ( - concurrent_results, - marker_submit_count, - record_observations, - ) = await _run_concurrent_marker_recoveries_at_response_create_gate( - service=service, - monkeypatch=monkeypatch, - recover_once=recover_once, - recover_peer=recover_peer, - ) - assert sorted(result.status_code for result in concurrent_results) == [200, 502], record_observations - recovered = next(result for result in concurrent_results if result.status_code == 200) - rejected = next(result for result in concurrent_results if result.status_code != 200) - assert rejected.json()["error"]["code"] == "bridge_continuity_persistence_failed" - assert marker_submit_count == 1 - assert recovered.status_code == 200, recovered.text - assert recovered.json()["id"] == "resp_abandoned_pending_recovered_1" + # The dispatch must be genuinely unanchored: no early durable injection, + # no session-level re-injection of the same anchor, no prefix trim. + assert third.status_code == 200, third.text + assert third.json()["id"] == "resp_quarantine_unsafe_fresh_1" assert connect_count == 3 - assert len(stale_upstream.sent_text) == 1 - stale_payload = json.loads(stale_upstream.sent_text[0]) - assert stale_payload["previous_response_id"] == "resp_bridge_custom_1" - assert stale_payload["input"] == [{"role": "user", "content": "scheduled continue"}] - assert len(recovered_upstream.sent_text) == 1 - recovered_payload = json.loads(recovered_upstream.sent_text[0]) - assert "previous_response_id" not in recovered_payload - recovered_last_user_text = recovered_payload["input"][-1]["content"][0]["text"] - assert recovered_last_user_text in {"second retry", "different second retry"} - assert recovered_payload["input"] == [ - historical_input[0], - historical_input[1], - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "first question"}], - }, - { - "type": "message", - "role": "developer", - "content": historical_developer_message["content"], - }, - { - "type": "message", - "role": "assistant", - "phase": "final_answer", - "content": [{"type": "output_text", "text": "historical task completed"}], - }, - agent_message, - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "first retry"}], - }, - { - "type": "message", - "role": "developer", - "content": developer_followup["content"], - }, - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": recovered_last_user_text}], - }, - ] - assert all(item.get("call_id") != "call_custom_shell" for item in recovered_payload["input"]) + assert len(fresh_upstream.sent_text) == 1 + fresh_payload = json.loads(fresh_upstream.sent_text[0]) + assert "previous_response_id" not in fresh_payload + assert fresh_payload["input"] == unsafe_suffix_resend - winning_full_resend = full_resend if recovered_last_user_text == "second retry" else peer_full_resend - winning_payload = proxy_module.ResponsesRequest.model_validate( - {"model": "gpt-5.1", "instructions": "Return exactly OK.", "input": winning_full_resend} - ) - assert isinstance(winning_payload.input, list) - expected_full_resend_fingerprint = proxy_module._fingerprint_input_items( - cast(list[proxy_module.JsonValue], winning_payload.input) - ) - durable_after_recovery = await service._durable_bridge.lookup_request_targets( - session_key_kind="turn_state_header", - session_key_value=session_headers["x-codex-turn-state"], - api_key_id=None, - turn_state=session_headers["x-codex-turn-state"], - session_header=session_headers["x-codex-session-id"], - previous_response_id=recovered.json()["id"], - ) - assert durable_after_recovery is not None - assert durable_after_recovery.latest_input_item_count == len(full_resend) - assert durable_after_recovery.latest_input_full_fingerprint == expected_full_resend_fingerprint - assert durable_after_recovery.recovery_is_required_for_latest_anchor() is False - assert durable_after_recovery.recovery_required_attempt_fingerprint is None - live_session = next(iter(service._http_bridge_sessions.values())) - assert live_session.last_completed_input_count == len(full_resend) - assert live_session.last_completed_input_prefix_fingerprint == expected_full_resend_fingerprint - - async with SessionLocal() as db_session: - attempts = list( - ( - await db_session.scalars( - select(HttpBridgeRecoveryAttemptRecord).where( - HttpBridgeRecoveryAttemptRecord.session_id == durable_lookup.session_id - ) - ) - ).all() - ) - assert len(attempts) == 1 - assert attempts[0].state == HttpBridgeRecoveryAttemptState.REPLAYED - assert attempts[0].response_id == recovered.json()["id"] - - recovered_output = recovered.json()["output"][0] - next_developer_followup = { - "type": "message", - "role": "developer", - "content": [{"type": "input_text", "text": "next-turn permissions"}], - } - next_user = { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "continue after recovery"}], + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_successor_claim_fences_the_retiring_release( + async_client, app_instance, monkeypatch +): + """Deterministic route-level regression for issue #1695. + + The retiring session's teardown releases its durable row concurrently with + the next request's successor claim. Here the predecessor's release is held + captive until the successor's claim (and its 200) have completed, then let + loose — on the pre-fix code the release still matched the fence (the + same-owner claim kept the epoch) and closed the row out from under the + live successor; the claim must advance the epoch so the late release + no-ops and a third turn keeps working. + """ + _install_bridge_settings_with_limits(monkeypatch, enabled=True, instance_id=socket.gethostname()) + account_id = await _import_account(async_client, "acc_http_bridge_fence", "http-bridge-fence@example.com") + account = await _get_account(account_id) + upstreams = [_ClosingBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket(), _FakeBridgeUpstreamWebSocket()] + connect_count = 0 + + async def fake_select_account_with_budget(self, deadline, **kwargs): + del self, deadline, kwargs + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, access_token, account_id_header, *, base_url=None, session=None + ): + del headers, access_token, account_id_header, base_url, session + nonlocal connect_count + upstream = upstreams[connect_count] + connect_count += 1 + return upstream + + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + service = get_proxy_service_for_app(app_instance) + coordinator = service._durable_bridge + real_release = coordinator.release_live_session + release_gate = asyncio.Event() + captive_releases: list[dict] = [] + + async def captive_release_live_session(**kwargs): + if not release_gate.is_set(): + captive_releases.append(kwargs) + return None + return await real_release(**kwargs) + + monkeypatch.setattr(coordinator, "release_live_session", captive_release_live_session) + + payload = { + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + "prompt_cache_key": f"http-bridge-fence-thread-{account_id}", } - next_full_resend = [ - *winning_full_resend, - recovered_output, - next_developer_followup, - next_user, - ] - followup = await async_client.post( - "/v1/responses", - headers=session_headers, - json={ - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": next_full_resend, - }, + first = await asyncio.wait_for(async_client.post("/v1/responses", json=payload), timeout=_TEST_SYNC_TIMEOUT_SECONDS) + assert first.status_code == 200 + # The successor claims while the predecessor's release is still captive — + # the ordering the CI flake hits nondeterministically. + second = await asyncio.wait_for( + async_client.post("/v1/responses", json=payload), timeout=_TEST_SYNC_TIMEOUT_SECONDS ) + assert second.status_code == 200, second.text + assert captive_releases, "the retiring session must have attempted its durable release" - assert followup.status_code == 200, followup.text - assert followup.json()["id"] == "resp_abandoned_pending_recovered_2" - assert connect_count == 3 - assert len(recovered_upstream.sent_text) == 2 - followup_payload = json.loads(recovered_upstream.sent_text[1]) - assert followup_payload["previous_response_id"] == recovered.json()["id"] - assert followup_payload["input"] == [ - recovered_output, - next_developer_followup, - next_user, - ] - assert not any(item.get("call_id") == "call_custom_shell" for item in followup_payload["input"]) - assert not any(item == agent_message for item in followup_payload["input"]) - assert not any(item == developer_followup for item in followup_payload["input"]) + # Let the predecessor's release land late, fenced on its old epoch. + release_gate.set() + for kwargs in captive_releases: + await real_release(**kwargs) - next_payload = proxy_module.ResponsesRequest.model_validate( - { - "model": "gpt-5.1", - "instructions": "Return exactly OK.", - "input": next_full_resend, - } - ) - assert isinstance(next_payload.input, list) - expected_next_fingerprint = proxy_module._fingerprint_input_items( - cast(list[proxy_module.JsonValue], next_payload.input) - ) - durable_after_followup = await service._durable_bridge.lookup_request_targets( - session_key_kind="turn_state_header", - session_key_value=session_headers["x-codex-turn-state"], - api_key_id=None, - turn_state=session_headers["x-codex-turn-state"], - session_header=session_headers["x-codex-session-id"], - previous_response_id=followup.json()["id"], - ) - assert durable_after_followup is not None - assert durable_after_followup.latest_input_item_count == len(next_full_resend) - assert durable_after_followup.latest_input_full_fingerprint == expected_next_fingerprint - assert live_session.last_completed_input_count == len(next_full_resend) - assert live_session.last_completed_input_prefix_fingerprint == expected_next_fingerprint + # The late release must not have closed the successor's row: a third turn + # on the same key keeps working instead of failing with 409. + third = await asyncio.wait_for(async_client.post("/v1/responses", json=payload), timeout=_TEST_SYNC_TIMEOUT_SECONDS) + assert third.status_code == 200, third.text diff --git a/tests/integration/test_http_upgrade_tolerance.py b/tests/integration/test_http_upgrade_tolerance.py new file mode 100644 index 0000000000..0e397467fe --- /dev/null +++ b/tests/integration/test_http_upgrade_tolerance.py @@ -0,0 +1,459 @@ +"""Regression tests for issue #1757: h2c upgrade offers must not break requests. + +JetBrains/Ktor clients opportunistically attach ``Connection: Upgrade`` + +``Upgrade: h2c`` + ``HTTP2-Settings`` to plain HTTP/1.1 POSTs. The stock +uvicorn httptools protocol treats any such request as a protocol switch and +wedges the parser: a body coalesced with the headers is silently dropped, and +a body written as a separate segment (Ktor's pattern) turns into +``400 Invalid HTTP request received.``. + +The suite covers three layers: + +- protocol-level tests driving ``UpgradeTolerantHttpToolsProtocol`` through a + fake transport with both client segmentations; +- canary tests pinning the stock behavior these fixes exist for (if a uvicorn + upgrade makes them fail, the subclass can likely be retired); +- a live-server test using the production protocol wiring over real sockets, + including a real WebSocket upgrade that must keep completing. +""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import pytest +import uvicorn +from uvicorn.protocols.http.httptools_impl import HttpToolsProtocol +from uvicorn.server import ServerState + +from app.cli import _load_http_protocol_class +from app.core.http_protocol import UpgradeTolerantH11Protocol +from app.core.http_protocol_httptools import UpgradeTolerantHttpToolsProtocol + +pytestmark = pytest.mark.integration + +_BODY = json.dumps({"model": "gpt-5.6-luna", "input": "hello", "stream": False}).encode() +_H2C_HEAD = ( + b"POST /echo HTTP/1.1\r\n" + b"Host: 127.0.0.1\r\n" + b"Connection: Upgrade, HTTP2-Settings\r\n" + b"Upgrade: h2c\r\n" + b"HTTP2-Settings: AAMAAABkAARAAAAAAAIAAAAA\r\n" + b"Content-Type: application/json\r\n" + b"Content-Length: " + str(len(_BODY)).encode() + b"\r\n" + b"\r\n" +) + + +async def _echo_app(scope: dict[str, Any], receive: Any, send: Any) -> None: + """Echo the request body and the header names the application observed.""" + if scope["type"] == "websocket": + await receive() + await send({"type": "websocket.accept"}) + message = await receive() + await send({"type": "websocket.send", "text": message.get("text", "")}) + await send({"type": "websocket.close"}) + return + + assert scope["type"] == "http" + body = b"" + while True: + message = await receive() + body += message.get("body", b"") + if not message.get("more_body", False): + break + payload = json.dumps( + { + "echo": body.decode(), + "header_names": sorted({name.decode() for name, _ in scope["headers"]}), + } + ).encode() + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [(b"content-type", b"application/json"), (b"content-length", str(len(payload)).encode())], + } + ) + await send({"type": "http.response.body", "body": payload}) + + +class _FakeTransport(asyncio.Transport): + def __init__(self) -> None: + super().__init__() + self.buffer = bytearray() + self.closed = False + self.protocol: asyncio.BaseProtocol | None = None + + def write(self, data: bytes | bytearray | memoryview) -> None: + self.buffer.extend(data) + + def is_closing(self) -> bool: + return self.closed + + def close(self) -> None: + self.closed = True + + def abort(self) -> None: + self.closed = True + + def pause_reading(self) -> None: + pass + + def resume_reading(self) -> None: + pass + + def set_protocol(self, protocol: asyncio.BaseProtocol) -> None: + self.protocol = protocol + + def get_extra_info(self, name: str, default: Any = None) -> Any: + if name == "sockname": + return ("127.0.0.1", 2455) + if name == "peername": + return ("127.0.0.1", 54321) + return default + + +def _make_protocol(protocol_class: type[Any]) -> tuple[Any, _FakeTransport]: + config = uvicorn.Config(app=_echo_app, lifespan="off") + config.load() + protocol = protocol_class(config=config, server_state=ServerState(), app_state={}) + transport = _FakeTransport() + protocol.connection_made(transport) + return protocol, transport + + +async def _wait_for_response(transport: _FakeTransport, timeout: float = 5.0) -> bytes: + async with asyncio.timeout(timeout): + while b"\r\n\r\n" not in transport.buffer or not bytes(transport.buffer).split(b"\r\n\r\n", 1)[1]: + await asyncio.sleep(0.01) + return bytes(transport.buffer) + + +def _parse_json_body(raw_response: bytes) -> dict[str, Any]: + _, _, body = raw_response.partition(b"\r\n\r\n") + return json.loads(body) + + +async def test_h2c_offer_with_coalesced_body_is_served_as_http11() -> None: + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(_H2C_HEAD + _BODY) + + raw_response = await _wait_for_response(transport) + assert raw_response.startswith(b"HTTP/1.1 200 OK"), raw_response + payload = _parse_json_body(raw_response) + assert payload["echo"] == _BODY.decode() + # The declined offer's hop-by-hop headers must not reach the application. + assert "upgrade" not in payload["header_names"] + assert "http2-settings" not in payload["header_names"] + assert "connection" not in payload["header_names"] + assert not transport.closed + + +async def test_h2c_offer_with_split_head_and_body_is_served_as_http11() -> None: + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(_H2C_HEAD) + await asyncio.sleep(0.01) + protocol.data_received(_BODY) + + raw_response = await _wait_for_response(transport) + assert raw_response.startswith(b"HTTP/1.1 200 OK"), raw_response + assert _parse_json_body(raw_response)["echo"] == _BODY.decode() + assert not transport.closed + + +async def test_h2c_offer_keeps_connection_reusable_for_next_request() -> None: + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(_H2C_HEAD + _BODY) + await _wait_for_response(transport) + transport.buffer.clear() + + follow_up = b"POST /echo HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Length: 2\r\n\r\nhi" + protocol.data_received(follow_up) + + raw_response = await _wait_for_response(transport) + assert raw_response.startswith(b"HTTP/1.1 200 OK"), raw_response + assert _parse_json_body(raw_response)["echo"] == "hi" + + +async def test_h2c_offer_with_repeated_connection_fields_is_served_as_http11() -> None: + """Repeated ``Connection`` fields must be combined when classifying the offer. + + uvicorn's ``_get_upgrade`` keeps only the tokens of the last ``Connection`` + field, so ``Connection: Upgrade`` followed by ``Connection: keep-alive`` + would hide the offer and reproduce the original body loss. + """ + head = ( + b"POST /echo HTTP/1.1\r\n" + b"Host: 127.0.0.1\r\n" + b"Connection: Upgrade, HTTP2-Settings\r\n" + b"Upgrade: h2c\r\n" + b"HTTP2-Settings: AAMAAABkAARAAAAAAAIAAAAA\r\n" + b"Connection: keep-alive\r\n" + b"Content-Type: application/json\r\n" + b"Content-Length: " + str(len(_BODY)).encode() + b"\r\n" + b"\r\n" + ) + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(head) + await asyncio.sleep(0.01) + protocol.data_received(_BODY) + + raw_response = await _wait_for_response(transport) + assert raw_response.startswith(b"HTTP/1.1 200 OK"), raw_response + payload = _parse_json_body(raw_response) + assert payload["echo"] == _BODY.decode() + assert "upgrade" not in payload["header_names"] + assert "http2-settings" not in payload["header_names"] + # The unrelated keep-alive token survives the sanitization. + assert "connection" in payload["header_names"] + + +async def test_pipelined_h2c_offers_in_one_segment_do_not_exhaust_the_stack() -> None: + """Many pipelined upgrade offers in one segment must not recurse per offer. + + Each declined offer replays the remaining bytes through a fresh parser. + Done recursively that was one stack frame per pipelined request, so a + single ~66KB segment of minimal h2c GETs (well under asyncio's 256KiB + per-read buffer) raised RecursionError out of ``data_received``, aborting + the connection — and pinned an O(depth x segment) pile of byte copies + while unwinding. The replay must be iterative. + """ + request = b"GET /echo HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: Upgrade\r\nUpgrade: h2c\r\n\r\n" + count = 2000 # ~148KB, double the depth that overflows the default 1000-frame stack + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(request * count) # raised RecursionError when replay was recursive + + async with asyncio.timeout(30.0): + while transport.buffer.count(b"HTTP/1.1 200 OK") < count: + await asyncio.sleep(0.05) + assert not transport.closed + + +async def test_h11_fallback_serves_h2c_offer_and_hides_upgrade_headers() -> None: + """The httptools-less fallback keeps the body and the header hygiene.""" + protocol, transport = _make_protocol(UpgradeTolerantH11Protocol) + + protocol.data_received(_H2C_HEAD) + await asyncio.sleep(0.01) + protocol.data_received(_BODY) + + raw_response = await _wait_for_response(transport) + assert raw_response.startswith(b"HTTP/1.1 200 OK"), raw_response + payload = _parse_json_body(raw_response) + assert payload["echo"] == _BODY.decode() + assert "upgrade" not in payload["header_names"] + assert "http2-settings" not in payload["header_names"] + assert "connection" not in payload["header_names"] + + +async def test_websocket_handshake_with_repeated_connection_fields_switches_protocols() -> None: + """Combined-field classification must not regress (or hide) WebSocket handoffs.""" + handshake = ( + b"GET /ws HTTP/1.1\r\n" + b"Host: 127.0.0.1\r\n" + b"Connection: Upgrade\r\n" + b"Upgrade: websocket\r\n" + b"Connection: keep-alive\r\n" + b"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + b"Sec-WebSocket-Version: 13\r\n" + b"\r\n" + ) + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(handshake) + + async with asyncio.timeout(5.0): + while b"\r\n\r\n" not in transport.buffer: + await asyncio.sleep(0.01) + assert bytes(transport.buffer).startswith(b"HTTP/1.1 101 Switching Protocols"), bytes(transport.buffer) + # The connection was handed off to the WebSocket protocol. + assert transport.protocol is not None + + +async def test_websocket_handshake_with_multiple_upgrade_tokens_reaches_the_websocket_stack() -> None: + """``Upgrade: websocket, h2c`` must be classified as a WebSocket handshake. + + The ``Upgrade`` field is a comma-separated protocol list (RFC 9110 + section 7.8) and the server may pick any offered protocol it supports. + Matching the raw field value against ``websocket`` would misclassify the + handshake as an ignorable offer and answer it *as the application* over + plain HTTP/1.1 (with the WebSocket headers stripped from the scope). The + handshake verdict belongs to the WebSocket stack: uvicorn's default + ``websockets`` implementation currently rejects multi-token ``Upgrade`` + values with 426, other implementations may complete the 101. + """ + handshake = ( + b"GET /ws HTTP/1.1\r\n" + b"Host: 127.0.0.1\r\n" + b"Connection: Upgrade\r\n" + b"Upgrade: websocket, h2c\r\n" + b"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + b"Sec-WebSocket-Version: 13\r\n" + b"\r\n" + ) + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(handshake) + + async with asyncio.timeout(5.0): + while b"\r\n\r\n" not in transport.buffer: + await asyncio.sleep(0.01) + status_line = bytes(transport.buffer).split(b"\r\n", 1)[0] + assert status_line in (b"HTTP/1.1 101 Switching Protocols", b"HTTP/1.1 426 Upgrade Required"), status_line + # The connection was handed off to the WebSocket protocol, not the app. + assert transport.protocol is not None + + +async def test_live_v1_responses_route_serves_split_h2c_offer(db_setup, monkeypatch: pytest.MonkeyPatch) -> None: + """Issue #1757's exact product path: split-written h2c POST to /v1/responses. + + The request must traverse the parser into the application instead of the + stock transport-layer ``400 Invalid HTTP request received.``. With an + empty account pool the route deterministically answers a JSON + ``no_accounts`` 503 — reaching that error proves the request body was + delivered and parsed (the balancer logs the requested model), which is + exactly what the wedged stock parser prevented. + """ + import app.main as main_module + + async def _noop_init_db() -> None: + return None + + monkeypatch.setattr(main_module, "init_db", _noop_init_db) + config = uvicorn.Config( + app=main_module.create_app(), + host="127.0.0.1", + port=0, + http=_load_http_protocol_class(), + log_level="warning", + ) + server = uvicorn.Server(config) + serve_task = asyncio.create_task(server.serve()) + try: + async with asyncio.timeout(10.0): + while not server.started: + await asyncio.sleep(0.01) + port = server.servers[0].sockets[0].getsockname()[1] + + body = json.dumps({"model": "gpt-5.2", "input": "hello", "stream": False}).encode() + head = ( + b"POST /v1/responses HTTP/1.1\r\n" + b"Host: 127.0.0.1\r\n" + b"Authorization: Bearer sk-bogus\r\n" + b"Connection: Upgrade, HTTP2-Settings\r\n" + b"Upgrade: h2c\r\n" + b"HTTP2-Settings: AAMAAABkAARAAAAAAAIAAAAA\r\n" + b"Content-Type: application/json\r\n" + b"Content-Length: " + str(len(body)).encode() + b"\r\n" + b"\r\n" + ) + reader, writer = await asyncio.open_connection("127.0.0.1", port) + writer.write(head) + await writer.drain() + await asyncio.sleep(0.05) + writer.write(body) + await writer.drain() + async with asyncio.timeout(10.0): + status_line = await reader.readline() + raw_headers = await reader.readuntil(b"\r\n\r\n") + content_length = next( + int(line.split(b":", 1)[1]) + for line in raw_headers.lower().splitlines() + if line.startswith(b"content-length:") + ) + payload = json.loads(await reader.readexactly(content_length)) + assert status_line == b"HTTP/1.1 503 Service Unavailable\r\n", status_line + assert payload["error"]["code"] == "no_accounts", payload + writer.close() + await writer.wait_closed() + finally: + server.should_exit = True + async with asyncio.timeout(15.0): + await serve_task + + +async def test_stock_httptools_protocol_still_breaks_on_h2c_offers() -> None: + """Canary pinning the upstream defect this module works around. + + The stock parser drops a coalesced body (the application observes an empty + body) and answers 400 when the body arrives as a separate segment. If a + uvicorn/httptools upgrade makes this test fail, upstream has fixed + https://github.com/Soju06/codex-lb/issues/1757 and + ``UpgradeTolerantHttpToolsProtocol`` can likely be retired. + """ + protocol, transport = _make_protocol(HttpToolsProtocol) + protocol.data_received(_H2C_HEAD + _BODY) + raw_response = await _wait_for_response(transport) + assert raw_response.startswith(b"HTTP/1.1 200 OK") + assert _parse_json_body(raw_response)["echo"] == "" # body silently dropped + + protocol, transport = _make_protocol(HttpToolsProtocol) + protocol.data_received(_H2C_HEAD) + await asyncio.sleep(0.01) + protocol.data_received(_BODY) + async with asyncio.timeout(5.0): + while b"Invalid HTTP request received." not in transport.buffer: + await asyncio.sleep(0.01) + # The body segment hits the wedged parser: the application never sees the + # payload and the client's request ends in a 400. + assert b'"echo": ""' in transport.buffer + assert b"HTTP/1.1 400 Bad Request" in transport.buffer + + +async def test_live_server_serves_h2c_offers_and_websocket_upgrades() -> None: + """End-to-end proof over real sockets with the production protocol wiring.""" + config = uvicorn.Config( + app=_echo_app, + host="127.0.0.1", + port=0, + http=_load_http_protocol_class(), + lifespan="off", + log_level="warning", + ) + server = uvicorn.Server(config) + serve_task = asyncio.create_task(server.serve()) + try: + async with asyncio.timeout(10.0): + while not server.started: + await asyncio.sleep(0.01) + port = server.servers[0].sockets[0].getsockname()[1] + + # Ktor's write pattern: headers first, body as a separate segment. + reader, writer = await asyncio.open_connection("127.0.0.1", port) + writer.write(_H2C_HEAD) + await writer.drain() + await asyncio.sleep(0.05) + writer.write(_BODY) + await writer.drain() + async with asyncio.timeout(10.0): + status_line = await reader.readline() + assert status_line == b"HTTP/1.1 200 OK\r\n" + raw_headers = await reader.readuntil(b"\r\n\r\n") + content_length = next( + int(line.split(b":", 1)[1]) + for line in raw_headers.lower().splitlines() + if line.startswith(b"content-length:") + ) + payload = json.loads(await reader.readexactly(content_length)) + assert payload["echo"] == _BODY.decode() + writer.close() + await writer.wait_closed() + + # A real WebSocket upgrade must keep switching protocols. + from websockets.asyncio.client import connect + + async with connect(f"ws://127.0.0.1:{port}/ws") as websocket: + await websocket.send("ping") + assert await websocket.recv() == "ping" + finally: + server.should_exit = True + async with asyncio.timeout(10.0): + await serve_task diff --git a/tests/integration/test_live_usage_ingest.py b/tests/integration/test_live_usage_ingest.py index db079ee890..9e16704921 100644 --- a/tests/integration/test_live_usage_ingest.py +++ b/tests/integration/test_live_usage_ingest.py @@ -1,17 +1,27 @@ from __future__ import annotations import asyncio +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any, cast import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.sql.dml import Delete +from app.core.clients import proxy as core_proxy from app.core.crypto import TokenEncryptor +from app.core.openai.requests import ResponsesRequest from app.core.usage import live_hub from app.core.usage.live_snapshots import LiveRateLimitSnapshot, LiveUsageWindow from app.core.utils.time import utcnow from app.db.models import Account, AccountStatus, UsageHistory from app.db.session import SessionLocal +from app.modules.accounts import repository as accounts_repository_module from app.modules.accounts.repository import AccountsRepository from app.modules.usage import live_ingest +from app.modules.usage import repository as usage_repository_module from app.modules.usage.repository import UsageRepository pytestmark = pytest.mark.integration @@ -43,6 +53,21 @@ def _snapshot() -> LiveRateLimitSnapshot: ) +async def _usage_rows_for(*account_ids: str) -> list[UsageHistory]: + async with SessionLocal() as session: + return list( + ( + await session.execute( + select(UsageHistory) + .where(UsageHistory.account_id.in_(account_ids)) + .order_by(UsageHistory.account_id, UsageHistory.window, UsageHistory.id) + ) + ) + .scalars() + .all() + ) + + async def _wait_for_rows(account_id: str, *, timeout: float = 5.0) -> tuple[UsageHistory | None, UsageHistory | None]: deadline = asyncio.get_event_loop().time() + timeout while True: @@ -234,22 +259,585 @@ async def test_live_ingestor_normalizes_monthly_only_snapshots(db_setup) -> None @pytest.mark.asyncio -async def test_live_ingestor_resolves_chatgpt_account_id(db_setup) -> None: +async def test_live_ingestor_settles_snapshot_after_duplicate_account_consolidation(db_setup) -> None: del db_setup + canonical_id = "acc_live_consolidated" + duplicate_id = "acc_live_consolidated__copy" + upstream_id = "workspace-live-consolidated" + email = "live-consolidated@example.com" + async with SessionLocal() as session: - await AccountsRepository(session).upsert( - _make_account("acc_live_resolved", "live-resolved@example.com", chatgpt_account_id="workspace-live-1") + repo = AccountsRepository(session) + await repo.upsert( + _make_account(canonical_id, email, chatgpt_account_id=upstream_id), + merge_by_email=False, + ) + await repo.upsert( + _make_account(duplicate_id, email, chatgpt_account_id=upstream_id), + merge_by_email=False, ) + snapshot = _snapshot() ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) - ingestor.start() + ingestor.publish( + snapshot, + account_id=duplicate_id, + chatgpt_account_id=upstream_id, + ) + queued = ingestor._queue.get_nowait() + + async with SessionLocal() as session: + saved = await AccountsRepository(session).upsert( + _make_account("acc_live_consolidated_reauth", email, chatgpt_account_id=upstream_id), + merge_by_email=False, + merge_by_chatgpt_identity=True, + ) + assert saved.id == canonical_id + assert await session.get(Account, duplicate_id) is None + + await ingestor._ingest(queued) + + rows = await _usage_rows_for(canonical_id, duplicate_id) + assert [row.account_id for row in rows] == [canonical_id, canonical_id] + primary_rows = [row for row in rows if row.window == "primary"] + secondary_rows = [row for row in rows if row.window == "secondary"] + assert len(primary_rows) == 1 + assert len(secondary_rows) == 1 + + primary = primary_rows[0] + secondary = secondary_rows[0] + assert snapshot.primary is not None + assert snapshot.secondary is not None + assert primary.used_percent == pytest.approx(snapshot.primary.used_percent) + assert primary.window_minutes == snapshot.primary.window_minutes + assert primary.reset_at == snapshot.primary.reset_at + assert primary.credits_has == snapshot.credits_has + assert primary.credits_unlimited == snapshot.credits_unlimited + assert primary.credits_balance == pytest.approx(snapshot.credits_balance) + assert secondary.used_percent == pytest.approx(snapshot.secondary.used_percent) + assert secondary.window_minutes == snapshot.secondary.window_minutes + assert secondary.reset_at == snapshot.secondary.reset_at + + +@pytest.mark.asyncio +async def test_sse_publication_tap_settles_queued_duplicate_snapshot_under_canonical_account( + monkeypatch: pytest.MonkeyPatch, + db_setup, +) -> None: + del db_setup + canonical_id = "acc_live_sse_canonical" + duplicate_id = "acc_live_sse_duplicate" + upstream_id = "workspace-live-sse" + email = "live-sse@example.com" + async with SessionLocal() as session: + repo = AccountsRepository(session) + await repo.upsert(_make_account(canonical_id, email, chatgpt_account_id=upstream_id), merge_by_email=False) + await repo.upsert(_make_account(duplicate_id, email, chatgpt_account_id=upstream_id), merge_by_email=False) + + rate_limit_event = ( + 'data: {"type":"codex.rate_limits","rate_limits":' + '{"primary":{"used_percent":33,"window_minutes":300,"reset_at":1700000300},' + '"secondary":{"used_percent":44,"window_minutes":10080,"reset_at":1700604800}}}\n\n' + ) + + @asynccontextmanager + async def _fake_http_session(_session): + yield cast(Any, object()) + + async def _fake_upstream_stream(**kwargs): + assert kwargs["account_id"] == upstream_id + assert kwargs["codex_lb_account_id"] == duplicate_id + yield rate_limit_event + + monkeypatch.setattr(core_proxy, "lease_http_session", _fake_http_session) + monkeypatch.setattr(core_proxy, "_stream_responses_with_session", _fake_upstream_stream) + + ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) + ingest_completed = asyncio.Event() + ingest_snapshot = ingestor._ingest + + async def _observed_ingest(item: live_ingest._QueuedSnapshot) -> None: + await ingest_snapshot(item) + ingest_completed.set() + + monkeypatch.setattr(ingestor, "_ingest", _observed_ingest) + live_hub.register_live_usage_publisher(ingestor.publish) try: - ingestor.publish(_snapshot(), chatgpt_account_id="workspace-live-1") - primary, secondary = await _wait_for_rows("acc_live_resolved") + events = [ + event + async for event in core_proxy.stream_responses( + ResponsesRequest(model="gpt-5.1", instructions="", input="hello", stream=True), + {}, + "access-token", + upstream_id, + session=cast(Any, object()), + codex_lb_account_id=duplicate_id, + ) + ] + assert events == [rate_limit_event] + + async with SessionLocal() as session: + saved = await AccountsRepository(session).upsert( + _make_account("acc_live_sse_reauth", email, chatgpt_account_id=upstream_id), + merge_by_email=False, + merge_by_chatgpt_identity=True, + ) + assert saved.id == canonical_id + + ingestor.start() + await asyncio.wait_for(ingest_completed.wait(), timeout=5.0) finally: await ingestor.stop() + live_hub.register_live_usage_publisher(None) - assert primary is not None and secondary is not None + rows = await _usage_rows_for(canonical_id, duplicate_id) + assert [row.account_id for row in rows] == [canonical_id, canonical_id] + assert {row.window for row in rows} == {"primary", "secondary"} + assert {row.used_percent for row in rows} == {33.0, 44.0} + + +@pytest.mark.asyncio +async def test_postgresql_live_ingest_serializes_identity_membership_through_snapshot_commit( + monkeypatch: pytest.MonkeyPatch, + db_setup, +) -> None: + del db_setup + bind = SessionLocal.kw["bind"] + if bind.dialect.name != "postgresql": + pytest.skip("PostgreSQL transaction-lock regression") + + canonical_id = "acc_live_pg_canonical" + duplicate_id = "acc_live_pg_duplicate" + upstream_id = "workspace-live-pg-consolidated" + email = "live-pg-consolidated@example.com" + async with SessionLocal() as session: + repo = AccountsRepository(session) + await repo.upsert( + _make_account(canonical_id, email, chatgpt_account_id=upstream_id), + merge_by_email=False, + ) + await repo.upsert( + _make_account(duplicate_id, email, chatgpt_account_id=upstream_id), + merge_by_email=False, + ) + + ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) + ingestor.publish( + _snapshot(), + account_id=duplicate_id, + chatgpt_account_id=upstream_id, + ) + queued = ingestor._queue.get_nowait() + + settlement_commit_started = asyncio.Event() + release_settlement_commit = asyncio.Event() + writer_lock_attempted = asyncio.Event() + release_writer_delete = asyncio.Event() + settlement_lock_keys: list[int] = [] + writer_lock_keys: list[int] = [] + settlement_session = SessionLocal() + writer_session = SessionLocal() + settlement_task: asyncio.Task[None] | None = None + writer_task: asyncio.Task[Account] | None = None + + def _lock_key(args: tuple[Any, ...], kwargs: dict[str, Any]) -> int: + parameters = args[0] if args else kwargs.get("params") + assert isinstance(parameters, dict) + lock_key = parameters["lock_key"] + assert isinstance(lock_key, int) + return lock_key + + settlement_execute = settlement_session.execute + + async def _settlement_execute(statement: Any, *args: Any, **kwargs: Any): + if "pg_advisory_xact_lock" in str(statement): + settlement_lock_keys.append(_lock_key(args, kwargs)) + return await settlement_execute(statement, *args, **kwargs) + + settlement_commit = settlement_session.commit + + async def _settlement_commit() -> None: + settlement_commit_started.set() + await asyncio.wait_for(release_settlement_commit.wait(), timeout=5.0) + await settlement_commit() + + writer_execute = writer_session.execute + + async def _writer_execute(statement: Any, *args: Any, **kwargs: Any): + if "pg_advisory_xact_lock" in str(statement): + writer_lock_keys.append(_lock_key(args, kwargs)) + writer_lock_attempted.set() + if isinstance(statement, Delete) and statement.table.name == Account.__tablename__: + await asyncio.wait_for(release_writer_delete.wait(), timeout=5.0) + return await writer_execute(statement, *args, **kwargs) + + monkeypatch.setattr(settlement_session, "execute", _settlement_execute) + monkeypatch.setattr(settlement_session, "commit", _settlement_commit) + monkeypatch.setattr(writer_session, "execute", _writer_execute) + + @asynccontextmanager + async def _settlement_session() -> AsyncIterator[AsyncSession]: + yield settlement_session + + monkeypatch.setattr(live_ingest, "get_background_session", _settlement_session) + + try: + settlement_task = asyncio.create_task(ingestor._ingest(queued)) + await asyncio.wait_for(settlement_commit_started.wait(), timeout=5.0) + assert settlement_lock_keys, "settlement must take the upstream identity lock" + + writer_task = asyncio.create_task( + AccountsRepository(writer_session).upsert( + _make_account("acc_live_pg_reauth", email, chatgpt_account_id=upstream_id), + merge_by_email=False, + merge_by_chatgpt_identity=True, + ) + ) + await asyncio.wait_for(writer_lock_attempted.wait(), timeout=5.0) + + assert writer_lock_keys[0] == settlement_lock_keys[0] + release_settlement_commit.set() + await asyncio.wait_for(settlement_task, timeout=5.0) + release_writer_delete.set() + saved = await asyncio.wait_for(writer_task, timeout=5.0) + assert saved.id == canonical_id + finally: + release_settlement_commit.set() + release_writer_delete.set() + for task in (settlement_task, writer_task): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather( + *(task for task in (settlement_task, writer_task) if task is not None), + return_exceptions=True, + ) + await settlement_session.rollback() + await writer_session.rollback() + await settlement_session.close() + await writer_session.close() + + async with SessionLocal() as session: + accounts = list((await session.execute(select(Account).order_by(Account.id))).scalars().all()) + rows = list((await session.execute(select(UsageHistory).order_by(UsageHistory.id))).scalars().all()) + assert [account.id for account in accounts] == [canonical_id] + assert [row.account_id for row in rows] == [canonical_id, canonical_id] + assert {row.window for row in rows} == {"primary", "secondary"} + + +@pytest.mark.asyncio +async def test_postgresql_live_ingest_waits_for_identity_consolidation_commit( + monkeypatch: pytest.MonkeyPatch, + db_setup, +) -> None: + del db_setup + bind = SessionLocal.kw["bind"] + if bind.dialect.name != "postgresql": + pytest.skip("PostgreSQL transaction-lock regression") + + canonical_id = "acc_live_pg_writer_first_canonical" + duplicate_id = "acc_live_pg_writer_first_duplicate" + upstream_id = "workspace-live-pg-writer-first" + email = "live-pg-writer-first@example.com" + async with SessionLocal() as session: + repo = AccountsRepository(session) + await repo.upsert(_make_account(canonical_id, email, chatgpt_account_id=upstream_id), merge_by_email=False) + await repo.upsert(_make_account(duplicate_id, email, chatgpt_account_id=upstream_id), merge_by_email=False) + + ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) + ingestor.publish(_snapshot(), account_id=duplicate_id, chatgpt_account_id=upstream_id) + queued = ingestor._queue.get_nowait() + + writer_commit_started = asyncio.Event() + release_writer_commit = asyncio.Event() + settlement_lock_attempted = asyncio.Event() + writer_session = SessionLocal() + writer_commit = writer_session.commit + real_settlement_lock = usage_repository_module.lock_postgresql_account_identities + writer_task: asyncio.Task[Account] | None = None + settlement_task: asyncio.Task[None] | None = None + + async def _writer_commit() -> None: + writer_commit_started.set() + await asyncio.wait_for(release_writer_commit.wait(), timeout=5.0) + await writer_commit() + + async def _observed_settlement_lock(session: AsyncSession, identities): + settlement_lock_attempted.set() + return await real_settlement_lock(session, identities) + + monkeypatch.setattr(writer_session, "commit", _writer_commit) + monkeypatch.setattr(usage_repository_module, "lock_postgresql_account_identities", _observed_settlement_lock) + + try: + writer_task = asyncio.create_task( + AccountsRepository(writer_session).upsert( + _make_account("acc_live_pg_writer_first_reauth", email, chatgpt_account_id=upstream_id), + merge_by_email=False, + merge_by_chatgpt_identity=True, + ) + ) + await asyncio.wait_for(writer_commit_started.wait(), timeout=5.0) + + settlement_task = asyncio.create_task(ingestor._ingest(queued)) + await asyncio.wait_for(settlement_lock_attempted.wait(), timeout=5.0) + assert not settlement_task.done() + + release_writer_commit.set() + saved = await asyncio.wait_for(writer_task, timeout=5.0) + await asyncio.wait_for(settlement_task, timeout=5.0) + assert saved.id == canonical_id + finally: + release_writer_commit.set() + for task in (writer_task, settlement_task): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather( + *(task for task in (writer_task, settlement_task) if task is not None), + return_exceptions=True, + ) + await writer_session.rollback() + await writer_session.close() + + async with SessionLocal() as session: + accounts = list((await session.execute(select(Account).order_by(Account.id))).scalars().all()) + rows = list((await session.execute(select(UsageHistory).order_by(UsageHistory.id))).scalars().all()) + assert [account.id for account in accounts] == [canonical_id] + assert [row.account_id for row in rows] == [canonical_id, canonical_id] + assert {row.window for row in rows} == {"primary", "secondary"} + + +@pytest.mark.asyncio +async def test_postgresql_live_ingest_recovers_when_current_identity_reconciliation_wins_owner_lock( + monkeypatch: pytest.MonkeyPatch, + db_setup, +) -> None: + del db_setup + bind = SessionLocal.kw["bind"] + if bind.dialect.name != "postgresql": + pytest.skip("PostgreSQL selected-owner lock regression") + + canonical_id = "acc_live_pg_current_identity_canonical" + selected_id = "acc_live_pg_current_identity_selected" + queued_identity = "workspace-live-pg-current-before" + current_identity = "workspace-live-pg-current-after" + email = "live-pg-current-identity@example.com" + async with SessionLocal() as session: + repo = AccountsRepository(session) + await repo.upsert( + _make_account(canonical_id, email, chatgpt_account_id=current_identity), + merge_by_email=False, + ) + selected = await repo.upsert( + _make_account(selected_id, email, chatgpt_account_id=queued_identity), + merge_by_email=False, + ) + + ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) + ingestor.publish( + _snapshot(), + account_id=selected_id, + chatgpt_account_id=queued_identity, + ) + queued = ingestor._queue.get_nowait() + + async with SessionLocal() as session: + moved = await AccountsRepository(session).rotate_tokens( + selected.id, + selected.access_token_encrypted, + selected.refresh_token_encrypted, + selected.id_token_encrypted, + utcnow(), + expected_refresh_token_encrypted=selected.refresh_token_encrypted, + chatgpt_account_id=current_identity, + ) + assert moved is True + + reconciliation_commit_started = asyncio.Event() + release_reconciliation_commit = asyncio.Event() + settlement_local_lookup_started = asyncio.Event() + settlement_session = SessionLocal() + reconciliation_session = SessionLocal() + settlement_task: asyncio.Task[None] | None = None + reconciliation_task: asyncio.Task[Account] | None = None + settlement_execute = settlement_session.execute + reconciliation_commit = reconciliation_session.commit + + async def _settlement_execute(statement: Any, *args: Any, **kwargs: Any): + sql = str(statement) + if sql.startswith("SELECT accounts.id, accounts.chatgpt_account_id") and "WHERE accounts.id =" in sql: + settlement_local_lookup_started.set() + return await settlement_execute(statement, *args, **kwargs) + + async def _reconciliation_commit() -> None: + reconciliation_commit_started.set() + await asyncio.wait_for(release_reconciliation_commit.wait(), timeout=5.0) + await reconciliation_commit() + + monkeypatch.setattr(settlement_session, "execute", _settlement_execute) + monkeypatch.setattr(reconciliation_session, "commit", _reconciliation_commit) + + @asynccontextmanager + async def _settlement_session() -> AsyncIterator[AsyncSession]: + yield settlement_session + + monkeypatch.setattr(live_ingest, "get_background_session", _settlement_session) + + try: + reconciliation_task = asyncio.create_task( + AccountsRepository(reconciliation_session).upsert( + _make_account("acc_live_pg_current_identity_reauth", email, chatgpt_account_id=current_identity), + merge_by_email=False, + merge_by_chatgpt_identity=True, + ) + ) + await asyncio.wait_for(reconciliation_commit_started.wait(), timeout=5.0) + + settlement_task = asyncio.create_task(ingestor._ingest(queued)) + await asyncio.wait_for(settlement_local_lookup_started.wait(), timeout=5.0) + assert not settlement_task.done() + + release_reconciliation_commit.set() + saved = await asyncio.wait_for(reconciliation_task, timeout=5.0) + await asyncio.wait_for(settlement_task, timeout=5.0) + assert saved.id == canonical_id + finally: + release_reconciliation_commit.set() + for task in (settlement_task, reconciliation_task): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather( + *(task for task in (settlement_task, reconciliation_task) if task is not None), + return_exceptions=True, + ) + await settlement_session.rollback() + await reconciliation_session.rollback() + await settlement_session.close() + await reconciliation_session.close() + + async with SessionLocal() as session: + accounts = list((await session.execute(select(Account).order_by(Account.id))).scalars().all()) + rows = list((await session.execute(select(UsageHistory).order_by(UsageHistory.id))).scalars().all()) + assert [account.id for account in accounts] == [canonical_id] + assert [account.chatgpt_account_id for account in accounts] == [current_identity] + assert [row.account_id for row in rows] == [canonical_id, canonical_id] + assert {row.window for row in rows} == {"primary", "secondary"} + assert {row.used_percent for row in rows} == {33.0, 44.0} + + +@pytest.mark.asyncio +async def test_postgresql_opposite_identity_moves_use_one_sorted_lock_order( + monkeypatch: pytest.MonkeyPatch, + db_setup, +) -> None: + del db_setup + bind = SessionLocal.kw["bind"] + if bind.dialect.name != "postgresql": + pytest.skip("PostgreSQL transaction-lock regression") + + first = _make_account("acc_identity_move_a", "identity-move-a@example.com", chatgpt_account_id="workspace-a") + second = _make_account("acc_identity_move_b", "identity-move-b@example.com", chatgpt_account_id="workspace-b") + async with SessionLocal() as session: + repo = AccountsRepository(session) + await repo.upsert(first, merge_by_email=False) + await repo.upsert(second, merge_by_email=False) + + real_identity_lock = accounts_repository_module.lock_postgresql_account_identities + arrival_guard = asyncio.Lock() + both_arrived = asyncio.Event() + arrival_count = 0 + + async def _synchronized_identity_lock(session: AsyncSession, identities): + nonlocal arrival_count + async with arrival_guard: + arrival_count += 1 + if arrival_count == 2: + both_arrived.set() + await asyncio.wait_for(both_arrived.wait(), timeout=5.0) + return await real_identity_lock(session, identities) + + monkeypatch.setattr(accounts_repository_module, "lock_postgresql_account_identities", _synchronized_identity_lock) + + async def _move(account: Account, incoming_identity: str) -> bool: + async with SessionLocal() as session: + return await AccountsRepository(session).rotate_tokens( + account.id, + account.access_token_encrypted, + account.refresh_token_encrypted, + account.id_token_encrypted, + utcnow(), + expected_refresh_token_encrypted=account.refresh_token_encrypted, + chatgpt_account_id=incoming_identity, + ) + + moved_first, moved_second = await asyncio.wait_for( + asyncio.gather(_move(first, "workspace-b"), _move(second, "workspace-a")), + timeout=5.0, + ) + assert moved_first is True + assert moved_second is True + + async with SessionLocal() as session: + identities = { + account_id: chatgpt_account_id + for account_id, chatgpt_account_id in ( + await session.execute(select(Account.id, Account.chatgpt_account_id)) + ).all() + } + assert identities == { + first.id: "workspace-b", + second.id: "workspace-a", + } + + +@pytest.mark.asyncio +async def test_live_ingestor_prefers_valid_local_owner_over_upstream_fallback(db_setup) -> None: + del db_setup + local_id = "acc_live_valid_local" + sibling_id = "acc_live_valid_local_sibling" + upstream_id = "workspace-live-shared" + async with SessionLocal() as session: + repo = AccountsRepository(session) + await repo.upsert( + _make_account(local_id, "live-valid-local@example.com", chatgpt_account_id=upstream_id), + merge_by_email=False, + ) + await repo.upsert( + _make_account(sibling_id, "live-valid-sibling@example.com", chatgpt_account_id=upstream_id), + merge_by_email=False, + ) + + snapshot = _snapshot() + ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) + ingestor.publish(snapshot, account_id=local_id, chatgpt_account_id=upstream_id) + queued = ingestor._queue.get_nowait() + + await ingestor._ingest(queued) + + rows = await _usage_rows_for(local_id, sibling_id) + assert len(rows) == 2 + assert {row.account_id for row in rows} == {local_id} + assert {row.window for row in rows} == {"primary", "secondary"} + + +@pytest.mark.asyncio +async def test_live_ingestor_resolves_chatgpt_account_id(db_setup) -> None: + del db_setup + account_id = "acc_live_resolved" + async with SessionLocal() as session: + await AccountsRepository(session).upsert( + _make_account(account_id, "live-resolved@example.com", chatgpt_account_id="workspace-live-1") + ) + + ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) + ingestor.publish(_snapshot(), chatgpt_account_id="workspace-live-1") + queued = ingestor._queue.get_nowait() + + await ingestor._ingest(queued) + + rows = await _usage_rows_for(account_id) + assert len(rows) == 2 + assert {row.account_id for row in rows} == {account_id} + assert {row.window for row in rows} == {"primary", "secondary"} @pytest.mark.asyncio @@ -268,3 +856,52 @@ async def test_live_ingestion_kill_switch_disables_publishing(monkeypatch, db_se finally: await live_ingest.stop_live_usage_ingestor() get_settings.cache_clear() + + +@pytest.mark.asyncio +async def test_nested_lifespan_stop_does_not_orphan_or_kill_the_outer_ingestor(db_setup) -> None: + del db_setup + + # Two app lifespans can be live in one process: the suite's async_client + # runs one on the session loop while a test opens a TestClient whose + # portal runs another. Each lifespan owns the instance start returned and + # stops exactly that instance. Before instance-scoped stop, the nested + # startup overwrote the module global, orphaned the outer ingestor as an + # unreferenced cycle, and the cyclic GC destroyed its consumer mid-await + # ("cannot reuse already awaited coroutine" — issue #1755's integration + # signature); the nested shutdown then cleared the global so the outer + # shutdown stopped nothing. And a nested shutdown that merely cleared the + # registration would leave the still-running outer ingestor deaf: it must + # instead restore the outer instance as the current registration. + def _pending_consumers() -> list[asyncio.Task[object]]: + return [t for t in asyncio.all_tasks() if not t.done() and t.get_name() == "live-usage-ingestor"] + + async with SessionLocal() as session: + await AccountsRepository(session).upsert(_make_account("acc_live_nested", "live-nested@example.com")) + + outer = live_ingest.start_live_usage_ingestor() + assert outer is not None + inner = live_ingest.start_live_usage_ingestor() + assert inner is not None and inner is not outer + assert live_ingest._ingestor is inner + + # Nested lifespan shutdown: releases the registration it owns and + # restores the still-running outer instance in its place. + await live_ingest.stop_live_usage_ingestor(inner) + assert live_ingest._ingestor is outer + assert live_ingest._displaced_ingestors == [] + assert outer._consumer is not None and not outer._consumer.done() + + # Outer ingestion RESUMES: a hub publication after the nested exit must + # flow to the outer instance and be ingested end to end. + live_hub.publish_live_usage(_snapshot(), account_id="acc_live_nested") + primary, secondary = await _wait_for_rows("acc_live_nested") + assert primary is not None and secondary is not None + assert primary.used_percent == pytest.approx(33.0) + + # Outer lifespan shutdown: stops its own instance, clears the restored + # registration, and no consumer survives for the suite fence. + await live_ingest.stop_live_usage_ingestor(outer) + assert live_ingest._ingestor is None + assert live_hub._publisher is None + assert _pending_consumers() == [] diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index 35b6bb2a4e..11a9b11345 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -1743,6 +1743,89 @@ async def _usage_history_reloptions() -> set[str]: assert expected_options <= await _usage_history_reloptions() +@pytest.mark.asyncio +async def test_account_pending_deletion_migration_upgrade_and_downgrade(tmp_path): + """Round-trip the pending-deletion marker migration through Alembic: + parent -> revision adds the two guarded marker columns and the partial + queue index, downgrade removes all three, the guarded upgrade tolerates + pre-existing columns, and an upgrade to head proves the revision sits on + the single-head path.""" + from alembic import command + from sqlalchemy import inspect as sa_inspect + + from app.db.migrate import _build_alembic_config + + db_url = f"sqlite+aiosqlite:///{tmp_path / 'account-pending-deletion.sqlite'}" + parent_revision = "20260812_120000_add_sticky_abandonment_scope" + pending_deletion_revision = "20260816_000000_add_account_pending_deletion" + marker_columns = {"delete_requested_at", "delete_history_requested"} + index_name = "idx_accounts_delete_requested_at" + + def _schema_state(sync_conn): + inspector = sa_inspect(sync_conn) + columns = {column["name"] for column in inspector.get_columns("accounts")} + indexes = {index["name"] for index in inspector.get_indexes("accounts")} + return {"columns": columns & marker_columns, "index_present": index_name in indexes} + + await to_thread.run_sync(lambda: run_upgrade(db_url, parent_revision, bootstrap_legacy=False)) + engine = create_async_engine(db_url, future=True) + try: + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state == {"columns": set(), "index_present": False} + + await to_thread.run_sync(lambda: run_upgrade(db_url, pending_deletion_revision, bootstrap_legacy=False)) + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state == {"columns": marker_columns, "index_present": True} + + # Downgrade refuses while a deletion is queued: the marker columns are + # the queue's only durable state, and dropping them would silently + # abandon an acknowledged deletion. + async with engine.begin() as conn: + await conn.execute( + text( + "INSERT INTO accounts (id, codex_installation_id, email, plan_type, " + "access_token_encrypted, refresh_token_encrypted, id_token_encrypted, " + "last_refresh, status, delete_requested_at, delete_history_requested) " + "VALUES ('acc_mig_pending', 'install-mig-pending', 'mig@example.com', 'plus', " + "X'00', X'00', X'00', '2026-08-16 00:00:00', 'deactivated', " + "'2026-08-16 00:00:00', 0)" + ) + ) + with pytest.raises(Exception, match="queued for"): + await to_thread.run_sync(lambda: command.downgrade(_build_alembic_config(db_url), parent_revision)) + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state == {"columns": marker_columns, "index_present": True} + async with engine.begin() as conn: + await conn.execute(text("DELETE FROM accounts WHERE id = 'acc_mig_pending'")) + + await to_thread.run_sync(lambda: command.downgrade(_build_alembic_config(db_url), parent_revision)) + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state == {"columns": set(), "index_present": False} + + # Guarded upgrade: a database where the columns already exist (e.g. a + # pre-merge build of this revision) must upgrade cleanly and still + # create the missing index. + async with engine.begin() as conn: + await conn.execute(text("ALTER TABLE accounts ADD COLUMN delete_requested_at DATETIME")) + await to_thread.run_sync(lambda: run_upgrade(db_url, pending_deletion_revision, bootstrap_legacy=False)) + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state == {"columns": marker_columns, "index_present": True} + + # Single-head path: upgrading to head from here must succeed and keep + # the marker schema in place. + await to_thread.run_sync(lambda: run_upgrade(db_url, "head", bootstrap_legacy=False)) + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state == {"columns": marker_columns, "index_present": True} + finally: + await engine.dispose() + + @pytest.mark.asyncio async def test_account_plan_downgrade_observations_migration_upgrade_and_downgrade(tmp_path): """Round-trip the plan-downgrade evidence migration through Alembic itself. @@ -2164,3 +2247,55 @@ def _schema_state(sync_conn): assert "conversation_folded_through" in state["state_columns"] finally: await engine.dispose() + + +@pytest.mark.asyncio +async def test_file_account_pins_migration_upgrade_and_downgrade(tmp_path): + from alembic import command + from sqlalchemy import inspect as sa_inspect + + from app.db.migrate import _build_alembic_config + + db_url = f"sqlite+aiosqlite:///{tmp_path / 'file-account-pins.sqlite'}" + parent_revision = "20260806_000000_add_anonymous_telemetry" + pin_revision = "20260813_000000_add_file_account_pins" + + def _schema_state(sync_conn): + inspector = sa_inspect(sync_conn) + if not inspector.has_table("file_account_pins"): + return None + columns = inspector.get_columns("file_account_pins") + file_id_column = next(column for column in columns if column["name"] == "file_id") + return { + "columns": {column["name"] for column in columns}, + "file_id_length": file_id_column["type"].length, + "primary_key": inspector.get_pk_constraint("file_account_pins")["constrained_columns"], + "indexes": {index["name"] for index in inspector.get_indexes("file_account_pins")}, + } + + await to_thread.run_sync(lambda: run_upgrade(db_url, parent_revision, bootstrap_legacy=False)) + engine = create_async_engine(db_url, future=True) + try: + async with engine.connect() as conn: + assert await conn.run_sync(_schema_state) is None + + await to_thread.run_sync(lambda: run_upgrade(db_url, pin_revision, bootstrap_legacy=False)) + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state == { + "columns": {"file_id", "account_id", "expires_at"}, + "file_id_length": None, + "primary_key": ["file_id"], + "indexes": {"ix_file_account_pins_expires_at"}, + } + + await to_thread.run_sync(lambda: command.downgrade(_build_alembic_config(db_url), parent_revision)) + async with engine.connect() as conn: + assert await conn.run_sync(_schema_state) is None + + result = await to_thread.run_sync(lambda: run_upgrade(db_url, "head", bootstrap_legacy=False)) + assert result.current_revision == _HEAD_REVISION + async with engine.connect() as conn: + assert await conn.run_sync(_schema_state) is not None + finally: + await engine.dispose() diff --git a/tests/integration/test_model_source_routing.py b/tests/integration/test_model_source_routing.py index caf86b630e..23e3aec80d 100644 --- a/tests/integration/test_model_source_routing.py +++ b/tests/integration/test_model_source_routing.py @@ -41,6 +41,7 @@ async def _create_model_source( supports_responses: bool = False, supports_streaming: bool = True, supports_audio_transcriptions: bool = False, + supports_embeddings: bool = False, ) -> str: model_entry: dict[str, object] = { "model": model, @@ -69,6 +70,7 @@ async def _create_model_source( "supportsChatCompletions": True, "supportsResponses": supports_responses, "supportsAudioTranscriptions": supports_audio_transcriptions, + "supportsEmbeddings": supports_embeddings, "models": [model_entry], }, ) @@ -526,6 +528,81 @@ async def test_responses_source_selector_can_require_streaming(async_client): assert streaming is None +@pytest.mark.asyncio +async def test_responses_model_is_source_owned_detects_streaming_source(async_client): + from app.modules.model_sources.selection import responses_model_is_source_owned + + model = "ws-guard-streaming-model" + await _create_model_source( + async_client, + name="ws-guard-streaming", + model=model, + base_url="http://127.0.0.1:9/v1", + supports_responses=True, + supports_streaming=True, + ) + + assert await responses_model_is_source_owned(model, None) is True + # A subscription model must stay on the WebSocket path. + assert await responses_model_is_source_owned("gpt-5.6-sol", None) is False + assert await responses_model_is_source_owned(None, None) is False + + +@pytest.mark.asyncio +async def test_responses_model_is_source_owned_requires_streaming(async_client): + """The guard mirrors the HTTP selector, which requires a streaming source.""" + from app.modules.model_sources.selection import responses_model_is_source_owned + + model = "ws-guard-non-streaming-model" + await _create_model_source( + async_client, + name="ws-guard-non-streaming", + model=model, + base_url="http://127.0.0.1:9/v1", + supports_responses=True, + supports_streaming=False, + ) + + assert await responses_model_is_source_owned(model, None) is False + + +@pytest.mark.asyncio +async def test_responses_model_is_source_owned_honors_enforced_model(async_client): + """An API key that forces a source-owned model must also be caught. + + The HTTP handlers build their candidate list from the enforced model as + well as the requested one; the WebSocket guard has to match or an enforced + source model would fall through to subscription-account selection. + """ + from app.modules.model_sources.selection import responses_model_is_source_owned + + model = "ws-guard-enforced-model" + await _create_model_source( + async_client, + name="ws-guard-enforced", + model=model, + base_url="http://127.0.0.1:9/v1", + supports_responses=True, + supports_streaming=True, + ) + enforcing_key = ApiKeyData( + id="key_ws_guard_enforced", + name="ws guard enforced", + key_prefix="sk-test-ws-enforced", + allowed_models=[], + enforced_model=model, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + + # The client asked for a subscription model, but the key forces the source. + assert await responses_model_is_source_owned("gpt-5.6-sol", enforcing_key) is True + + @pytest.mark.asyncio async def test_responses_source_raw_alias_lookup_requires_exact_allowlist(async_client): import app.modules.proxy.api as proxy_api @@ -583,6 +660,47 @@ async def test_responses_source_raw_alias_lookup_requires_exact_allowlist(async_ assert selected_model == model +@pytest.mark.asyncio +async def test_responses_model_is_source_owned_prefers_the_raw_alias(async_client): + """WebSocket parity for the raw-alias candidate (see the HTTP test above). + + Request preparation normalizes ``gpt-5-high`` to ``gpt-5`` before the + WebSocket guards run, so the guard helper must accept the client's raw + model and offer it to source selection ahead of the normalized one — the + HTTP path routes the identical request via ``raw_source_model``. + """ + from app.modules.model_sources.selection import responses_model_is_source_owned + + model = "gpt-5-high" + await _create_model_source( + async_client, + name="ws-guard-raw-alias", + model=model, + base_url="http://127.0.0.1:9/v1", + supports_responses=True, + supports_streaming=True, + ) + exact_key = ApiKeyData( + id="key_ws_raw_alias", + name="ws raw alias", + key_prefix="sk-test-ws-raw-alias", + allowed_models=[model], + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + + assert await responses_model_is_source_owned("gpt-5", exact_key, raw_model=model) is True + # Without the raw candidate the exact allowlist hides the source. This is + # the pre-fix WebSocket behaviour; keeping it false proves the assertion + # above matched through the raw candidate, not some other fallback. + assert await responses_model_is_source_owned("gpt-5", exact_key) is False + + @pytest.mark.asyncio async def test_chat_source_selector_can_require_streaming(async_client): from app.modules.model_sources.repository import ModelSourcesRepository @@ -945,7 +1063,7 @@ async def cancelled_stream() -> AsyncIterator[bytes]: @pytest.mark.asyncio -async def test_downstream_disconnect_closes_source_stream(async_client, monkeypatch): +async def test_cancelled_buffered_stream_releases_reservation_when_close_fails(async_client, monkeypatch): from starlette.requests import Request import app.modules.proxy.api as proxy_api @@ -953,24 +1071,19 @@ async def test_downstream_disconnect_closes_source_stream(async_client, monkeypa from app.modules.model_sources.forwarding import SourceUsageHolder released: list[object] = [] - stream_closed = False async def record_release(reservation: object) -> None: released.append(reservation) - async def skip_log(*args, **kwargs) -> None: - del args, kwargs + async def fail_close(_stream: object) -> None: + raise RuntimeError("close failed") monkeypatch.setattr(proxy_api, "_release_reservation", record_release) - monkeypatch.setattr(proxy_api, "_log_source_chat_completion", skip_log) + monkeypatch.setattr(proxy_api, "_aclose_stream", fail_close) - async def source_stream() -> AsyncIterator[bytes]: - nonlocal stream_closed - try: - yield b"data: partial\n\n" - await asyncio.sleep(60) - finally: - stream_closed = True + async def cancelled_stream() -> AsyncIterator[bytes]: + yield b"data: partial\n\n" + raise asyncio.CancelledError() request = Request( { @@ -983,8 +1096,8 @@ async def source_stream() -> AsyncIterator[bytes]: } ) source = ModelSource( - id="src_disconnect", - name="disconnect", + id="src_cancelled_close_fails", + name="cancelled-close-fails", kind="openai_compatible", base_url="http://127.0.0.1:9/v1", is_enabled=True, @@ -992,53 +1105,53 @@ async def source_stream() -> AsyncIterator[bytes]: supports_responses=False, ) reservation = ApiKeyUsageReservationData( - reservation_id="resv_disconnect", - key_id="key_disconnect", - model="disconnect-model", + reservation_id="resv_cancelled_close_fails", + key_id="key_cancelled_close_fails", + model="cancelled-model", ) - response_stream = cast( - AsyncGenerator[bytes, None], - proxy_api._source_chat_stream_with_settlement( - source_stream(), - usage_holder=SourceUsageHolder(), - request=request, + + with pytest.raises(RuntimeError, match="close failed"): + await proxy_api._buffered_limited_source_chat_stream_response( + request, source=source, api_key=None, - model="disconnect-model", + model="cancelled-model", reservation=reservation, - ), - ) - - assert await anext(response_stream) == b"data: partial\n\n" - await response_stream.aclose() + stream=cancelled_stream(), + usage_holder=SourceUsageHolder(), + rate_limit_headers={}, + ) assert released == [reservation] - assert stream_closed is True @pytest.mark.asyncio -async def test_source_stream_disconnect_logs_cancelled_not_error(async_client, db_setup, monkeypatch): - """Regression for #1552: a downstream disconnect mid-stream on a - model-source route is a normal client-side terminal — recorded as - status=cancelled (like the main proxy path), counted in cancelled_count, - and excluded from the error rate and top_error.""" - from datetime import timedelta - +async def test_cancelled_buffered_stream_finishes_usage_settlement(async_client, monkeypatch): from starlette.requests import Request import app.modules.proxy.api as proxy_api from app.db.models import ModelSource - from app.modules.model_sources.forwarding import SourceUsageHolder - from app.modules.request_logs.repository import RequestLogsRepository + from app.modules.model_sources.forwarding import SourceUsage, SourceUsageHolder - async def record_release(reservation: object) -> None: - del reservation + settlement_started = asyncio.Event() + settlement_can_finish = asyncio.Event() + settled: list[object] = [] + logs: list[dict[str, object]] = [] - monkeypatch.setattr(proxy_api, "_release_reservation", record_release) + async def settle(reservation: object, **_kwargs: object) -> bool: + settlement_started.set() + await settlement_can_finish.wait() + settled.append(reservation) + return True - async def source_stream() -> AsyncIterator[bytes]: - yield b"data: partial\n\n" - await asyncio.sleep(60) + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(kwargs) + + monkeypatch.setattr(proxy_api, "_settle_source_reservation", settle) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) + + async def complete_stream() -> AsyncIterator[bytes]: + yield b"data: done\n\n" request = Request( { @@ -1051,520 +1164,1936 @@ async def source_stream() -> AsyncIterator[bytes]: } ) source = ModelSource( - id="src_cx_log", - name="cx-log", + id="src_settlement_cancelled", + name="settlement-cancelled", kind="openai_compatible", base_url="http://127.0.0.1:9/v1", is_enabled=True, supports_chat_completions=True, supports_responses=False, ) - response_stream = cast( - AsyncGenerator[bytes, None], - proxy_api._source_chat_stream_with_settlement( - source_stream(), - usage_holder=SourceUsageHolder(), - request=request, + reservation = ApiKeyUsageReservationData( + reservation_id="resv_settlement_cancelled", + key_id="key_settlement_cancelled", + model="cancelled-model", + ) + usage_holder = SourceUsageHolder(usage=SourceUsage(input_tokens=3, output_tokens=5)) + + task = asyncio.create_task( + proxy_api._buffered_limited_source_chat_stream_response( + request, source=source, api_key=None, - model="cx-log-model", - reservation=None, - ), + model="cancelled-model", + reservation=reservation, + stream=complete_stream(), + usage_holder=usage_holder, + rate_limit_headers={}, + ) ) + await asyncio.wait_for(settlement_started.wait(), timeout=1) + task.cancel() + settlement_can_finish.set() - assert await anext(response_stream) == b"data: partial\n\n" - await response_stream.aclose() - - async with SessionLocal() as session: - row = (await session.execute(select(RequestLog).where(RequestLog.model_source_id == "src_cx_log"))).scalar_one() - assert row.status == "cancelled" - assert row.error_code == "client_disconnected" + with pytest.raises(asyncio.CancelledError): + await task - # The status classification is what every metric surface keys on: - # the disconnect must not join the error numerator or top_error. - aggregate = await RequestLogsRepository(session).aggregate_usage_metrics_since(utcnow() - timedelta(minutes=5)) - assert aggregate.request_count == 1 - assert aggregate.error_count == 0 - assert aggregate.cancelled_count == 1 - assert aggregate.top_error is None + assert settled == [reservation] + assert logs[-1]["status"] == "cancelled" + assert logs[-1]["error_code"] == "client_disconnected" + assert logs[-1]["usage"] == usage_holder.usage @pytest.mark.asyncio -async def test_opportunistic_key_routes_to_source_without_account_pool(async_client, source_upstream): - await _enable_api_key_auth(async_client) - - async def completion(_request: web.Request) -> web.Response: - return web.json_response( - { - "id": "chatcmpl_opportunistic", - "object": "chat.completion", - "created": 1, - "model": "opportunistic-model", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, - } - ) +async def test_cancelled_buffered_stream_logs_disconnect(async_client, monkeypatch): + from starlette.requests import Request - base_url = await source_upstream(completion) - model = "opportunistic-model" - source_id = await _create_model_source(async_client, name="opportunistic", model=model, base_url=base_url) - created = await async_client.post( - "/api/api-keys/", - json={ - "name": "opportunistic-source-key", - "assignedSourceIds": [source_id], - "trafficClass": "opportunistic", - }, - ) - assert created.status_code == 200 - key = created.json()["key"] + import app.modules.proxy.api as proxy_api + from app.db.models import ModelSource + from app.modules.model_sources.forwarding import SourceUsageHolder - # No subscription accounts exist, so opportunistic admission would deny - # with 429 if it (incorrectly) gated the account-free source path. - response = await async_client.post( - "/v1/chat/completions", - headers={"Authorization": f"Bearer {key}"}, - json={"model": model, "messages": [{"role": "user", "content": "hi"}]}, - ) + released: list[object] = [] + logs: list[dict[str, object]] = [] - assert response.status_code == 200 - assert response.json()["id"] == "chatcmpl_opportunistic" + async def record_release(reservation: object) -> None: + released.append(reservation) + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(kwargs) -@pytest.mark.asyncio -async def test_source_credential_decrypt_failure_maps_to_error_and_releases_reservation( - async_client, source_upstream, monkeypatch -): - await _enable_api_key_auth(async_client) + monkeypatch.setattr(proxy_api, "_release_reservation", record_release) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) - async def completion(_request: web.Request) -> web.Response: - return web.json_response({"unreachable": True}) + async def cancelled_stream() -> AsyncIterator[bytes]: + yield b"data: partial\n\n" + raise asyncio.CancelledError() - base_url = await source_upstream(completion) - model = "credential-fail-model" - source_id = await _create_model_source(async_client, name="credential-fail", model=model, base_url=base_url) - created = await async_client.post( - "/api/api-keys/", - json={ - "name": "credential-fail-key", - "assignedSourceIds": [source_id], - "limits": [ - {"limitType": "total_tokens", "limitWindow": "weekly", "maxValue": 1_000}, - ], - }, + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } ) - assert created.status_code == 200 - key = created.json()["key"] - - from app.core.crypto import TokenEncryptor - - def broken_decrypt(self, value): - raise ValueError("decryption boom") - - monkeypatch.setattr(TokenEncryptor, "decrypt", broken_decrypt) - - response = await async_client.post( - "/v1/chat/completions", - headers={"Authorization": f"Bearer {key}"}, - json={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + source = ModelSource( + id="src_buffered_cancelled_log", + name="buffered-cancelled-log", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_buffered_cancelled_log", + key_id="key_buffered_cancelled_log", + model="cancelled-model", ) - assert response.status_code == 502 - assert response.json()["error"]["code"] == "model_source_credentials_error" + with pytest.raises(asyncio.CancelledError): + await proxy_api._buffered_limited_source_chat_stream_response( + request, + source=source, + api_key=None, + model="cancelled-model", + reservation=reservation, + stream=cancelled_stream(), + usage_holder=SourceUsageHolder(), + rate_limit_headers={}, + ) + + assert released == [reservation] + assert logs[-1]["status"] == "cancelled" + assert logs[-1]["error_code"] == "client_disconnected" + assert logs[-1]["error_message"] == "client disconnected during source stream buffering" + + +@pytest.mark.asyncio +async def test_source_completion_success_log_finishes_after_cancellation(async_client, monkeypatch): + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.core.openai.chat_requests import ChatCompletionsRequest + from app.db.models import ModelSource + from app.modules.model_sources.forwarding import SourceChatCompletion, SourceUsage + + log_started = asyncio.Event() + allow_log_finish = asyncio.Event() + logs: list[dict[str, object]] = [] + + async def fake_forward(*_args: object, **_kwargs: object) -> SourceChatCompletion: + return SourceChatCompletion( + payload={"id": "chatcmpl_cancelled_after_settlement"}, + usage=SourceUsage(input_tokens=3, output_tokens=5), + timings=None, + upstream_status_code=200, + ) + + async def settle(*_args: object, **_kwargs: object) -> bool: + return True + + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(kwargs) + log_started.set() + await allow_log_finish.wait() + + monkeypatch.setattr(proxy_api, "forward_chat_completion", fake_forward) + monkeypatch.setattr(proxy_api, "_settle_source_reservation", settle) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } + ) + source = ModelSource( + id="src_completion_cancelled_log", + name="completion-cancelled-log", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + payload = ChatCompletionsRequest.model_validate( + { + "model": "completion-cancelled-log", + "messages": [{"role": "user", "content": "hello"}], + "stream": False, + } + ) + + task = asyncio.create_task( + proxy_api._source_chat_completion_response( + request, + payload, + source=source, + model="completion-cancelled-log", + api_key=None, + reservation=None, + rate_limit_headers={}, + ) + ) + await asyncio.wait_for(log_started.wait(), timeout=1) + task.cancel() + allow_log_finish.set() + + with pytest.raises(asyncio.CancelledError): + await task + + assert logs[-1]["status"] == "success" + assert logs[-1]["usage"] == SourceUsage(input_tokens=3, output_tokens=5) + + +@pytest.mark.asyncio +async def test_source_stream_setup_cancellation_logs_visible_error_even_if_release_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.core.openai.chat_requests import ChatCompletionsRequest + from app.db.models import ModelSource + + logs: list[dict[str, object]] = [] + release_attempts: list[object] = [] + + async def cancel_during_open(*_args: object, **_kwargs: object) -> object: + raise asyncio.CancelledError + + async def fail_release(reservation: object) -> None: + release_attempts.append(reservation) + raise RuntimeError("sqlite busy") + + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(dict(kwargs)) + + monkeypatch.setattr(proxy_api, "stream_source_chat_completion", cancel_during_open) + monkeypatch.setattr(proxy_api, "_release_reservation_deferring_cancellation", fail_release) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } + ) + source = ModelSource( + id="src_stream_setup_cancel", + name="stream-setup-cancel", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_stream_setup_cancel", + key_id="key_stream_setup_cancel", + model="stream-setup-cancel", + ) + payload = ChatCompletionsRequest.model_validate( + { + "model": "stream-setup-cancel", + "messages": [{"role": "user", "content": "hello"}], + "stream": True, + } + ) + + with pytest.raises(asyncio.CancelledError): + await proxy_api._source_chat_completion_response( + request, + payload, + source=source, + model="stream-setup-cancel", + api_key=None, + reservation=reservation, + rate_limit_headers={}, + ) + + assert release_attempts == [reservation] + assert logs == [ + { + "source": source, + "api_key": None, + "model": "stream-setup-cancel", + "status": "cancelled", + "error_code": "client_disconnected", + "error_message": "client disconnected during source stream setup", + } + ] + + +@pytest.mark.asyncio +async def test_source_request_setup_cancellation_logs_disconnect_even_if_release_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.core.openai.chat_requests import ChatCompletionsRequest + from app.db.models import ModelSource + + logs: list[dict[str, object]] = [] + release_attempts: list[object] = [] + + async def cancel_during_forward(*_args: object, **_kwargs: object) -> object: + raise asyncio.CancelledError + + async def fail_release(reservation: object) -> None: + release_attempts.append(reservation) + raise RuntimeError("sqlite busy") + + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(dict(kwargs)) + + monkeypatch.setattr(proxy_api, "forward_chat_completion", cancel_during_forward) + monkeypatch.setattr(proxy_api, "_release_reservation_deferring_cancellation", fail_release) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } + ) + source = ModelSource( + id="src_request_setup_cancel_release_fail", + name="request-setup-cancel-release-fail", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_request_setup_cancel_release_fail", + key_id="key_request_setup_cancel_release_fail", + model="request-setup-cancel-release-fail", + ) + payload = ChatCompletionsRequest.model_validate( + { + "model": "request-setup-cancel-release-fail", + "messages": [{"role": "user", "content": "hello"}], + "stream": False, + } + ) + + with pytest.raises(asyncio.CancelledError): + await proxy_api._source_chat_completion_response( + request, + payload, + source=source, + model="request-setup-cancel-release-fail", + api_key=None, + reservation=reservation, + rate_limit_headers={}, + ) + + assert release_attempts == [reservation] + assert logs == [ + { + "source": source, + "api_key": None, + "model": "request-setup-cancel-release-fail", + "status": "cancelled", + "error_code": "client_disconnected", + "error_message": "client disconnected during source request setup", + } + ] + + +@pytest.mark.asyncio +async def test_buffered_stream_cancellation_logs_disconnect_even_if_release_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.db.models import ModelSource + from app.modules.model_sources.forwarding import SourceUsageHolder + + logs: list[dict[str, object]] = [] + release_attempts: list[object] = [] + + async def fail_release(reservation: object) -> None: + release_attempts.append(reservation) + raise RuntimeError("sqlite busy") + + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(dict(kwargs)) + + monkeypatch.setattr(proxy_api, "_release_reservation_deferring_cancellation", fail_release) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) + + async def cancelled_stream() -> AsyncIterator[bytes]: + yield b"data: partial\n\n" + raise asyncio.CancelledError() + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } + ) + source = ModelSource( + id="src_buffered_cancel_release_fail", + name="buffered-cancel-release-fail", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_buffered_cancel_release_fail", + key_id="key_buffered_cancel_release_fail", + model="buffered-cancel-release-fail", + ) + + with pytest.raises(asyncio.CancelledError): + await proxy_api._buffered_limited_source_chat_stream_response( + request, + source=source, + api_key=None, + model="buffered-cancel-release-fail", + reservation=reservation, + stream=cancelled_stream(), + usage_holder=SourceUsageHolder(), + rate_limit_headers={}, + ) + + assert release_attempts == [reservation] + assert logs[-1]["status"] == "cancelled" + assert logs[-1]["error_code"] == "client_disconnected" + assert logs[-1]["error_message"] == "client disconnected during source stream buffering" + + +@pytest.mark.asyncio +async def test_source_stream_body_teardown_survives_repeated_cancellation(monkeypatch: pytest.MonkeyPatch): + from contextlib import AsyncExitStack + + import app.modules.model_sources.forwarding as forwarding_module + from app.db.models import ModelSource + + stream_blocked = asyncio.Event() + release_started = asyncio.Event() + allow_release = asyncio.Event() + release_finished = asyncio.Event() + + class _SlowLease: + async def __aenter__(self) -> None: + return None + + async def __aexit__(self, exc_type: object, exc: object, tb: object) -> bool: + release_started.set() + await allow_release.wait() + release_finished.set() + return False + + stack = AsyncExitStack() + await stack.enter_async_context(_SlowLease()) + + class _FakeContent: + def iter_chunked(self, _size: int) -> AsyncIterator[bytes]: + async def gen() -> AsyncIterator[bytes]: + yield b"data: chunk\n\n" + stream_blocked.set() + await asyncio.Event().wait() + + return gen() + + class _FakeResponse: + status = 200 + content = _FakeContent() + + async def fake_open(*_args: object, **_kwargs: object) -> object: + return stack, _FakeResponse() + + monkeypatch.setattr(forwarding_module, "_open_source_stream", fake_open) + + source = ModelSource( + id="src_body_teardown_repeated_cancel", + name="body-teardown-repeated-cancel", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + stream = await forwarding_module.stream_chat_completion(source, {"model": "body-teardown"}) + + async def consume() -> None: + async for _chunk in stream.body: + pass + + task = asyncio.create_task(consume()) + await asyncio.wait_for(stream_blocked.wait(), timeout=1) + await asyncio.sleep(0) + + task.cancel() + await asyncio.wait_for(release_started.wait(), timeout=1) + # Second cancellation delivery while the exit stack is unwinding: teardown + # must still return the pooled HTTP lease. + task.cancel() + await asyncio.sleep(0) + allow_release.set() + + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=1) + + assert release_finished.is_set() + + +@pytest.mark.asyncio +async def test_open_source_stream_cleanup_finishes_after_cancellation(monkeypatch: pytest.MonkeyPatch): + import app.modules.model_sources.forwarding as forwarding_module + from app.db.models import ModelSource + + cleanup_started = asyncio.Event() + allow_cleanup_finish = asyncio.Event() + cleanup_finished = asyncio.Event() + + class _FailingPostContext: + async def __aenter__(self): + raise asyncio.CancelledError() + + async def __aexit__(self, exc_type, exc, tb) -> bool: + del exc_type, exc, tb + return False + + class _Session: + def post(self, *_args: object, **_kwargs: object) -> _FailingPostContext: + return _FailingPostContext() + + class _SessionLease: + async def __aenter__(self) -> _Session: + return _Session() + + async def __aexit__(self, exc_type, exc, tb) -> bool: + del exc_type, exc, tb + cleanup_started.set() + await allow_cleanup_finish.wait() + cleanup_finished.set() + return False + + monkeypatch.setattr(forwarding_module, "lease_http_session", lambda: _SessionLease()) + + source = ModelSource( + id="src_open_cancelled_cleanup", + name="open-cancelled-cleanup", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + + task = asyncio.create_task( + forwarding_module._open_source_stream( + source, + "/chat/completions", + {"model": "open-cancelled-cleanup"}, + encryptor=None, + ) + ) + await asyncio.wait_for(cleanup_started.wait(), timeout=1) + task.cancel() + allow_cleanup_finish.set() + + with pytest.raises(asyncio.CancelledError): + await task + + assert cleanup_finished.is_set() is True + + +@pytest.mark.asyncio +async def test_forward_chat_completion_cleanup_finishes_after_cancellation(monkeypatch: pytest.MonkeyPatch): + import app.modules.model_sources.forwarding as forwarding_module + from app.db.models import ModelSource + + cleanup_started = asyncio.Event() + allow_cleanup_finish = asyncio.Event() + cleanup_finished = asyncio.Event() + + class _Response: + status = 200 + + async def json(self, content_type=None): + del content_type + return { + "id": "chatcmpl_forward_cancelled_cleanup", + "usage": {"prompt_tokens": 3, "completion_tokens": 5}, + } + + class _PostContext: + async def __aenter__(self) -> _Response: + return _Response() + + async def __aexit__(self, exc_type, exc, tb) -> bool: + del exc_type, exc, tb + return False + + class _Session: + def post(self, *_args: object, **_kwargs: object) -> _PostContext: + return _PostContext() + + class _SessionLease: + async def __aenter__(self) -> _Session: + return _Session() + + async def __aexit__(self, exc_type, exc, tb) -> bool: + del exc_type, exc, tb + cleanup_started.set() + await allow_cleanup_finish.wait() + cleanup_finished.set() + return False + + monkeypatch.setattr(forwarding_module, "lease_http_session", lambda: _SessionLease()) + + source = ModelSource( + id="src_forward_cancelled_cleanup", + name="forward-cancelled-cleanup", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + + task = asyncio.create_task( + forwarding_module.forward_chat_completion( + source, + {"model": "forward-cancelled-cleanup", "messages": [{"role": "user", "content": "hello"}]}, + ) + ) + await asyncio.wait_for(cleanup_started.wait(), timeout=1) + task.cancel() + allow_cleanup_finish.set() + + with pytest.raises(asyncio.CancelledError): + await task + + assert cleanup_finished.is_set() is True + + +@pytest.mark.asyncio +async def test_downstream_disconnect_closes_source_stream(async_client, monkeypatch): + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.db.models import ModelSource + from app.modules.model_sources.forwarding import SourceUsageHolder + + released: list[object] = [] + stream_closed = False + + async def record_release(reservation: object) -> None: + released.append(reservation) + + async def skip_log(*args, **kwargs) -> None: + del args, kwargs + + monkeypatch.setattr(proxy_api, "_release_reservation", record_release) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", skip_log) + + async def source_stream() -> AsyncIterator[bytes]: + nonlocal stream_closed + try: + yield b"data: partial\n\n" + await asyncio.sleep(60) + finally: + stream_closed = True + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } + ) + source = ModelSource( + id="src_disconnect", + name="disconnect", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_disconnect", + key_id="key_disconnect", + model="disconnect-model", + ) + response_stream = cast( + AsyncGenerator[bytes, None], + proxy_api._source_chat_stream_with_settlement( + source_stream(), + usage_holder=SourceUsageHolder(), + request=request, + source=source, + api_key=None, + model="disconnect-model", + reservation=reservation, + ), + ) + + assert await anext(response_stream) == b"data: partial\n\n" + await response_stream.aclose() + + assert released == [reservation] + assert stream_closed is True + + +@pytest.mark.asyncio +async def test_source_stream_disconnect_logs_cancelled_not_error(async_client, db_setup, monkeypatch): + """Regression for #1552: a downstream disconnect mid-stream on a + model-source route is a normal client-side terminal — recorded as + status=cancelled (like the main proxy path), counted in cancelled_count, + and excluded from the error rate and top_error.""" + from datetime import timedelta + + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.db.models import ModelSource + from app.modules.model_sources.forwarding import SourceUsageHolder + from app.modules.request_logs.repository import RequestLogsRepository + + async def record_release(reservation: object) -> None: + del reservation + + monkeypatch.setattr(proxy_api, "_release_reservation", record_release) + + async def source_stream() -> AsyncIterator[bytes]: + yield b"data: partial\n\n" + await asyncio.sleep(60) + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } + ) + source = ModelSource( + id="src_cx_log", + name="cx-log", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + response_stream = cast( + AsyncGenerator[bytes, None], + proxy_api._source_chat_stream_with_settlement( + source_stream(), + usage_holder=SourceUsageHolder(), + request=request, + source=source, + api_key=None, + model="cx-log-model", + reservation=None, + ), + ) + + assert await anext(response_stream) == b"data: partial\n\n" + await response_stream.aclose() + + async with SessionLocal() as session: + row = (await session.execute(select(RequestLog).where(RequestLog.model_source_id == "src_cx_log"))).scalar_one() + assert row.status == "cancelled" + assert row.error_code == "client_disconnected" + + # The status classification is what every metric surface keys on: + # the disconnect must not join the error numerator or top_error. + aggregate = await RequestLogsRepository(session).aggregate_usage_metrics_since(utcnow() - timedelta(minutes=5)) + assert aggregate.request_count == 1 + assert aggregate.error_count == 0 + assert aggregate.cancelled_count == 1 + assert aggregate.top_error is None + + +@pytest.mark.asyncio +async def test_source_stream_settlement_cancellation_logs_cancelled_not_success(monkeypatch: pytest.MonkeyPatch): + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.db.models import ModelSource + from app.modules.model_sources.forwarding import SourceUsage, SourceUsageHolder + + settle_started = asyncio.Event() + allow_settle_finish = asyncio.Event() + released: list[object] = [] + logs: list[dict[str, object]] = [] + + async def settle(*_args: object, **_kwargs: object) -> bool: + settle_started.set() + await allow_settle_finish.wait() + return True + + async def record_release(reservation: object) -> None: + released.append(reservation) + + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(dict(kwargs)) + + monkeypatch.setattr(proxy_api, "_settle_source_reservation", settle) + monkeypatch.setattr(proxy_api, "_release_reservation", record_release) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) + + async def source_stream() -> AsyncIterator[bytes]: + yield b"data: partial\n\n" + + request = Request( + { + "type": "http", + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/v1/chat/completions", + "raw_path": b"/v1/chat/completions", + "query_string": b"", + "headers": [], + "client": ("127.0.0.1", 0), + "server": ("testserver", 80), + } + ) + source = ModelSource( + id="src_stream_settlement_cancel", + name="stream-settlement-cancel", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_stream_settlement_cancel", + key_id="key_stream_settlement_cancel", + model="stream-settlement-cancel", + ) + usage_holder = SourceUsageHolder(usage=SourceUsage(input_tokens=3, output_tokens=5)) + + async def consume_stream() -> None: + async for _chunk in proxy_api._source_chat_stream_with_settlement( + source_stream(), + usage_holder=usage_holder, + request=request, + source=source, + api_key=None, + model="stream-settlement-cancel", + reservation=reservation, + ): + pass + + task = asyncio.create_task(consume_stream()) + await asyncio.wait_for(settle_started.wait(), timeout=1) + task.cancel() + allow_settle_finish.set() + + with pytest.raises(asyncio.CancelledError): + await task + + assert released == [] + assert logs[-1]["status"] == "cancelled" + assert logs[-1]["error_code"] == "client_disconnected" + assert logs[-1]["error_message"] == "client disconnected during source usage settlement" + assert logs[-1]["usage"] == usage_holder.usage + + +@pytest.mark.asyncio +async def test_opportunistic_key_routes_to_source_without_account_pool(async_client, source_upstream): + await _enable_api_key_auth(async_client) + + async def completion(_request: web.Request) -> web.Response: + return web.json_response( + { + "id": "chatcmpl_opportunistic", + "object": "chat.completion", + "created": 1, + "model": "opportunistic-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ) + + base_url = await source_upstream(completion) + model = "opportunistic-model" + source_id = await _create_model_source(async_client, name="opportunistic", model=model, base_url=base_url) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "opportunistic-source-key", + "assignedSourceIds": [source_id], + "trafficClass": "opportunistic", + }, + ) + assert created.status_code == 200 + key = created.json()["key"] + + # No subscription accounts exist, so opportunistic admission would deny + # with 429 if it (incorrectly) gated the account-free source path. + response = await async_client.post( + "/v1/chat/completions", + headers={"Authorization": f"Bearer {key}"}, + json={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + ) + + assert response.status_code == 200 + assert response.json()["id"] == "chatcmpl_opportunistic" + + +@pytest.mark.asyncio +async def test_source_credential_decrypt_failure_maps_to_error_and_releases_reservation( + async_client, source_upstream, monkeypatch +): + await _enable_api_key_auth(async_client) + + async def completion(_request: web.Request) -> web.Response: + return web.json_response({"unreachable": True}) + + base_url = await source_upstream(completion) + model = "credential-fail-model" + source_id = await _create_model_source(async_client, name="credential-fail", model=model, base_url=base_url) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "credential-fail-key", + "assignedSourceIds": [source_id], + "limits": [ + {"limitType": "total_tokens", "limitWindow": "weekly", "maxValue": 1_000}, + ], + }, + ) + assert created.status_code == 200 + key = created.json()["key"] + + from app.core.crypto import TokenEncryptor + + def broken_decrypt(self, value): + raise ValueError("decryption boom") + + monkeypatch.setattr(TokenEncryptor, "decrypt", broken_decrypt) + + response = await async_client.post( + "/v1/chat/completions", + headers={"Authorization": f"Bearer {key}"}, + json={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + ) + + assert response.status_code == 502 + assert response.json()["error"]["code"] == "model_source_credentials_error" + + async with SessionLocal() as session: + result = await session.execute( + select(ApiKeyUsageReservation).where(ApiKeyUsageReservation.status == "reserved") + ) + assert result.scalars().all() == [] + + +def _chat_completion_body(model: str) -> dict[str, object]: + return { + "id": "chatcmpl_sanitized", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "4"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}, + } + + +@pytest.mark.asyncio +async def test_source_chat_payload_drops_empty_tools_and_reasoning_toggles(async_client, source_upstream): + captured: dict[str, object] = {} + + async def capture(request: web.Request) -> web.Response: + captured.update(await request.json()) + return web.json_response(_chat_completion_body("sanitized-model")) + + base_url = await source_upstream(capture) + model = "sanitized-model" + await _create_model_source(async_client, name="sanitized", model=model, base_url=base_url) + + response = await async_client.post( + "/v1/chat/completions", + json={ + "model": model, + "messages": [{"role": "user", "content": "Kiek yra 2+2?"}], + "tools": [], + "tool_choice": "none", + "include_reasoning": True, + "separate_reasoning": True, + "stream_reasoning": True, + "reasoning_effort": "low", + "max_tokens": 200, + }, + ) + + assert response.status_code == 200 + assert captured["model"] == model + assert captured["max_tokens"] == 200 + for key in ( + "tools", + "tool_choice", + "parallel_tool_calls", + "include_reasoning", + "separate_reasoning", + "stream_reasoning", + "reasoning", + "reasoning_effort", + ): + assert key not in captured + + +@pytest.mark.asyncio +async def test_source_chat_payload_enforced_reasoning_stays_stripped_for_plain_model(async_client, source_upstream): + await _enable_api_key_auth(async_client) + captured: dict[str, object] = {} + + async def capture(request: web.Request) -> web.Response: + captured.update(await request.json()) + return web.json_response(_chat_completion_body("plain-enforced-model")) + + base_url = await source_upstream(capture) + model = "plain-enforced-model" + source_id = await _create_model_source( + async_client, + name="plain-enforced", + model=model, + base_url=base_url, + ) + key_response = await async_client.post( + "/api/api-keys/", + json={ + "name": "plain enforced source key", + "enforcedReasoningEffort": "high", + "sourceAssignmentScopeEnabled": True, + "assignedSourceIds": [source_id], + }, + ) + assert key_response.status_code == 200 + key = key_response.json()["key"] + + response = await async_client.post( + "/v1/chat/completions", + headers={"Authorization": f"Bearer {key}"}, + json={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + ) + + assert response.status_code == 200 + assert "reasoning" not in captured + assert "reasoning_effort" not in captured + + +@pytest.mark.asyncio +async def test_source_chat_without_usage_ignores_limits_for_other_models(async_client, source_upstream): + await _enable_api_key_auth(async_client) + + async def completion_without_usage(_request: web.Request) -> web.Response: + body = _chat_completion_body("source-unlimited-by-filter") + body.pop("usage", None) + return web.json_response(body) + + base_url = await source_upstream(completion_without_usage) + model = "source-unlimited-by-filter" + source_id = await _create_model_source( + async_client, + name="source-unlimited-by-filter", + model=model, + base_url=base_url, + ) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "source limit for other model", + "assignedSourceIds": [source_id], + "limits": [ + { + "limitType": "total_tokens", + "limitWindow": "weekly", + "maxValue": 5, + "modelFilter": "some-other-model", + }, + ], + }, + ) + assert created.status_code == 200 + key = created.json()["key"] + + response = await async_client.post( + "/v1/chat/completions", + headers={"Authorization": f"Bearer {key}"}, + json={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + ) + + assert response.status_code == 200 + assert response.json()["id"] == "chatcmpl_sanitized" + + async with SessionLocal() as session: + result = await session.execute( + select(ApiKeyUsageReservation).where(ApiKeyUsageReservation.status == "reserved") + ) + assert result.scalars().all() == [] + + +@pytest.mark.asyncio +async def test_source_chat_prefers_raw_alias_like_model_slug(async_client, source_upstream): + captured: dict[str, object] = {} + + async def capture(request: web.Request) -> web.Response: + captured.update(await request.json()) + return web.json_response(_chat_completion_body("gpt-5-high")) + + base_url = await source_upstream(capture) + model = "gpt-5-high" + await _create_model_source( + async_client, + name="alias-like-source", + model=model, + base_url=base_url, + ) + + response = await async_client.post( + "/v1/chat/completions", + json={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + ) + + assert response.status_code == 200 + assert captured["model"] == model + assert response.json()["model"] == model + + +@pytest.mark.asyncio +async def test_source_chat_raw_alias_lookup_requires_exact_allowlist(async_client): + import app.modules.proxy.api as proxy_api + + model = "gpt-5-high" + await _create_model_source( + async_client, + name="alias-like-allowlist-source", + model=model, + base_url="http://127.0.0.1:9/v1", + ) + canonical_only_key = ApiKeyData( + id="key_canonical_only", + name="canonical only", + key_prefix="sk-test-canonical", + allowed_models=["gpt-5"], + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + exact_key = ApiKeyData( + id="key_exact_alias", + name="exact alias", + key_prefix="sk-test-exact", + allowed_models=[model], + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) - async with SessionLocal() as session: - result = await session.execute( - select(ApiKeyUsageReservation).where(ApiKeyUsageReservation.status == "reserved") - ) - assert result.scalars().all() == [] + canonical_selection = await proxy_api._select_chat_model_source( + "gpt-5", + canonical_only_key, + raw_model=model, + ) + exact_selection = await proxy_api._select_chat_model_source( + "gpt-5", + exact_key, + raw_model=model, + ) + assert canonical_selection is None + assert exact_selection is not None + source, selected_model = exact_selection + assert source.name == "alias-like-allowlist-source" + assert selected_model == model -def _chat_completion_body(model: str) -> dict[str, object]: - return { - "id": "chatcmpl_sanitized", - "object": "chat.completion", - "created": 1, - "model": model, - "choices": [{"index": 0, "message": {"role": "assistant", "content": "4"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}, - } + +@pytest.mark.asyncio +async def test_v1_models_metadata_reflects_reasoning_optin(async_client): + await _create_model_source( + async_client, + name="reasoning-metadata", + model="reasoning-metadata-model", + base_url="http://127.0.0.1:9/v1", + raw_metadata_json='{"supports_reasoning": true}', + ) + await _create_model_source( + async_client, + name="plain-metadata", + model="plain-metadata-model", + base_url="http://127.0.0.1:9/v1", + ) + + response = await async_client.get("/v1/models") + assert response.status_code == 200 + by_id = {item["id"]: item for item in response.json()["data"]} + + assert by_id["reasoning-metadata-model"]["supports_reasoning"] is True + assert by_id["plain-metadata-model"]["supports_reasoning"] is False @pytest.mark.asyncio -async def test_source_chat_payload_drops_empty_tools_and_reasoning_toggles(async_client, source_upstream): +async def test_v1_models_context_window_override_applies_to_source_model(async_client, monkeypatch): + # Source-catalog models synthesize `max_context_window == context_window` + # purely so Codex clients can parse the entry; that parseability default + # must not clamp an operator raise override to the un-raised window. + await _create_model_source( + async_client, + name="override-source", + model="override-source-model", + base_url="http://127.0.0.1:9/v1", + ) + + from app.core.config.settings import get_settings + from app.modules.proxy import api as proxy_api_module + + patched = get_settings().model_copy(update={"model_context_window_overrides": {"override-source-model": 32_768}}) + monkeypatch.setattr(proxy_api_module, "get_settings", lambda: patched) + + response = await async_client.get("/v1/models") + assert response.status_code == 200 + item = next(m for m in response.json()["data"] if m["id"] == "override-source-model") + assert item["metadata"]["context_window"] == 32_768 + assert item["metadata"]["input_context_window"] == 32_768 + assert item["capabilities"]["context_length"] == 32_768 + assert item["contextLength"] == 32_768 + assert item["context_length"] == 32_768 + + +@pytest.mark.asyncio +async def test_source_chat_payload_keeps_reasoning_toggles_for_optin_model(async_client, source_upstream): captured: dict[str, object] = {} async def capture(request: web.Request) -> web.Response: captured.update(await request.json()) - return web.json_response(_chat_completion_body("sanitized-model")) + return web.json_response(_chat_completion_body("reasoning-model")) base_url = await source_upstream(capture) - model = "sanitized-model" - await _create_model_source(async_client, name="sanitized", model=model, base_url=base_url) + model = "reasoning-model" + await _create_model_source( + async_client, + name="reasoning-optin", + model=model, + base_url=base_url, + raw_metadata_json='{"supports_reasoning": true}', + ) response = await async_client.post( "/v1/chat/completions", json={ "model": model, - "messages": [{"role": "user", "content": "Kiek yra 2+2?"}], - "tools": [], - "tool_choice": "none", + "messages": [{"role": "user", "content": "hi"}], "include_reasoning": True, - "separate_reasoning": True, - "stream_reasoning": True, + "reasoning_effort": "high", + }, + ) + + assert response.status_code == 200 + assert captured["include_reasoning"] is True + assert captured["reasoning_effort"] == "high" + assert "tools" not in captured + + +@pytest.mark.asyncio +async def test_source_chat_payload_overrides_enforced_reasoning_object(async_client, source_upstream): + await _enable_api_key_auth(async_client) + captured: dict[str, object] = {} + + async def capture(request: web.Request) -> web.Response: + captured.update(await request.json()) + return web.json_response(_chat_completion_body("reasoning-enforced-model")) + + base_url = await source_upstream(capture) + model = "reasoning-enforced-model" + source_id = await _create_model_source( + async_client, + name="reasoning-enforced", + model=model, + base_url=base_url, + raw_metadata_json='{"supports_reasoning": true}', + ) + key_response = await async_client.post( + "/api/api-keys/", + json={ + "name": "reasoning enforced source key", + "enforcedReasoningEffort": "high", + "sourceAssignmentScopeEnabled": True, + "assignedSourceIds": [source_id], + }, + ) + assert key_response.status_code == 200 + key = key_response.json()["key"] + + response = await async_client.post( + "/v1/chat/completions", + headers={"Authorization": f"Bearer {key}"}, + json={ + "model": model, + "messages": [{"role": "user", "content": "hi"}], + "reasoning": {"effort": "low", "summary": "auto"}, "reasoning_effort": "low", - "max_tokens": 200, }, ) assert response.status_code == 200 - assert captured["model"] == model - assert captured["max_tokens"] == 200 - for key in ( - "tools", - "tool_choice", - "parallel_tool_calls", - "include_reasoning", - "separate_reasoning", - "stream_reasoning", - "reasoning", - "reasoning_effort", - ): - assert key not in captured + assert captured["reasoning"] == {"effort": "high", "summary": "auto"} + assert captured["reasoning_effort"] == "high" + + +@pytest.mark.asyncio +async def test_dashboard_models_endpoint_lists_source_models(async_client): + model = "picker-source-model" + await _create_model_source( + async_client, + name="picker", + model=model, + base_url="http://127.0.0.1:9/v1", + ) + + response = await async_client.get("/api/models") + assert response.status_code == 200 + models = response.json()["models"] + ids = [entry["id"] for entry in models] + assert model in ids + assert ids.count(model) == 1 + source_entry = next(entry for entry in models if entry["id"] == model) + assert source_entry["sourceOnly"] is True + + +@pytest.mark.asyncio +async def test_allowlisted_source_model_routes_through(async_client, source_upstream): + await _enable_api_key_auth(async_client) + + async def completion(_request: web.Request) -> web.Response: + return web.json_response(_chat_completion_body("allowlisted-model")) + + base_url = await source_upstream(completion) + model = "allowlisted-model" + source_id = await _create_model_source(async_client, name="allowlisted", model=model, base_url=base_url) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "allowlisted-key", + "assignedSourceIds": [source_id], + "allowedModels": [model], + }, + ) + assert created.status_code == 200 + key = created.json()["key"] + + allowed = await async_client.post( + "/v1/chat/completions", + headers={"Authorization": f"Bearer {key}"}, + json={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + ) + assert allowed.status_code == 200 + + denied = await async_client.post( + "/v1/chat/completions", + headers={"Authorization": f"Bearer {key}"}, + json={"model": "some-other-model", "messages": [{"role": "user", "content": "hi"}]}, + ) + assert denied.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "reasoning_controls", + [ + {"reasoning_effort": "max"}, + {"thinking": "minimal"}, + {"reasoning_effort": "max", "reasoning": {"summary": "auto"}}, + {"thinking": False, "enable_thinking": True}, + {"thinking": "disabled", "enable_thinking": True}, + {"thinking": {"summary": "auto", "enabled": True}}, + {"thinking": {"summary": "auto"}, "enable_thinking": True}, + ], +) +async def test_source_chat_reasoning_allowlist_rejects_before_source_dispatch( + async_client, + source_upstream, + reasoning_controls, +): + await _enable_api_key_auth(async_client) + source_hits = 0 + + async def completion(_request: web.Request) -> web.Response: + nonlocal source_hits + source_hits += 1 + return web.json_response(_chat_completion_body("source-reasoning-policy")) + + base_url = await source_upstream(completion) + model = "source-reasoning-policy" + source_id = await _create_model_source(async_client, name="source-reasoning-policy", model=model, base_url=base_url) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "source-reasoning-policy-key", + "assignedSourceIds": [source_id], + "allowedReasoningEfforts": ["low"], + }, + ) + assert created.status_code == 200 + + response = await async_client.post( + "/v1/chat/completions", + headers={"Authorization": f"Bearer {created.json()['key']}"}, + json={ + "model": model, + "messages": [{"role": "user", "content": "hi"}], + **reasoning_controls, + }, + ) + + assert response.status_code == 403 + assert response.json()["error"]["code"] == "reasoning_effort_not_allowed" + assert source_hits == 0 + + +@pytest.mark.asyncio +async def test_source_chat_reasoning_allowlist_preserves_client_plane_effort(async_client, source_upstream): + await _enable_api_key_auth(async_client) + captured: dict[str, object] = {} + + async def completion(request: web.Request) -> web.Response: + captured.update(await request.json()) + return web.json_response(_chat_completion_body("source-client-plane-reasoning")) + + base_url = await source_upstream(completion) + model = "source-client-plane-reasoning" + source_id = await _create_model_source( + async_client, + name="source-client-plane-reasoning", + model=model, + base_url=base_url, + raw_metadata_json='{"supports_reasoning": true}', + ) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "source-client-plane-reasoning-key", + "assignedSourceIds": [source_id], + "allowedReasoningEfforts": ["minimal"], + }, + ) + assert created.status_code == 200 + + response = await async_client.post( + "/v1/chat/completions", + headers={"Authorization": f"Bearer {created.json()['key']}"}, + json={ + "model": model, + "messages": [{"role": "user", "content": "hi"}], + "reasoning_effort": "minimal", + "thinking": { + "effort": "minimal", + "type": "disabled", + "enabled": False, + "vendor_hint": "keep", + }, + }, + ) + + assert response.status_code == 200 + assert captured["reasoning_effort"] == "minimal" + assert captured["thinking"] == {"effort": "minimal", "vendor_hint": "keep"} + assert "reasoning" not in captured @pytest.mark.asyncio -async def test_source_chat_payload_enforced_reasoning_stays_stripped_for_plain_model(async_client, source_upstream): +async def test_source_chat_reasoning_allowlist_preserves_enable_thinking(async_client, source_upstream): await _enable_api_key_auth(async_client) captured: dict[str, object] = {} - async def capture(request: web.Request) -> web.Response: + async def completion(request: web.Request) -> web.Response: captured.update(await request.json()) - return web.json_response(_chat_completion_body("plain-enforced-model")) + return web.json_response(_chat_completion_body("source-enable-thinking")) - base_url = await source_upstream(capture) - model = "plain-enforced-model" + base_url = await source_upstream(completion) + model = "source-enable-thinking" source_id = await _create_model_source( async_client, - name="plain-enforced", + name="source-enable-thinking", model=model, base_url=base_url, + raw_metadata_json='{"supports_reasoning": true}', ) - key_response = await async_client.post( + created = await async_client.post( "/api/api-keys/", json={ - "name": "plain enforced source key", - "enforcedReasoningEffort": "high", - "sourceAssignmentScopeEnabled": True, + "name": "source-enable-thinking-key", "assignedSourceIds": [source_id], + "allowedReasoningEfforts": ["medium"], }, ) - assert key_response.status_code == 200 - key = key_response.json()["key"] + assert created.status_code == 200 response = await async_client.post( "/v1/chat/completions", - headers={"Authorization": f"Bearer {key}"}, - json={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + headers={"Authorization": f"Bearer {created.json()['key']}"}, + json={ + "model": model, + "messages": [{"role": "user", "content": "hi"}], + "enable_thinking": True, + }, ) assert response.status_code == 200 + assert captured["enable_thinking"] is True assert "reasoning" not in captured assert "reasoning_effort" not in captured @pytest.mark.asyncio -async def test_source_chat_without_usage_ignores_limits_for_other_models(async_client, source_upstream): +@pytest.mark.parametrize( + ("thinking", "enable_thinking", "expected_thinking"), + [ + ( + {"type": "enabled", "budget_tokens": 2048}, + False, + {"type": "enabled", "budget_tokens": 2048}, + ), + ( + {"enabled": True, "summary": "auto", "vendor_hint": "keep"}, + False, + {"enabled": True, "summary": "auto", "vendor_hint": "keep"}, + ), + ( + {"effort": " ", "enabled": True, "budget_tokens": 2048, "vendor_hint": "keep"}, + False, + {"enabled": True, "budget_tokens": 2048, "vendor_hint": "keep"}, + ), + ({"enabled": False}, True, None), + ({"type": "disabled"}, True, None), + ], +) +async def test_source_chat_reasoning_allowlist_preserves_implicit_thinking_object( + async_client, + source_upstream, + thinking, + enable_thinking, + expected_thinking, +): await _enable_api_key_auth(async_client) + captured: dict[str, object] = {} - async def completion_without_usage(_request: web.Request) -> web.Response: - body = _chat_completion_body("source-unlimited-by-filter") - body.pop("usage", None) - return web.json_response(body) + async def completion(request: web.Request) -> web.Response: + captured.update(await request.json()) + return web.json_response(_chat_completion_body("source-implicit-thinking")) - base_url = await source_upstream(completion_without_usage) - model = "source-unlimited-by-filter" + base_url = await source_upstream(completion) + model = "source-implicit-thinking" source_id = await _create_model_source( async_client, - name="source-unlimited-by-filter", + name=model, model=model, base_url=base_url, + raw_metadata_json='{"supports_reasoning": true}', ) created = await async_client.post( "/api/api-keys/", json={ - "name": "source limit for other model", + "name": "source-implicit-thinking-key", "assignedSourceIds": [source_id], - "limits": [ - { - "limitType": "total_tokens", - "limitWindow": "weekly", - "maxValue": 5, - "modelFilter": "some-other-model", - }, - ], + "allowedReasoningEfforts": ["medium"], }, ) assert created.status_code == 200 - key = created.json()["key"] + request_payload = { + "model": model, + "messages": [{"role": "user", "content": "hi"}], + "thinking": thinking, + } + if enable_thinking: + request_payload["enable_thinking"] = True response = await async_client.post( "/v1/chat/completions", - headers={"Authorization": f"Bearer {key}"}, - json={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + headers={"Authorization": f"Bearer {created.json()['key']}"}, + json=request_payload, ) assert response.status_code == 200 - assert response.json()["id"] == "chatcmpl_sanitized" - - async with SessionLocal() as session: - result = await session.execute( - select(ApiKeyUsageReservation).where(ApiKeyUsageReservation.status == "reserved") - ) - assert result.scalars().all() == [] + if expected_thinking is None: + assert "thinking" not in captured + else: + assert captured["thinking"] == expected_thinking + if enable_thinking: + assert captured["enable_thinking"] is True + assert "reasoning" not in captured + assert "reasoning_effort" not in captured @pytest.mark.asyncio -async def test_source_chat_prefers_raw_alias_like_model_slug(async_client, source_upstream): +@pytest.mark.parametrize("alias_source", ["requested", "enforced"]) +async def test_source_chat_reasoning_allowlist_materializes_canonicalized_model_alias_effort( + async_client, + source_upstream, + alias_source, +): + await _enable_api_key_auth(async_client) captured: dict[str, object] = {} - async def capture(request: web.Request) -> web.Response: + async def completion(request: web.Request) -> web.Response: captured.update(await request.json()) - return web.json_response(_chat_completion_body("gpt-5-high")) + return web.json_response(_chat_completion_body("gpt-5.6-sol")) - base_url = await source_upstream(capture) - model = "gpt-5-high" - await _create_model_source( + base_url = await source_upstream(completion) + model = "gpt-5.6-sol" + source_id = await _create_model_source( async_client, - name="alias-like-source", + name="source-canonical-model-alias-effort", model=model, base_url=base_url, + raw_metadata_json='{"supports_reasoning": true}', ) + key_payload = { + "name": "source-canonical-model-alias-effort-key", + "assignedSourceIds": [source_id], + "allowedReasoningEfforts": ["xhigh"], + } + if alias_source == "enforced": + key_payload["enforcedModel"] = f"{model}-xhigh" + created = await async_client.post("/api/api-keys/", json=key_payload) + assert created.status_code == 200 response = await async_client.post( "/v1/chat/completions", - json={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + headers={"Authorization": f"Bearer {created.json()['key']}"}, + json={ + "model": f"{model}-xhigh" if alias_source == "requested" else model, + "messages": [{"role": "user", "content": "hi"}], + }, ) assert response.status_code == 200 assert captured["model"] == model - assert response.json()["model"] == model + assert captured["reasoning_effort"] == "xhigh" + assert "reasoning" not in captured @pytest.mark.asyncio -async def test_source_chat_raw_alias_lookup_requires_exact_allowlist(async_client): - import app.modules.proxy.api as proxy_api +@pytest.mark.parametrize( + ("with_allowlist", "reasoning", "enable_thinking", "thinking_effort"), + [ + (False, None, False, None), + (True, None, False, None), + (True, {"effort": "low"}, False, None), + (True, {"effort": "low"}, True, None), + (True, {"effort": "low"}, False, " "), + ], +) +async def test_source_responses_preserves_effortless_provider_thinking_object( + async_client, + source_upstream, + with_allowlist, + reasoning, + enable_thinking, + thinking_effort, +): + if with_allowlist: + await _enable_api_key_auth(async_client) + captured: dict[str, object] = {} - model = "gpt-5-high" - await _create_model_source( + async def responses(request: web.Request) -> web.Response: + captured.update(await request.json()) + return web.json_response( + { + "id": "resp_provider_thinking", + "object": "response", + "status": "completed", + "model": "source-provider-thinking", + "output": [], + } + ) + + base_url = await source_upstream(responses) + model = "source-provider-thinking" + source_id = await _create_model_source( async_client, - name="alias-like-allowlist-source", + name="source-provider-thinking", model=model, - base_url="http://127.0.0.1:9/v1", - ) - canonical_only_key = ApiKeyData( - id="key_canonical_only", - name="canonical only", - key_prefix="sk-test-canonical", - allowed_models=["gpt-5"], - enforced_model=None, - enforced_reasoning_effort=None, - enforced_service_tier=None, - expires_at=None, - is_active=True, - created_at=utcnow(), - last_used_at=None, - ) - exact_key = ApiKeyData( - id="key_exact_alias", - name="exact alias", - key_prefix="sk-test-exact", - allowed_models=[model], - enforced_model=None, - enforced_reasoning_effort=None, - enforced_service_tier=None, - expires_at=None, - is_active=True, - created_at=utcnow(), - last_used_at=None, - ) - - canonical_selection = await proxy_api._select_chat_model_source( - "gpt-5", - canonical_only_key, - raw_model=model, - ) - exact_selection = await proxy_api._select_chat_model_source( - "gpt-5", - exact_key, - raw_model=model, + base_url=base_url, + supports_responses=True, ) + thinking = {"type": "adaptive", "budget": 4096, "budget_tokens": 2048, "vendor_hint": "keep"} + if thinking_effort is not None: + thinking["effort"] = thinking_effort + headers: dict[str, str] = {} + if with_allowlist: + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "source-provider-thinking-key", + "assignedSourceIds": [source_id], + "allowedReasoningEfforts": ["low"], + }, + ) + assert created.status_code == 200 + headers["Authorization"] = f"Bearer {created.json()['key']}" - assert canonical_selection is None - assert exact_selection is not None - source, selected_model = exact_selection - assert source.name == "alias-like-allowlist-source" - assert selected_model == model - + request_payload = { + "model": model, + "instructions": "hi", + "input": [], + "thinking": thinking, + } + if reasoning is not None: + request_payload["reasoning"] = reasoning + if enable_thinking: + request_payload["enable_thinking"] = True -@pytest.mark.asyncio -async def test_v1_models_metadata_reflects_reasoning_optin(async_client): - await _create_model_source( - async_client, - name="reasoning-metadata", - model="reasoning-metadata-model", - base_url="http://127.0.0.1:9/v1", - raw_metadata_json='{"supports_reasoning": true}', - ) - await _create_model_source( - async_client, - name="plain-metadata", - model="plain-metadata-model", - base_url="http://127.0.0.1:9/v1", - ) + response = await async_client.post("/v1/responses", headers=headers, json=request_payload) - response = await async_client.get("/v1/models") assert response.status_code == 200 - by_id = {item["id"]: item for item in response.json()["data"]} - - assert by_id["reasoning-metadata-model"]["supports_reasoning"] is True - assert by_id["plain-metadata-model"]["supports_reasoning"] is False + expected_thinking = {key: value for key, value in thinking.items() if key != "effort"} + assert captured["thinking"] == expected_thinking + if reasoning is None: + assert "reasoning" not in captured + else: + assert captured["reasoning"] == reasoning + if enable_thinking: + assert "enable_thinking" not in captured @pytest.mark.asyncio -async def test_source_chat_payload_keeps_reasoning_toggles_for_optin_model(async_client, source_upstream): +@pytest.mark.parametrize( + ("key_policy", "reasoning_control", "expected_control"), + [ + ("none", {"thinking": "minimal"}, {"thinking": "minimal"}), + ("unrestricted", {"thinking": "minimal"}, {"thinking": "minimal"}), + ("allowlisted", {"thinking": "minimal"}, {"reasoning": {"effort": "minimal"}}), + ("enforced", {"thinking": "low"}, {"reasoning": {"effort": "minimal"}}), + ("none", {"reasoning": {"effort": "minimal"}}, {"reasoning": {"effort": "minimal"}}), + ("allowlisted", {"reasoning": {"effort": "minimal"}}, {"reasoning": {"effort": "minimal"}}), + ], +) +async def test_source_responses_preserves_client_plane_reasoning_effort( + async_client, + source_upstream, + key_policy, + reasoning_control, + expected_control, +): + if key_policy != "none": + await _enable_api_key_auth(async_client) captured: dict[str, object] = {} - async def capture(request: web.Request) -> web.Response: + async def responses(request: web.Request) -> web.Response: captured.update(await request.json()) - return web.json_response(_chat_completion_body("reasoning-model")) + return web.json_response( + { + "id": "resp_provider_reasoning_alias", + "object": "response", + "status": "completed", + "model": "source-provider-reasoning-alias", + "output": [], + } + ) - base_url = await source_upstream(capture) - model = "reasoning-model" - await _create_model_source( + base_url = await source_upstream(responses) + model = "source-provider-reasoning-alias" + source_id = await _create_model_source( async_client, - name="reasoning-optin", + name=model, model=model, base_url=base_url, - raw_metadata_json='{"supports_reasoning": true}', + supports_responses=True, ) + headers: dict[str, str] = {} + if key_policy != "none": + key_payload = { + "name": "source-provider-reasoning-alias-key", + "assignedSourceIds": [source_id], + } + if key_policy == "allowlisted": + key_payload["allowedReasoningEfforts"] = ["minimal"] + elif key_policy == "enforced": + key_payload["enforcedReasoningEffort"] = "minimal" + created = await async_client.post( + "/api/api-keys/", + json=key_payload, + ) + assert created.status_code == 200 + headers["Authorization"] = f"Bearer {created.json()['key']}" response = await async_client.post( - "/v1/chat/completions", + "/v1/responses", + headers=headers, json={ "model": model, - "messages": [{"role": "user", "content": "hi"}], - "include_reasoning": True, - "reasoning_effort": "high", + "instructions": "hi", + "input": [], + **reasoning_control, }, ) assert response.status_code == 200 - assert captured["include_reasoning"] is True - assert captured["reasoning_effort"] == "high" - assert "tools" not in captured + for field in ("thinking", "reasoning"): + if field in expected_control: + assert captured[field] == expected_control[field] + else: + assert field not in captured @pytest.mark.asyncio -async def test_source_chat_payload_overrides_enforced_reasoning_object(async_client, source_upstream): +async def test_source_responses_reasoning_allowlist_strips_conflicting_aliases(async_client, source_upstream): await _enable_api_key_auth(async_client) captured: dict[str, object] = {} - async def capture(request: web.Request) -> web.Response: + async def responses(request: web.Request) -> web.Response: captured.update(await request.json()) - return web.json_response(_chat_completion_body("reasoning-enforced-model")) + return web.json_response( + { + "id": "resp_reasoning_policy", + "object": "response", + "status": "completed", + "model": "source-responses-reasoning-policy", + "output": [], + } + ) - base_url = await source_upstream(capture) - model = "reasoning-enforced-model" + base_url = await source_upstream(responses) + model = "source-responses-reasoning-policy" source_id = await _create_model_source( async_client, - name="reasoning-enforced", + name="source-responses-reasoning-policy", model=model, base_url=base_url, - raw_metadata_json='{"supports_reasoning": true}', + supports_responses=True, ) - key_response = await async_client.post( + created = await async_client.post( "/api/api-keys/", - json={ - "name": "reasoning enforced source key", - "enforcedReasoningEffort": "high", - "sourceAssignmentScopeEnabled": True, - "assignedSourceIds": [source_id], - }, - ) - assert key_response.status_code == 200 - key = key_response.json()["key"] - - response = await async_client.post( - "/v1/chat/completions", - headers={"Authorization": f"Bearer {key}"}, - json={ - "model": model, - "messages": [{"role": "user", "content": "hi"}], - "reasoning": {"effort": "low", "summary": "auto"}, - "reasoning_effort": "low", - }, - ) - - assert response.status_code == 200 - assert captured["reasoning"] == {"effort": "high", "summary": "auto"} - assert captured["reasoning_effort"] == "high" - - -@pytest.mark.asyncio -async def test_dashboard_models_endpoint_lists_source_models(async_client): - model = "picker-source-model" - await _create_model_source( - async_client, - name="picker", - model=model, - base_url="http://127.0.0.1:9/v1", + json={ + "name": "source-responses-reasoning-policy-key", + "assignedSourceIds": [source_id], + "allowedReasoningEfforts": ["low"], + }, + ) + assert created.status_code == 200 + + response = await async_client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {created.json()['key']}"}, + json={ + "model": model, + "instructions": "hi", + "input": [], + "reasoning": {"effort": "low"}, + "thinking": "max", + }, ) - response = await async_client.get("/api/models") assert response.status_code == 200 - models = response.json()["models"] - ids = [entry["id"] for entry in models] - assert model in ids - assert ids.count(model) == 1 - source_entry = next(entry for entry in models if entry["id"] == model) - assert source_entry["sourceOnly"] is True + assert captured["reasoning"] == {"effort": "low"} + assert "thinking" not in captured @pytest.mark.asyncio -async def test_allowlisted_source_model_routes_through(async_client, source_upstream): +@pytest.mark.parametrize( + "reasoning_controls", + [ + {"reasoningEffort": " ", "thinking": "max"}, + {"thinking": False, "enable_thinking": True}, + {"thinking": "disabled", "enable_thinking": True}, + {"thinking": {"summary": "auto", "enabled": True}}, + {"thinking": {"summary": "auto"}, "enable_thinking": True}, + ], +) +async def test_source_responses_reasoning_allowlist_rejects_effort_hidden_by_inactive_alias( + async_client, + source_upstream, + reasoning_controls, +): await _enable_api_key_auth(async_client) + source_hits = 0 - async def completion(_request: web.Request) -> web.Response: - return web.json_response(_chat_completion_body("allowlisted-model")) + async def responses(_request: web.Request) -> web.Response: + nonlocal source_hits + source_hits += 1 + return web.json_response( + { + "id": "resp_blank_reasoning_alias", + "object": "response", + "status": "completed", + "model": "source-blank-reasoning-alias", + "output": [], + } + ) - base_url = await source_upstream(completion) - model = "allowlisted-model" - source_id = await _create_model_source(async_client, name="allowlisted", model=model, base_url=base_url) + base_url = await source_upstream(responses) + model = "source-blank-reasoning-alias" + source_id = await _create_model_source( + async_client, + name="source-blank-reasoning-alias", + model=model, + base_url=base_url, + supports_responses=True, + ) created = await async_client.post( "/api/api-keys/", json={ - "name": "allowlisted-key", + "name": "source-blank-reasoning-alias-key", "assignedSourceIds": [source_id], - "allowedModels": [model], + "allowedReasoningEfforts": ["low"], }, ) assert created.status_code == 200 - key = created.json()["key"] - allowed = await async_client.post( - "/v1/chat/completions", - headers={"Authorization": f"Bearer {key}"}, - json={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + response = await async_client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {created.json()['key']}"}, + json={ + "model": model, + "instructions": "hi", + "input": [], + **reasoning_controls, + }, ) - assert allowed.status_code == 200 - denied = await async_client.post( - "/v1/chat/completions", - headers={"Authorization": f"Bearer {key}"}, - json={"model": "some-other-model", "messages": [{"role": "user", "content": "hi"}]}, - ) - assert denied.status_code == 403 + assert response.status_code == 403 + assert response.json()["error"]["code"] == "reasoning_effort_not_allowed" + assert source_hits == 0 @pytest.mark.asyncio @@ -1708,3 +3237,313 @@ async def stream_handler(request: web.Request) -> web.StreamResponse: assert b'"content":"hello"' in received assert b"[DONE]" in received + + +@pytest.mark.asyncio +async def test_source_responses_payload_restores_declared_minimal_effort(async_client, source_upstream): + """The minimal rewrite must be undone for a source that declared the effort. + + This pins the wiring, not just the helper: the restore lives inside + _source_responses_response, and both the call and the threading of the + replaced effort through enforcement have to survive for the source to see + ``minimal`` instead of the ``low`` fallback. + """ + captured: dict[str, object] = {} + + async def capture(request: web.Request) -> web.Response: + captured.update(await request.json()) + return web.json_response({"id": "resp_source_reasoning", "status": "completed", "output": []}) + + base_url = await source_upstream(capture) + model = "reasoning-levels-model" + await _create_model_source( + async_client, + name="reasoning-levels", + model=model, + base_url=base_url, + supports_responses=True, + raw_metadata_json='{"supports_reasoning": true, "supported_reasoning_levels": ["minimal", "low", "high"]}', + ) + + response = await async_client.post( + "/v1/responses", + json={ + "model": model, + "input": [{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}], + "reasoning": {"effort": "minimal"}, + }, + ) + + assert response.status_code == 200 + reasoning = captured["reasoning"] + assert isinstance(reasoning, dict) + assert reasoning["effort"] == "minimal" + + +@pytest.mark.asyncio +async def test_codex_responses_payload_restores_declared_minimal_effort(async_client, source_upstream): + """The codex-native route must thread the replaced effort too. + + Codex CLI talks to this route, and ``--reasoning-effort minimal`` is where + the rewrite originates, so this call site matters more than the /v1 one. + It forces streaming for source-routed requests, hence the SSE upstream. + """ + captured: dict[str, object] = {} + frames = b'data: {"type":"response.completed","response":{"id":"resp_codex","status":"completed"}}\n\n' + + async def capture(request: web.Request) -> web.StreamResponse: + captured.update(await request.json()) + response = web.StreamResponse(status=200, headers={"Content-Type": "text/event-stream"}) + await response.prepare(request) + await response.write(frames) + await response.write_eof() + return response + + base_url = await source_upstream(capture) + model = "codex-reasoning-levels-model" + await _create_model_source( + async_client, + name="codex-reasoning-levels", + model=model, + base_url=base_url, + supports_responses=True, + raw_metadata_json='{"supports_reasoning": true, "supported_reasoning_levels": ["minimal", "low", "high"]}', + ) + + async with async_client.stream( + "POST", + "/backend-api/codex/responses", + json={ + "model": model, + "instructions": "hi", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}], + "stream": True, + "reasoning": {"effort": "minimal"}, + }, + ) as response: + assert response.status_code == 200 + async for _ in response.aiter_bytes(): + pass + + reasoning = captured["reasoning"] + assert isinstance(reasoning, dict) + assert reasoning["effort"] == "minimal" + + +@pytest.mark.asyncio +async def test_source_embeddings_routes_payload_and_settles_usage(async_client, source_upstream) -> None: + await _enable_api_key_auth(async_client) + captured: dict[str, object] = {} + + async def embed(request: web.Request) -> web.Response: + captured["path"] = request.path + captured["authorization"] = request.headers.get("authorization") + captured["payload"] = await request.json() + return web.json_response( + { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], + "model": "all-minilm:latest", + "usage": {"prompt_tokens": 21, "total_tokens": 21}, + } + ) + + base_url = await source_upstream(embed) + model = "all-minilm:latest" + source_id = await _create_model_source( + async_client, + name="embedder", + model=model, + base_url=base_url, + input_per_1m=0.02, + supports_embeddings=True, + ) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "embeddings-source-key", + "assignedSourceIds": [source_id], + "limits": [ + {"limitType": "total_tokens", "limitWindow": "weekly", "maxValue": 1_000}, + ], + }, + ) + assert created.status_code == 200 + key = created.json()["key"] + + response = await async_client.post( + "/v1/embeddings", + headers={"Authorization": f"Bearer {key}"}, + json={"model": model, "input": ["hello", "world"], "encoding_format": "float"}, + ) + + assert response.status_code == 200 + body = response.json() + assert body["object"] == "list" + assert body["data"][0]["embedding"] == [0.1, 0.2, 0.3] + assert captured["path"] == "/v1/embeddings" + assert captured["authorization"] == "Bearer token-embedder" + # Extra OpenAI params pass through verbatim. + assert captured["payload"] == { + "model": model, + "input": ["hello", "world"], + "encoding_format": "float", + } + + async with SessionLocal() as session: + result = await session.execute(select(RequestLog).where(RequestLog.model == model)) + log = result.scalar_one() + assert log.account_id is None + assert log.model_source_id == source_id + assert log.source == "model_source" + assert log.input_tokens == 21 + assert log.output_tokens == 0 + assert log.status == "success" + + +@pytest.mark.asyncio +async def test_source_embeddings_unknown_model_returns_model_not_found(async_client) -> None: + await _enable_api_key_auth(async_client) + created = await async_client.post("/api/api-keys/", json={"name": "embeddings-404-key"}) + assert created.status_code == 200 + key = created.json()["key"] + + response = await async_client.post( + "/v1/embeddings", + headers={"Authorization": f"Bearer {key}"}, + json={"model": "no-such-embedder", "input": "hello"}, + ) + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "model_not_found" + + # Rejection happens before source selection succeeds, so no source was + # contacted: the attempt must not appear in the request log at all. + async with SessionLocal() as session: + result = await session.execute(select(RequestLog).where(RequestLog.model == "no-such-embedder")) + assert result.scalars().all() == [] + + +@pytest.mark.asyncio +async def test_source_embeddings_transport_failure_logs_without_upstream_status(async_client) -> None: + await _enable_api_key_auth(async_client) + model = "unreachable-embedder" + closed_port = _free_port() + source_id = await _create_model_source( + async_client, + name="unreachable-embedder-source", + model=model, + base_url=f"http://127.0.0.1:{closed_port}/v1", + supports_embeddings=True, + ) + created = await async_client.post( + "/api/api-keys/", + json={"name": "embeddings-unreachable-key", "assignedSourceIds": [source_id]}, + ) + assert created.status_code == 200 + key = created.json()["key"] + + response = await async_client.post( + "/v1/embeddings", + headers={"Authorization": f"Bearer {key}"}, + json={"model": model, "input": "hello"}, + ) + + assert response.status_code == 502 + assert response.json()["error"]["code"] == "model_source_unreachable" + + # The attempt reached dispatch, so it is logged -- but no upstream response + # ever arrived, so there is no upstream status code to carry. + async with SessionLocal() as session: + result = await session.execute(select(RequestLog).where(RequestLog.model == model)) + log = result.scalar_one() + assert log.status == "error" + assert log.model_source_id == source_id + assert log.upstream_status_code is None + + +@pytest.mark.asyncio +async def test_source_embeddings_upstream_error_passes_through_and_logs(async_client, source_upstream) -> None: + await _enable_api_key_auth(async_client) + + async def embed(request: web.Request) -> web.Response: + return web.json_response( + {"error": {"message": "model exploded", "type": "server_error"}}, + status=500, + ) + + base_url = await source_upstream(embed) + model = "broken-embedder" + source_id = await _create_model_source( + async_client, + name="broken-embedder-source", + model=model, + base_url=base_url, + supports_embeddings=True, + ) + created = await async_client.post( + "/api/api-keys/", + json={"name": "embeddings-error-key", "assignedSourceIds": [source_id]}, + ) + assert created.status_code == 200 + key = created.json()["key"] + + response = await async_client.post( + "/v1/embeddings", + headers={"Authorization": f"Bearer {key}"}, + json={"model": model, "input": "hello"}, + ) + + assert response.status_code == 500 + assert "model exploded" in response.json()["error"]["message"] + + async with SessionLocal() as session: + result = await session.execute(select(RequestLog).where(RequestLog.model == model)) + log = result.scalar_one() + assert log.status == "error" + assert log.model_source_id == source_id + + +@pytest.mark.asyncio +async def test_source_embeddings_without_usage_fails_closed_for_limited_key(async_client, source_upstream) -> None: + await _enable_api_key_auth(async_client) + + async def embed(request: web.Request) -> web.Response: + return web.json_response( + { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.5]}], + "model": "usage-less-embedder", + } + ) + + base_url = await source_upstream(embed) + model = "usage-less-embedder" + source_id = await _create_model_source( + async_client, + name="usage-less-embedder-source", + model=model, + base_url=base_url, + supports_embeddings=True, + ) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "embeddings-limited-key", + "assignedSourceIds": [source_id], + "limits": [ + {"limitType": "total_tokens", "limitWindow": "weekly", "maxValue": 1_000}, + ], + }, + ) + assert created.status_code == 200 + key = created.json()["key"] + + response = await async_client.post( + "/v1/embeddings", + headers={"Authorization": f"Bearer {key}"}, + json={"model": model, "input": "hello"}, + ) + + assert response.status_code == 502 + assert response.json()["error"]["code"] == "usage_unavailable" diff --git a/tests/integration/test_oauth_flow.py b/tests/integration/test_oauth_flow.py index 067d93d1e7..a09058609c 100644 --- a/tests/integration/test_oauth_flow.py +++ b/tests/integration/test_oauth_flow.py @@ -43,6 +43,50 @@ def _oauth_flow_schema(db_setup): del db_setup +async def _drain_global_oauth_store() -> None: + """Reset the module-global OAuth store AND await its tasks to completion. + + ``_OAUTH_STORE.reset()`` cancels poll tasks but does not await them, so a + cancelled (or still-pending) device poller can keep running into the next + test on the shared session loop -- exchanging the device code against + whatever ``exchange_device_token`` is monkeypatched to at that moment (or + the real network client) and committing slot/status writes to the next + test's freshly reset database (issue #1794, same family as #1755). Awaiting + every task here guarantees nothing owned by the store crosses a test + boundary, and retrieves cancelled tasks' exceptions so they cannot surface + as unrelated "Task exception was never retrieved" noise in a later test. + """ + + store = oauth_module._OAUTH_STORE + async with store.lock: + tasks = [ + flow.poll_task for flow in store._flows.values() if flow.poll_task is not None and not flow.poll_task.done() + ] + stop_task = store._callback_server_stop_task + if stop_task is not None and not stop_task.done(): + tasks.append(stop_task) + await store.reset() + for task in tasks: + task.cancel() + with contextlib.suppress(Exception, asyncio.CancelledError): + await task + + +@pytest.fixture(autouse=True) +async def _isolate_global_oauth_store(): + """Fence ``oauth_module._OAUTH_STORE`` at BOTH edges of every test. + + The per-test ``_OAUTH_STORE.reset()`` calls this fixture replaces only ran + at each test's start, so whichever test happened to run next inherited the + previous test's live poll tasks for its whole fixture setup -- the + order-dependent "different victim per run" flake of issue #1794. + """ + + await _drain_global_oauth_store() + yield + await _drain_global_oauth_store() + + def _encode_jwt(payload: dict) -> str: raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") body = base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") @@ -104,7 +148,6 @@ async def manual_callback(self, callback_url: str, flow_id: str | None = None): @pytest.mark.asyncio async def test_manual_callback_service_sanitizes_unexpected_exception(monkeypatch, caplog): - await oauth_module._OAUTH_STORE.reset() caplog.set_level(logging.ERROR, logger=oauth_module.logger.name) # Persist the flow durably (real flows are written to the shared DB at start) # so the reconciliation gate keeps it rather than dropping it as stale. @@ -162,8 +205,6 @@ def test_oauth_error_html_escapes_message(): @pytest.mark.asyncio async def test_device_oauth_flow_creates_account(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - email = "device@example.com" raw_account_id = "acc_device" @@ -220,7 +261,6 @@ async def fake_sleep(_: float) -> None: @pytest.mark.asyncio async def test_starting_new_device_flow_cancels_previous_pending_poll(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() issued = 0 first_poll_started = asyncio.Event() first_poll_cancelled = asyncio.Event() @@ -276,8 +316,6 @@ async def fake_exchange_device_token(*, device_auth_id: str, **_): await asyncio.wait_for(first_poll_cancelled.wait(), timeout=1) assert first_task.cancelled() - await oauth_module._OAUTH_STORE.reset() - @pytest.mark.asyncio async def test_device_oauth_reauth_reuses_existing_row_for_same_chatgpt_identity( @@ -299,8 +337,6 @@ async def test_device_oauth_reauth_reuses_existing_row_for_same_chatgpt_identity new tokens onto its historical row instead of forking a duplicate. """ - await oauth_module._OAUTH_STORE.reset() - settings = await async_client.put( "/api/settings", json={ @@ -352,6 +388,7 @@ async def _run_device_flow_once() -> None: start = await async_client.post("/api/oauth/start", json={"forceMethod": "device"}) assert start.status_code == 200 assert start.json()["method"] == "device" + flow_id = start.json()["flowId"] complete = await async_client.post("/api/oauth/complete", json={}) assert complete.status_code == 200 @@ -359,9 +396,13 @@ async def _run_device_flow_once() -> None: await asyncio.sleep(0) + # Poll THIS flow's status (like the dashboard does): the flowId-less + # endpoint reads the store's "latest flow" pointer, which a finishing + # neighbor poller's cleanup can re-latch to its own already-successful + # flow -- reporting success before this flow's poller persisted. payload = None for _ in range(20): - status = await async_client.get("/api/oauth/status") + status = await async_client.get("/api/oauth/status", params={"flowId": flow_id}) assert status.status_code == 200 payload = status.json() if payload["status"] == "success": @@ -388,8 +429,6 @@ async def test_device_oauth_flow_heals_deactivated_account_when_import_without_o async_client, monkeypatch, ): - await oauth_module._OAUTH_STORE.reset() - settings = await async_client.put( "/api/settings", json={ @@ -834,8 +873,6 @@ async def test_device_oauth_flow_keeps_same_email_distinct_upstream_identities_i async_client, monkeypatch, ): - await oauth_module._OAUTH_STORE.reset() - enable_separate = await async_client.put( "/api/settings", json={ @@ -901,6 +938,7 @@ async def _run_device_flow_once() -> dict[str, str | None]: start = await async_client.post("/api/oauth/start", json={"forceMethod": "device"}) assert start.status_code == 200 assert start.json()["method"] == "device" + flow_id = start.json()["flowId"] complete = await async_client.post("/api/oauth/complete", json={}) assert complete.status_code == 200 @@ -908,9 +946,11 @@ async def _run_device_flow_once() -> dict[str, str | None]: await asyncio.sleep(0) + # Poll THIS flow's status; see the reauth test above for why the + # flowId-less "latest flow" endpoint is racy across sequential flows. payload: dict[str, str | None] | None = None for _ in range(20): - status = await async_client.get("/api/oauth/status") + status = await async_client.get("/api/oauth/status", params={"flowId": flow_id}) assert status.status_code == 200 payload = status.json() if payload["status"] in {"success", "error"}: @@ -949,8 +989,6 @@ async def _run_device_flow_once() -> dict[str, str | None]: @pytest.mark.asyncio async def test_oauth_start_with_existing_account_marks_success(async_client): - await oauth_module._OAUTH_STORE.reset() - encryptor = TokenEncryptor() account = Account( id="acc_existing", @@ -978,8 +1016,6 @@ async def test_oauth_start_with_existing_account_marks_success(async_client): @pytest.mark.asyncio async def test_oauth_start_with_existing_account_clears_stale_flows(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1025,8 +1061,6 @@ async def fake_callback_server_start(self) -> None: @pytest.mark.asyncio async def test_terminal_oauth_flows_are_bounded_outside_full_reset(): - await oauth_module._OAUTH_STORE.reset() - retained_limit = oauth_module._MAX_RETAINED_TERMINAL_OAUTH_FLOWS async with oauth_module._OAUTH_STORE.lock: @@ -1056,8 +1090,6 @@ async def test_terminal_oauth_flows_are_bounded_outside_full_reset(): @pytest.mark.asyncio async def test_expired_pending_browser_oauth_flows_are_pruned(): - await oauth_module._OAUTH_STORE.reset() - now = time.time() async with oauth_module._OAUTH_STORE.lock: expired = oauth_module.OAuthState( @@ -1087,8 +1119,6 @@ async def test_expired_pending_browser_oauth_flows_are_pruned(): @pytest.mark.asyncio async def test_only_expired_pending_browser_flow_no_longer_keeps_callback_server_alive(): - await oauth_module._OAUTH_STORE.reset() - async with oauth_module._OAUTH_STORE.lock: flow = oauth_module.OAuthState( flow_id="expired-flow", @@ -1107,7 +1137,6 @@ async def test_only_expired_pending_browser_flow_no_longer_keeps_callback_server @pytest.mark.asyncio async def test_callback_server_remains_reserved_until_stop_completes(): - await oauth_module._OAUTH_STORE.reset() stop_started = asyncio.Event() release_stop = asyncio.Event() @@ -1133,8 +1162,6 @@ async def stop(self) -> None: @pytest.mark.asyncio async def test_oauth_start_falls_back_to_device_on_os_error(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_browser_flow(self): raise OSError("no port") @@ -1147,8 +1174,15 @@ async def fake_device_code(**_): expires_in_seconds=30, ) + # Park the spawned device poller instead of letting it hit the REAL token + # exchange client (this test only asserts the browser->device fallback); + # the autouse store fence cancels and awaits it at teardown. + async def fake_exchange_device_token(**_): + await asyncio.Event().wait() + monkeypatch.setattr(oauth_module.OauthService, "_start_browser_flow", fake_browser_flow) monkeypatch.setattr(oauth_module, "request_device_code", fake_device_code) + monkeypatch.setattr(oauth_module, "exchange_device_token", fake_exchange_device_token) start = await async_client.post("/api/oauth/start", json={}) assert start.status_code == 200 @@ -1159,8 +1193,6 @@ async def fake_device_code(**_): @pytest.mark.asyncio async def test_device_oauth_flow_reports_proxy_route_errors(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_oauth_route(*_args, **_kwargs): raise UpstreamProxyRouteError("default_pool_unconfigured", account_id=None) @@ -1174,8 +1206,6 @@ async def fake_oauth_route(*_args, **_kwargs): @pytest.mark.asyncio async def test_manual_callback_returns_success_and_creates_account(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1227,8 +1257,6 @@ async def fake_exchange_authorization_code(**_): @pytest.mark.asyncio async def test_manual_callback_returns_error_message_for_invalid_state(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1262,8 +1290,6 @@ async def fake_callback_server_start(self) -> None: @pytest.mark.asyncio async def test_oauth_status_binds_camel_case_flow_id(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1309,8 +1335,6 @@ async def fake_callback_server_start(self) -> None: @pytest.mark.asyncio async def test_manual_callback_error_resolves_state_before_marking_flow_failed(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1377,7 +1401,6 @@ async def fake_callback_server_start(self) -> None: @pytest.mark.asyncio async def test_unknown_flow_error_does_not_mutate_latest_oauth_status(): - await oauth_module._OAUTH_STORE.reset() async with SessionLocal() as session: service = oauth_module.OauthService(AccountsRepository(session)) @@ -1407,7 +1430,6 @@ async def test_unknown_flow_error_does_not_mutate_latest_oauth_status(): @pytest.mark.asyncio async def test_missing_flow_error_does_not_mutate_latest_oauth_status(): - await oauth_module._OAUTH_STORE.reset() async with SessionLocal() as session: service = oauth_module.OauthService(AccountsRepository(session)) @@ -1437,8 +1459,6 @@ async def test_missing_flow_error_does_not_mutate_latest_oauth_status(): @pytest.mark.asyncio async def test_manual_callback_unknown_state_does_not_mutate_latest_flow(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1467,8 +1487,6 @@ async def fake_callback_server_start(self) -> None: @pytest.mark.asyncio async def test_concurrent_browser_oauth_flows_keep_callbacks_isolated(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1547,7 +1565,6 @@ async def fake_exchange_authorization_code(**kwargs): @pytest.mark.asyncio async def test_callback_server_idle_stop_releases_store_lock_before_cleanup(): - await oauth_module._OAUTH_STORE.reset() async with SessionLocal() as session: service = oauth_module.OauthService(AccountsRepository(session)) @@ -1575,8 +1592,6 @@ async def stop(self) -> None: @pytest.mark.asyncio async def test_existing_account_cleanup_releases_store_lock_before_callback_server_stop(): - await oauth_module._OAUTH_STORE.reset() - class ExistingAccountRepo: async def list_accounts(self): return [object()] @@ -1607,7 +1622,6 @@ async def stop(self) -> None: @pytest.mark.asyncio async def test_new_browser_flow_waits_for_stopping_callback_server_before_reusing_slot(monkeypatch): - await oauth_module._OAUTH_STORE.reset() stop_started = asyncio.Event() release_stop = asyncio.Event() started_servers: list[object] = [] @@ -1653,8 +1667,6 @@ async def stop(self) -> None: @pytest.mark.asyncio async def test_manual_callback_idempotent_success_requires_requested_flow(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -2059,7 +2071,6 @@ async def test_device_complete_ack_stays_pending_when_own_poller_already_succeed and must NOT spawn a second poll of the consumed device code. """ - await oauth_module._OAUTH_STORE.reset() async with SessionLocal() as session: service = oauth_module.OauthService(AccountsRepository(session)) diff --git a/tests/integration/test_openai_client_compat.py b/tests/integration/test_openai_client_compat.py index 984d3bbee0..d63ec851d2 100644 --- a/tests/integration/test_openai_client_compat.py +++ b/tests/integration/test_openai_client_compat.py @@ -4,6 +4,7 @@ import json import httpx +import httpx2 import openai import pytest from httpx import ASGITransport @@ -52,7 +53,9 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, response = await admin_client.post("/api/accounts/import", files=files) assert response.status_code == 200 - async with httpx.AsyncClient(transport=transport, base_url="http://testserver/v1") as http_client: + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=app_instance), base_url="http://testserver/v1" + ) as http_client: client = openai.AsyncOpenAI(api_key="test", base_url="http://testserver/v1", http_client=http_client) result = await client.responses.create(model="gpt-5.1", input="hi") @@ -91,8 +94,8 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, response = await admin_client.post("/api/accounts/import", files=files) assert response.status_code == 200 - async with httpx.AsyncClient( - transport=transport, + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=app_instance), base_url="http://testserver/backend-api/codex", ) as http_client: client = openai.AsyncOpenAI( @@ -127,7 +130,9 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, response = await admin_client.post("/api/accounts/import", files=files) assert response.status_code == 200 - async with httpx.AsyncClient(transport=transport, base_url="http://testserver/v1") as http_client: + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=app_instance), base_url="http://testserver/v1" + ) as http_client: client = openai.AsyncOpenAI(api_key="test", base_url="http://testserver/v1", http_client=http_client) result = await client.chat.completions.create( model="gpt-5.2", diff --git a/tests/integration/test_openai_compat_features.py b/tests/integration/test_openai_compat_features.py index c3c44dd615..586c1b5ae9 100644 --- a/tests/integration/test_openai_compat_features.py +++ b/tests/integration/test_openai_compat_features.py @@ -299,7 +299,9 @@ async def test_v1_responses_preserves_explicit_prompt_cache_for_model_source(asy async def fake_select(model, api_key, *, raw_model=None, require_streaming=False): return source, model - async def fake_source_response(request, payload, *, source, api_key, rate_limit_headers): + async def fake_source_response( + request, payload, *, source, api_key, rate_limit_headers, pre_normalization_effort=None + ): seen["payload"] = payload.model_dump_for_forwarding() return JSONResponse({"id": "resp_prompt_cache_source", "status": "completed", "output": []}) diff --git a/tests/integration/test_proxy_api_extended.py b/tests/integration/test_proxy_api_extended.py index 44b00dee0d..39fe8d5c52 100644 --- a/tests/integration/test_proxy_api_extended.py +++ b/tests/integration/test_proxy_api_extended.py @@ -227,6 +227,13 @@ async def test_openapi_operation_ids_are_unique_and_thread_goal_methods_stable(a assert thread_goal["get"]["operationId"] == "thread_goal_get_backend_api_codex_thread_goal_get_get" assert thread_goal["post"]["operationId"] == "thread_goal_get_backend_api_codex_thread_goal_get_post" + assert schema["paths"]["/v1/responses"]["post"]["requestBody"]["content"]["application/json"]["schema"] == { + "$ref": "#/components/schemas/V1ResponsesRequest" + } + assert schema["paths"]["/backend-api/codex/responses/compact"]["post"]["requestBody"]["content"][ + "application/json" + ]["schema"] == {"$ref": "#/components/schemas/ResponsesCompactRequest"} + @pytest.mark.asyncio async def test_proxy_compact_not_implemented(async_client, monkeypatch): @@ -2965,6 +2972,66 @@ async def stream_responses(self, *args, **kwargs): assert any("response.completed" in chunk for chunk in chunks) +@pytest.mark.asyncio +async def test_codex_route_stream_responses_keeps_client_alive_while_bridge_cooldown_delays_first_event( + monkeypatch, +): + upstream_started = asyncio.Event() + release_upstream = asyncio.Event() + + class _FakeService: + async def rate_limit_headers(self): + return {} + + async def stream_responses(self, *args, **kwargs): + del args, kwargs + upstream_started.set() + _signal_propagated_capacity_startup_ready() + await release_upstream.wait() + yield _sse_event({"type": "response.in_progress", "response": {"id": "resp_cooldown_wait"}}) + yield _sse_event({"type": "response.completed", "response": {"id": "resp_cooldown_wait"}}) + + settings = SimpleNamespace( + http_responses_session_bridge_enabled=False, + sse_keepalive_interval_seconds=0.01, + proxy_account_stream_recovery_reserve=1, + proxy_api_key_fair_share_congestion_threshold_pct=0, + ) + monkeypatch.setattr(proxy_api_module, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_api_module.proxy_service_module, "get_settings", lambda: settings) + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/backend-api/codex/responses", + "headers": [], + } + ) + payload = proxy_api_module.ResponsesRequest.model_validate( + {"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True} + ) + + response = await proxy_api_module._stream_responses( + request, + payload, + ProxyContext(service=cast(proxy_module.ProxyService, _FakeService())), + api_key=None, + enforce_openai_sdk_contract=False, + ) + + assert isinstance(response, StreamingResponse) + assert upstream_started.is_set() is True + iterator = response.body_iterator.__aiter__() + first_chunk = await asyncio.wait_for(iterator.__anext__(), timeout=0.2) + assert first_chunk == CODEX_KEEPALIVE_FRAME + release_upstream.set() + second_chunk = cast(str, await asyncio.wait_for(iterator.__anext__(), timeout=0.2)) + third_chunk = cast(str, await asyncio.wait_for(iterator.__anext__(), timeout=0.2)) + assert "response.in_progress" in second_chunk + assert "response.completed" in third_chunk + + @pytest.mark.asyncio async def test_proxy_stream_retries_rate_limit_then_success(async_client, monkeypatch): expected_account_id_1 = await _import_account(async_client, "acc_1", "one@example.com") @@ -3056,6 +3123,10 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert by_account[expected_account_id_1].error_code == "stream_idle_timeout" assert by_account[expected_account_id_2].status == "success" + service = get_proxy_service_for_app(async_client._transport.app) + idle_runtime = service._load_balancer._runtime.get(expected_account_id_1) + assert idle_runtime is None or idle_runtime.error_count == 0 + @pytest.mark.asyncio async def test_proxy_stream_drops_forwarded_headers(async_client, monkeypatch): diff --git a/tests/integration/test_proxy_chat_completions.py b/tests/integration/test_proxy_chat_completions.py index 3a5c1dfb4e..f50724af29 100644 --- a/tests/integration/test_proxy_chat_completions.py +++ b/tests/integration/test_proxy_chat_completions.py @@ -4,10 +4,13 @@ import json import pytest +from sqlalchemy import select from starlette.responses import JSONResponse import app.modules.proxy.api as proxy_api import app.modules.proxy.service as proxy_module +from app.db.models import ApiKeyUsageReservation +from app.db.session import SessionLocal pytestmark = pytest.mark.integration @@ -57,6 +60,88 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert any("chat.completion.chunk" in line for line in lines) +@pytest.mark.asyncio +async def test_v1_chat_completions_stream_truncated_eof_emits_error_and_done(async_client, monkeypatch): + # #given + email = "chat-stream-truncated@example.com" + raw_account_id = "acc_chat_stream_truncated" + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + imported = await async_client.post("/api/accounts/import", files=files) + assert imported.status_code == 200 + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False): + del payload, headers, access_token, account_id, base_url, raise_for_status + yield 'data: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + + # #when + payload = {"model": "gpt-5.2", "messages": [{"role": "user", "content": "hi"}], "stream": True} + async with async_client.stream("POST", "/v1/chat/completions", json=payload) as response: + assert response.status_code == 200 + lines = [line async for line in response.aiter_lines() if line] + + # #then + assert json.loads(lines[-2][len("data: ") :])["error"]["code"] == "upstream_stream_truncated" + assert lines[-1] == "data: [DONE]" + + +@pytest.mark.asyncio +async def test_v1_chat_completions_stream_terminal_error_without_payload_uses_default_error( + async_client, + monkeypatch, +): + email = "chat-stream-terminal-error@example.com" + raw_account_id = "acc_chat_stream_terminal_error" + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + imported = await async_client.post("/api/accounts/import", files=files) + assert imported.status_code == 200 + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False): + del payload, headers, access_token, account_id, base_url, raise_for_status + yield 'data: {"type":"response.output_text.delta","delta":"partial"}\n\n' + yield 'data: {"type":"response.failed","response":{"id":"r1","status":"failed"}}\n\n' + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + + payload = {"model": "gpt-5.2", "messages": [{"role": "user", "content": "hi"}], "stream": True} + async with async_client.stream("POST", "/v1/chat/completions", json=payload) as response: + assert response.status_code == 200 + lines = [line async for line in response.aiter_lines() if line] + + assert json.loads(lines[-2][len("data: ") :])["error"]["code"] == "upstream_error" + assert lines[-1] == "data: [DONE]" + + +@pytest.mark.asyncio +async def test_v1_chat_completions_omits_synthesized_tools(async_client, monkeypatch): + email = "chatnotools@example.com" + raw_account_id = "acc_chatnotools" + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + response = await async_client.post("/api/accounts/import", files=files) + assert response.status_code == 200 + + seen_payload: dict[str, object] = {} + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False, **kwargs): + del headers, access_token, account_id, base_url, raise_for_status, kwargs + seen_payload.update(payload.to_payload()) + yield 'data: {"type":"response.output_text.delta","delta":"hi"}\n\n' + yield 'data: {"type":"response.completed","response":{"id":"resp_chat_no_tools"}}\n\n' + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + + payload = {"model": "gpt-5.2", "messages": [{"role": "user", "content": "hi"}], "stream": True} + async with async_client.stream("POST", "/v1/chat/completions", json=payload) as resp: + assert resp.status_code == 200 + _ = [line async for line in resp.aiter_lines() if line] + + assert "tools" not in seen_payload + + @pytest.mark.asyncio async def test_v1_chat_completions_opportunistic_denial_runs_before_api_key_reservation(async_client, monkeypatch): settings = await async_client.put( @@ -181,6 +266,104 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert body["object"] == "chat.completion" +@pytest.mark.asyncio +async def test_v1_chat_completions_non_stream_truncated_eof_returns_502(async_client, monkeypatch): + # #given + email = "chat-nonstr-truncated@example.com" + raw_account_id = "acc_chat_nonstr_truncated" + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + imported = await async_client.post("/api/accounts/import", files=files) + assert imported.status_code == 200 + + async def passthrough_probe(stream, **_kwargs): + return stream, None + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False): + del payload, headers, access_token, account_id, base_url, raise_for_status + yield 'data: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + monkeypatch.setattr(proxy_api, "_probe_chat_stream_startup_error", passthrough_probe) + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + + # #when + response = await async_client.post( + "/v1/chat/completions", + json={"model": "gpt-5.2", "messages": [{"role": "user", "content": "hi"}]}, + ) + + # #then + assert response.status_code == 502 + body = response.json() + assert body["error"]["code"] == "upstream_stream_truncated" + assert body["error"]["type"] == "server_error" + + +@pytest.mark.asyncio +async def test_v1_chat_completions_non_stream_rate_limit_closes_stream_and_returns_429(async_client, monkeypatch): + # #given + email = "chat-nonstr-429@example.com" + raw_account_id = "acc_chat_nonstr_429" + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + imported = await async_client.post("/api/accounts/import", files=files) + assert imported.status_code == 200 + + settings = await async_client.put( + "/api/settings", + json={ + "stickyThreadsEnabled": False, + "preferEarlierResetAccounts": False, + "totpRequiredOnLogin": False, + "apiKeyAuthEnabled": True, + }, + ) + assert settings.status_code == 200 + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "chat-nonstr-429", + "limits": [{"limitType": "total_tokens", "limitWindow": "weekly", "maxValue": 100_000}], + }, + ) + assert created.status_code == 200 + key = created.json()["key"] + key_id = created.json()["id"] + + async def passthrough_probe(stream, **_kwargs): + return stream, None + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False): + del payload, headers, access_token, account_id, base_url, raise_for_status + yield ( + 'data: {"type":"response.failed","response":{"error":' + '{"message":"limit","type":"rate_limit_error","code":"rate_limit_exceeded"}}}\n\n' + ) + + monkeypatch.setattr(proxy_api, "_probe_chat_stream_startup_error", passthrough_probe) + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + + # #when + response = await async_client.post( + "/v1/chat/completions", + headers={"Authorization": f"Bearer {key}"}, + json={"model": "gpt-5.2", "messages": [{"role": "user", "content": "hi"}]}, + ) + + # #then + assert response.status_code == 429 + body = response.json() + assert body["error"]["code"] == "rate_limit_exceeded" + async with SessionLocal() as session: + reservations = ( + (await session.execute(select(ApiKeyUsageReservation).where(ApiKeyUsageReservation.api_key_id == key_id))) + .scalars() + .all() + ) + assert len(reservations) == 1 + assert reservations[0].status == "released" + + @pytest.mark.asyncio async def test_v1_chat_completions_non_stream_deduplicates_tool_call_snapshots(async_client, monkeypatch): email = "chat-tool-snapshot@example.com" diff --git a/tests/integration/test_proxy_compact.py b/tests/integration/test_proxy_compact.py index bb6b2fb151..6bb285ca26 100644 --- a/tests/integration/test_proxy_compact.py +++ b/tests/integration/test_proxy_compact.py @@ -1,8 +1,8 @@ from __future__ import annotations -import base64 import contextlib import json +import logging from datetime import timedelta, timezone from typing import cast from unittest.mock import AsyncMock @@ -22,8 +22,10 @@ from app.db.session import SessionLocal from app.modules.api_keys.repository import ApiKeysRepository from app.modules.api_keys.service import ApiKeyCreateData, ApiKeysService +from app.modules.proxy.account_cache import get_account_selection_cache from app.modules.proxy.rate_limit_cache import get_rate_limit_headers_cache from app.modules.usage.repository import AdditionalUsageRepository, UsageRepository +from tests.integration.compact_test_helpers import _make_auth_json pytestmark = pytest.mark.integration @@ -34,9 +36,9 @@ async def test_proxy_compact_forwarded_bridge_settlement_failure_surfaces_code_a monkeypatch, ): """A forwarded owner must not report compact success when its sole API-key - usage settlement fails. The `usage_settlement_failed` error is surfaced - without another upstream or account-health attempt, and a fresh repository - releases the held quota.""" + usage settlement fails. After cleanup-ready, the receiver keeps HTTP 200 + and surfaces `usage_settlement_failed` on the SSE body so origin cannot + replay. A fresh repository still releases the held quota.""" from app.core.config.settings import get_settings from app.core.openai.requests import ResponsesCompactRequest, ResponsesRequest from app.db.models import ApiKeyUsageReservation @@ -84,6 +86,7 @@ async def test_proxy_compact_forwarded_bridge_settlement_failure_surfaces_code_a request_service_tier=None, request_usage_budget=estimate_api_key_request_usage(compact_model), ) + assert reservation is not None async with SessionLocal() as session: row = await session.get(ApiKeyUsageReservation, reservation.reservation_id) assert row is not None @@ -156,8 +159,10 @@ async def fail_finalize(self, reservation_id: str, **kwargs: object) -> None: headers=headers, ) - assert response.status_code == 502, response.text - assert response.json()["error"]["code"] == "usage_settlement_failed" + assert response.status_code == 200, response.text + assert "text/event-stream" in response.headers.get("content-type", "") + assert "response.failed" in response.text + assert "usage_settlement_failed" in response.text assert compact_calls == [raw_account_id] assert finalize_attempts == [reservation.reservation_id] handle_stream_error.assert_not_awaited() @@ -168,28 +173,6 @@ async def fail_finalize(self, reservation_id: str, **kwargs: object) -> None: assert row.status == "released" -def _encode_jwt(payload: dict) -> str: - raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") - body = base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") - return f"header.{body}.sig" - - -def _make_auth_json(account_id: str, email: str, *, plan_type: str = "plus") -> dict: - payload = { - "email": email, - "chatgpt_account_id": account_id, - "https://api.openai.com/auth": {"chatgpt_plan_type": plan_type}, - } - return { - "tokens": { - "idToken": _encode_jwt(payload), - "accessToken": "access-token", - "refreshToken": "refresh-token", - "accountId": account_id, - }, - } - - class _JsonResponse: def __init__(self, payload: dict[str, object]) -> None: self.status = 200 @@ -212,8 +195,39 @@ async def _return_self(): return _return_self().__await__() +class _SseContent: + async def iter_chunked(self, size: int): + del size + yield ( + b'data: {"type":"response.output_item.done","output_index":0,' + b'"item":{"id":"msg_compact_summary_1","type":"message","role":"assistant",' + b'"status":"completed","content":[{"type":"output_text","text":"enc_compact_summary_1"}]}}\n\n' + b'data: {"type":"response.completed","response":' + b'{"object":"response","id":"resp_compact_summary_1","status":"completed","output":[]}}\n\n' + ) + + +class _SseResponse: + status = 200 + reason = "OK" + headers: dict[str, str] = {"content-type": "text/event-stream"} + content = _SseContent() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + def __await__(self): + async def _return_self(): + return self + + return _return_self().__await__() + + class _JsonSession: - def __init__(self, response: _JsonResponse) -> None: + def __init__(self, response: object) -> None: self._response = response self.calls: list[dict[str, object]] = [] @@ -762,17 +776,7 @@ async def test_proxy_compact_success_preserves_compaction_payload(async_client, response = await async_client.post("/api/accounts/import", files=files) assert response.status_code == 200 - session = _JsonSession( - _JsonResponse( - { - "object": "response.compaction", - "compaction_summary": { - "encrypted_content": "enc_compact_summary_1", - "summary_text": "condensed thread state", - }, - } - ) - ) + session = _JsonSession(_SseResponse()) @contextlib.asynccontextmanager async def lease_session(session_override=None): @@ -787,14 +791,20 @@ async def lease_session(session_override=None): assert response.status_code == 200 body = response.json() assert body["object"] == "response.compaction" - assert body["compaction_summary"] == { - "encrypted_content": "enc_compact_summary_1", - "summary_text": "condensed thread state", - } - assert _session_call_url(session).endswith("/codex/responses/compact") + assert body["id"] == "resp_compact_summary_1" + assert body["output"] == [ + { + "type": "compaction", + "status": "completed", + "encrypted_content": "enc_compact_summary_1", + } + ] + assert _session_call_url(session).endswith("/codex/responses") call_json = _session_call_json(session) - assert "stream" not in call_json - assert "store" not in call_json + assert call_json["stream"] is True + assert call_json["store"] is False + call_headers = cast(dict[str, str], session.calls[0]["headers"]) + assert call_headers["Accept"] == "text/event-stream" @pytest.mark.asyncio @@ -1377,8 +1387,9 @@ async def test_proxy_compact_forwarded_bridge_preflight_budget_exhausted_settles reaches the OWNER instance via the internal bridge forward — where ``owns_reservation`` is false so ``compact_responses`` is the SOLE settler — and whose preflight budget is exhausted MUST settle (release) the API-key - usage reservation before raising the ``502 upstream_request_timeout``, so - held API-key quota is not leaked. + usage reservation before the forwarded stream emits the terminal + ``response.failed`` / ``upstream_request_timeout`` event, so held API-key + quota is not leaked. This drives the REAL external surface, not a handcrafted service call: it POSTs a signed forwarded request to the internal bridge endpoint @@ -1390,13 +1401,15 @@ async def test_proxy_compact_forwarded_bridge_preflight_budget_exhausted_settles terminal ``compaction_trigger`` and calls ``compact_responses`` with ``owns_reservation`` false — so ``_compact_or_stream_responses``'s ``finally`` does NOT release the reservation and ``compact_responses`` alone must settle - it. Pre-fix the budget-exhausted terminal raised via - ``_raise_proxy_budget_exhausted`` without settling (through the outer - ``except ProxyResponseError`` handler and the log-only ``finally``), leaving - the reservation row ``reserved`` (leaked held quota); post-fix the row is - ``released``. PR #1254 fixed the sibling transport-failure / permanent-refresh - preflight raises but left the budget-exhausted terminal out of scope; this - completes that invariant. + it. On this forwarded streaming surface the owner reports the failure as the + terminal SSE event rather than a direct JSON ``502`` envelope, but the + settlement invariant is the same: pre-fix the budget-exhausted terminal + raised via ``_raise_proxy_budget_exhausted`` without settling (through the + outer ``except ProxyResponseError`` handler and the log-only ``finally``), + leaving the reservation row ``reserved`` (leaked held quota); post-fix the + row is ``released``. PR #1254 fixed the sibling transport-failure / + permanent-refresh preflight raises but left the budget-exhausted terminal + out of scope; this completes that invariant. """ import app.modules.proxy._service.compact as compact_module from app.core.config.settings import get_settings @@ -1452,6 +1465,7 @@ async def test_proxy_compact_forwarded_bridge_preflight_budget_exhausted_settles request_service_tier=None, request_usage_budget=estimate_api_key_request_usage(compact_model), ) + assert reservation is not None async with SessionLocal() as session: row = await session.get(ApiKeyUsageReservation, reservation.reservation_id) assert row is not None @@ -1498,18 +1512,20 @@ async def test_proxy_compact_forwarded_bridge_preflight_budget_exhausted_settles headers=headers, ) - # Budget exhaustion surfaces as a 502 upstream_request_timeout from the owner. - assert response.status_code == 502, response.text - assert response.json()["error"]["code"] == "upstream_request_timeout" + # On the forwarded streaming surface the owner emits a terminal SSE failure + # event instead of a direct JSON 502 envelope, but it still settles the + # reservation before that failure reaches the caller. + assert response.status_code == 200, response.text + assert "text/event-stream" in response.headers.get("content-type", "") + assert "response.failed" in response.text + assert "upstream_request_timeout" in response.text - # The forwarded reservation row was RELEASED by compact_responses (sole - # settler) before the terminal raised (the fix). Pre-fix it stayed "reserved" - # — leaked held API-key quota — because owns_reservation is false on the - # forwarded path so the route's finally does not release it. async with SessionLocal() as session: row = await session.get(ApiKeyUsageReservation, reservation.reservation_id) assert row is not None - assert row.status == "released", f"forwarded reservation leaked held quota; status={row.status!r}" + assert row.status == "released", ( + f"forwarded receiver leaked the reservation after terminal SSE failure; status={row.status!r}" + ) @pytest.mark.asyncio @@ -1612,15 +1628,18 @@ async def test_proxy_compact_output_round_trips_into_followup_responses_without_ "object": "response.compaction", "output": [ { - "type": "message", + "type": "compaction", "id": "msg_compact_round_trip", - "role": "assistant", - "content": [{"type": "output_text", "text": "preserve me exactly"}], + "encrypted_content": "preserve me exactly", }, {"type": "reasoning", "encrypted_content": "enc_round_trip_state"}, ], "retained_items": [{"type": "item_reference", "id": "msg_original_round_trip"}], } + expected_compact_window = { + **compact_window, + "output": [{"type": "compaction", "encrypted_content": "preserve me exactly"}], + } seen_inputs: list[object] = [] async def fake_compact(payload, headers, access_token, account_id): @@ -1636,7 +1655,7 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, compact_payload = {"model": "gpt-5.1", "instructions": "compact", "input": []} compact_response = await async_client.post("/backend-api/codex/responses/compact", json=compact_payload) assert compact_response.status_code == 200 - assert compact_response.json() == compact_window + assert compact_response.json() == expected_compact_window stream_payload = { "model": "gpt-5.1", @@ -1647,4 +1666,389 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, response = await async_client.post("/backend-api/codex/responses", json=stream_payload) assert response.status_code == 200 - assert seen_inputs == [compact_window["output"]] + assert seen_inputs == [expected_compact_window["output"]] + + +_NEUTRAL_FULL_RESEND_INPUT: list[dict[str, object]] = [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [{"type": "output_text", "text": "hi there"}]}, + {"role": "user", "content": "please compact"}, +] + + +async def _import_account(async_client, *, email: str, raw_account_id: str) -> str: + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + response = await async_client.post("/api/accounts/import", files=files) + assert response.status_code == 200 + return generate_unique_account_id(raw_account_id, email) + + +async def _mark_account_status(account_id: str, status: AccountStatus) -> None: + async with SessionLocal() as session: + account = await session.get(Account, account_id) + assert account is not None + account.status = status + if status in (AccountStatus.RATE_LIMITED, AccountStatus.QUOTA_EXCEEDED): + account.reset_at = int(utcnow().replace(tzinfo=timezone.utc).timestamp()) + 3600 + await session.commit() + get_account_selection_cache().invalidate() + + +def _pin_previous_response_owner(monkeypatch, owner_account_id: str) -> None: + async def fake_owner(self, *, previous_response_id, api_key, session_id=None, surface, **kwargs): + del self, previous_response_id, api_key, session_id, surface, kwargs + return owner_account_id + + monkeypatch.setattr(proxy_module.ProxyService, "_resolve_websocket_previous_response_owner", fake_owner) + + +def _recording_compact( + calls: list[tuple[str | None, dict[str, object], dict[str, str]]], + *, + fail_accounts_with: dict[str, ProxyResponseError] | None = None, +): + async def fake_compact(payload, headers, access_token, account_id): + del access_token + calls.append((account_id, cast(dict[str, object], payload.to_payload()), dict(headers))) + if fail_accounts_with and account_id in fail_accounts_with: + raise fail_accounts_with[account_id] + return CompactResponsePayload.model_validate({"object": "response.compaction", "output": []}) + + return fake_compact + + +def _usage_limit_429() -> ProxyResponseError: + return ProxyResponseError( + 429, + { + "error": { + "type": "usage_limit_reached", + "message": "limit reached", + "plan_type": "plus", + "resets_at": int(utcnow().replace(tzinfo=timezone.utc).timestamp()) + 3600, + } + }, + ) + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_selection_time_quota_loss_replays_account_neutral_full_resend( + async_client, monkeypatch +): + """A previous-response-pinned compact whose owner is quota-excluded at + selection time recovers by dropping the anchor and replaying the verified + account-neutral full resend on a healthy account instead of wedging.""" + owner_account_id = await _import_account( + async_client, email="compact-quota-owner@example.com", raw_account_id="acc_quota_owner" + ) + await _import_account(async_client, email="compact-quota-alt@example.com", raw_account_id="acc_quota_alt") + await _mark_account_status(owner_account_id, AccountStatus.RATE_LIMITED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_quota_anchor", + }, + ) + + assert response.status_code == 200, response.text + assert [account_id for account_id, _payload, _headers in calls] == ["acc_quota_alt"] + replay_payload = calls[0][1] + assert "previous_response_id" not in replay_payload + assert replay_payload["input"] == _NEUTRAL_FULL_RESEND_INPUT + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_in_request_quota_429_replays_account_neutral_full_resend( + async_client, monkeypatch +): + """An owner that 429s mid-request (pre-visible quota failover) is excluded + and the verified account-neutral full resend recovers on the other account + instead of re-raising the owner's 429 forever.""" + owner_account_id = await _import_account( + async_client, email="compact-429-owner@example.com", raw_account_id="acc_429_owner" + ) + await _import_account(async_client, email="compact-429-alt@example.com", raw_account_id="acc_429_alt") + _pin_previous_response_owner(monkeypatch, owner_account_id) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr( + proxy_module, + "core_compact_responses", + _recording_compact(calls, fail_accounts_with={"acc_429_owner": _usage_limit_429()}), + ) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_429_anchor", + }, + ) + + assert response.status_code == 200, response.text + assert [account_id for account_id, _payload, _headers in calls] == ["acc_429_owner", "acc_429_alt"] + owner_payload, replay_payload = calls[0][1], calls[1][1] + assert owner_payload["previous_response_id"] == "resp_429_anchor" + assert "previous_response_id" not in replay_payload + assert replay_payload["input"] == owner_payload["input"] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_post_selection_401_stays_owner_bound(async_client, monkeypatch): + """A repeated 401 after the forced refresh excludes the owner for a + non-quota reason, so account-neutral replay must not activate and the + owner's authentication failure surfaces unchanged.""" + owner_account_id = await _import_account( + async_client, email="compact-401-owner@example.com", raw_account_id="acc_401_owner" + ) + await _import_account(async_client, email="compact-401-alt@example.com", raw_account_id="acc_401_alt") + _pin_previous_response_owner(monkeypatch, owner_account_id) + + async def fake_ensure_fresh(self, account, *, force=False, timeout_seconds=None): + del self, force, timeout_seconds + return account + + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + unauthorized = ProxyResponseError(401, openai_error("unauthorized", "token rejected")) + monkeypatch.setattr( + proxy_module, + "core_compact_responses", + _recording_compact(calls, fail_accounts_with={"acc_401_owner": unauthorized}), + ) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_401_anchor", + }, + ) + + assert response.status_code == 401 + assert [account_id for account_id, _payload, _headers in calls] == ["acc_401_owner", "acc_401_owner"] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_quota_loss_stays_owner_bound_without_retained_prior_output( + async_client, monkeypatch +): + """A multi-item history without retained assistant output cannot be proven + a full resend (it may be a delta the owner account resolves through the + anchor), so nothing is sent to another account.""" + owner_account_id = await _import_account( + async_client, email="compact-delta-owner@example.com", raw_account_id="acc_delta_owner" + ) + await _import_account(async_client, email="compact-delta-alt@example.com", raw_account_id="acc_delta_alt") + await _mark_account_status(owner_account_id, AccountStatus.RATE_LIMITED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": [ + {"role": "user", "content": "first delta turn"}, + {"role": "user", "content": "second delta turn"}, + ], + "previous_response_id": "resp_delta_anchor", + }, + ) + + assert response.status_code >= 400 + assert calls == [] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_quota_loss_stays_owner_bound_with_account_scoped_history( + async_client, monkeypatch +): + """A history carrying account-scoped state (an encrypted compaction item) + fails the shared account-neutral fresh-replay gate and never crosses + accounts.""" + owner_account_id = await _import_account( + async_client, email="compact-scoped-owner@example.com", raw_account_id="acc_scoped_owner" + ) + await _import_account(async_client, email="compact-scoped-alt@example.com", raw_account_id="acc_scoped_alt") + await _mark_account_status(owner_account_id, AccountStatus.RATE_LIMITED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": [ + {"type": "compaction", "encrypted_content": "enc_owner_scoped_state"}, + *_NEUTRAL_FULL_RESEND_INPUT, + ], + "previous_response_id": "resp_scoped_anchor", + }, + ) + + assert response.status_code >= 400 + assert calls == [] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_quota_loss_stays_owner_bound_with_session_identity(async_client, monkeypatch): + """A session identity on the request can bind live or durable HTTP-bridge + continuity that still names the lost owner; without rebinding machinery the + request stays owner-bound.""" + owner_account_id = await _import_account( + async_client, email="compact-session-owner@example.com", raw_account_id="acc_session_owner" + ) + await _import_account(async_client, email="compact-session-alt@example.com", raw_account_id="acc_session_alt") + await _mark_account_status(owner_account_id, AccountStatus.RATE_LIMITED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_session_anchor", + }, + headers={"session_id": "sid-compact-session-bound"}, + ) + + assert response.status_code >= 400 + assert calls == [] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_non_quota_loss_stays_owner_bound(async_client, monkeypatch): + """An owner unselectable for a non-quota reason (operator pause) keeps + today's fail-closed surface; recovery requires quota-caused owner loss.""" + owner_account_id = await _import_account( + async_client, email="compact-paused-owner@example.com", raw_account_id="acc_paused_owner" + ) + await _import_account(async_client, email="compact-paused-alt@example.com", raw_account_id="acc_paused_alt") + await _mark_account_status(owner_account_id, AccountStatus.PAUSED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_paused_anchor", + }, + ) + + assert response.status_code >= 400 + assert calls == [] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_policy_skip_stays_owner_bound_despite_quota_status(async_client, monkeypatch): + """An owner the selector skips for routing policy (API-key assignment + scope) must stay owner-bound even when its persisted status happens to be + quota-exhausted: policy, not quota, caused the selection loss.""" + owner_account_id = await _import_account( + async_client, email="compact-policy-owner@example.com", raw_account_id="acc_policy_owner" + ) + alt_account_id = await _import_account( + async_client, email="compact-policy-alt@example.com", raw_account_id="acc_policy_alt" + ) + await _mark_account_status(owner_account_id, AccountStatus.RATE_LIMITED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + settings_resp = await async_client.put( + "/api/settings", + json={"totpRequiredOnLogin": False, "apiKeyAuthEnabled": True}, + ) + assert settings_resp.status_code == 200 + _key_id, key = await _create_api_key( + name="compact-policy-scope-key", + assigned_account_ids=[alt_account_id], + ) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_policy_anchor", + }, + headers={"authorization": f"Bearer {key}"}, + ) + + assert response.status_code >= 400 + assert calls == [] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_with_additional_turn_state_pin_records_fail_closed( + async_client, monkeypatch, caplog +): + """A previous-response pin accompanied by a turn-state pin on the same + owner stays owner-bound (recovery never activates for additional owner + pins), but the unavailable owner must still record the compact + continuity_fail_closed outcome on the common pinned-selection failure + path instead of skipping the recording branch entirely.""" + owner_account_id = await _import_account( + async_client, email="compact-multipin-owner@example.com", raw_account_id="acc_multipin_owner" + ) + await _import_account(async_client, email="compact-multipin-alt@example.com", raw_account_id="acc_multipin_alt") + await _mark_account_status(owner_account_id, AccountStatus.RATE_LIMITED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + async def fake_turn_state_owner(self, *, turn_state, api_key, fail_on_missing=True): + del self, turn_state, api_key, fail_on_missing + return owner_account_id + + monkeypatch.setattr(proxy_module.ProxyService, "_resolve_compact_turn_state_owner", fake_turn_state_owner) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + with caplog.at_level(logging.INFO): + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_multipin_anchor", + }, + headers={"x-codex-turn-state": "ts-multipin-owner-bound"}, + ) + + assert response.status_code >= 400 + assert calls == [] + assert "blocked_reason=additional_owner_pins" in caplog.text + assert "continuity_fail_closed surface=compact reason=owner_account_unavailable" in caplog.text diff --git a/tests/integration/test_proxy_compact_triggers.py b/tests/integration/test_proxy_compact_triggers.py new file mode 100644 index 0000000000..dfc811c614 --- /dev/null +++ b/tests/integration/test_proxy_compact_triggers.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import json +from typing import cast + +import pytest + +import app.modules.proxy.service as proxy_module +from app.core.openai.models import CompactResponsePayload +from tests.integration.compact_test_helpers import _make_auth_json + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +async def test_proxy_compact_rejects_duplicate_compaction_trigger_before_upstream(async_client, monkeypatch): + email = "compact-duplicate-trigger@example.com" + raw_account_id = "acc_compact_duplicate_trigger" + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + response = await async_client.post("/api/accounts/import", files=files) + assert response.status_code == 200 + + async def fake_compact(*args, **kwargs): + del args, kwargs + pytest.fail("compact should not be called when input contains duplicate compaction_trigger items") + + monkeypatch.setattr(proxy_module, "core_compact_responses", fake_compact) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": [ + {"role": "user", "content": "hello"}, + {"type": "compaction_trigger"}, + {"type": "compaction_trigger"}, + ], + }, + ) + + assert response.status_code == 400 + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["code"] == "invalid_request_error" + assert error["param"] == "input" + + +@pytest.mark.asyncio +async def test_proxy_compact_rejects_non_terminal_compaction_trigger_before_instruction_hoist( + async_client, + monkeypatch, +): + email = "compact-trigger-hoist@example.com" + raw_account_id = "acc_compact_trigger_hoist" + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + response = await async_client.post("/api/accounts/import", files=files) + assert response.status_code == 200 + + async def fake_compact(*args, **kwargs): + del args, kwargs + pytest.fail("compact should not be called when a trailing developer message hides a non-terminal trigger") + + monkeypatch.setattr(proxy_module, "core_compact_responses", fake_compact) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "", + "input": [ + {"role": "user", "content": "hello"}, + {"type": "compaction_trigger"}, + {"role": "developer", "content": "still trailing after the trigger"}, + ], + }, + ) + + assert response.status_code == 400 + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["code"] == "invalid_request_error" + assert error["param"] == "input" + + +@pytest.mark.asyncio +async def test_proxy_compact_preserves_single_terminal_compaction_trigger(async_client, monkeypatch): + email = "compact-single-trigger@example.com" + raw_account_id = "acc_compact_single_trigger" + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + response = await async_client.post("/api/accounts/import", files=files) + assert response.status_code == 200 + + seen_payloads: list[dict[str, object]] = [] + + async def fake_compact(payload, *args, **kwargs): + del args, kwargs + seen_payloads.append(cast(dict[str, object], payload.to_payload())) + return CompactResponsePayload.model_validate({"object": "response.compaction", "output": []}) + + monkeypatch.setattr(proxy_module, "core_compact_responses", fake_compact) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": [ + {"role": "user", "content": "hello"}, + {"type": "compaction_trigger"}, + ], + }, + ) + + assert response.status_code == 200 + assert len(seen_payloads) == 1 + assert seen_payloads[0]["input"] == [ + {"role": "user", "content": "hello"}, + {"type": "compaction_trigger"}, + ] + + +@pytest.mark.asyncio +async def test_v1_proxy_compact_keeps_trigger_handling_unchanged(async_client, monkeypatch): + email = "v1-compact-trigger-unchanged@example.com" + raw_account_id = "acc_v1_compact_trigger_unchanged" + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + response = await async_client.post("/api/accounts/import", files=files) + assert response.status_code == 200 + + seen_payloads: list[dict[str, object]] = [] + + async def fake_compact(payload, *args, **kwargs): + del args, kwargs + seen_payloads.append(cast(dict[str, object], payload.to_payload())) + return CompactResponsePayload.model_validate({"object": "response.compaction", "output": []}) + + monkeypatch.setattr(proxy_module, "core_compact_responses", fake_compact) + + response = await async_client.post( + "/v1/responses/compact", + json={ + "model": "gpt-5.1", + "input": [ + {"role": "user", "content": "hello"}, + {"type": "compaction_trigger"}, + {"type": "compaction_trigger"}, + ], + }, + ) + + assert response.status_code == 200 + assert len(seen_payloads) == 1 + assert seen_payloads[0]["input"] == [ + {"role": "user", "content": "hello"}, + {"type": "compaction_trigger"}, + ] diff --git a/tests/integration/test_proxy_files.py b/tests/integration/test_proxy_files.py index db71df0ebf..45068df607 100644 --- a/tests/integration/test_proxy_files.py +++ b/tests/integration/test_proxy_files.py @@ -10,16 +10,25 @@ from __future__ import annotations +import asyncio import base64 import json +from datetime import datetime, timedelta from typing import cast import pytest +from sqlalchemy import func, select, text +import app.modules.proxy.file_pin_repository as file_pin_repository_module import app.modules.proxy.service as proxy_module from app.core.auth.refresh import RefreshError from app.core.clients.files import FileProxyError from app.core.clients.proxy import ProxyResponseError +from app.db.models import FileAccountPin, StickySessionKind +from app.db.session import SessionLocal +from app.modules.proxy.affinity import _codex_backend_identity, _codex_session_selection_key +from app.modules.proxy.file_pin_repository import FileAccountPinRepository +from app.modules.proxy.sticky_repository import StickySessionsRepository pytestmark = pytest.mark.integration @@ -46,11 +55,12 @@ def _make_auth_json(account_id: str, email: str) -> dict: } -async def _import_account(async_client, account_id: str, email: str) -> None: +async def _import_account(async_client, account_id: str, email: str) -> str: auth_json = _make_auth_json(account_id, email) files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} response = await async_client.post("/api/accounts/import", files=files) assert response.status_code == 200 + return response.json()["accountId"] @pytest.mark.asyncio @@ -517,6 +527,243 @@ async def test_resolve_file_account_for_responses_returns_pin_when_no_other_affi assert prepared.affinity_policy.codex_session_source == "session_header" +@pytest.mark.asyncio +async def test_file_account_pin_is_resolved_by_another_proxy_replica(async_client, monkeypatch): + await _import_account(async_client, "acc_cross_replica", "cross-replica@example.com") + + from app.core.openai.requests import ResponsesRequest + from app.dependencies import get_proxy_service_for_app + + origin = get_proxy_service_for_app(async_client._transport.app) + other_replica = proxy_module.ProxyService(origin._repo_factory) + await origin._pin_file_account("file_cross_replica", "acc_cross_replica") + + assert await other_replica._resolve_file_account("file_cross_replica") == "acc_cross_replica" + + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.2", + "instructions": "Summarize the uploaded file.", + "input": [ + { + "role": "user", + "content": [{"type": "input_file", "file_id": "file_cross_replica"}], + } + ], + } + ) + assert await other_replica._resolve_file_account_for_responses(payload, {}) == "acc_cross_replica" + + preferred_accounts: list[str | None] = [] + + async def fake_proxy_files_call(**kwargs): + preferred_accounts.append(await kwargs["resolve_preferred_account_id"]()) + result = {"status": "success"} + await kwargs["on_success"](result, "acc_cross_replica") + return result, "acc_cross_replica" + + monkeypatch.setattr(other_replica, "_proxy_files_call", fake_proxy_files_call) + assert await other_replica.finalize_file("file_cross_replica", {}) == {"status": "success"} + assert preferred_accounts == ["acc_cross_replica"] + + +@pytest.mark.asyncio +async def test_file_account_pin_repository_ignores_expired_rows(async_client): + del async_client + async with SessionLocal() as session: + database_now = await session.scalar(select(func.now())) + assert isinstance(database_now, datetime) + session.add( + FileAccountPin( + file_id="file_expired_repository", + account_id="acc_expired_repository", + expires_at=database_now, + ) + ) + await session.commit() + + async with SessionLocal() as session: + repository = FileAccountPinRepository(session) + assert await repository.get_live_account_id("file_expired_repository") is None + + +@pytest.mark.asyncio +async def test_file_account_pin_same_owner_claim_is_idempotent_and_renews_expiry(async_client): + del async_client + async with SessionLocal() as session: + await FileAccountPinRepository(session).claim( + "file_same_owner", + "acc_same_owner", + ttl_seconds=60, + ) + async with SessionLocal() as session: + first_pin = await session.get(FileAccountPin, "file_same_owner") + assert first_pin is not None + first_expiry = first_pin.expires_at + + async with SessionLocal() as session: + await FileAccountPinRepository(session).claim( + "file_same_owner", + "acc_same_owner", + ttl_seconds=120, + ) + async with SessionLocal() as session: + renewed_pin = await session.get(FileAccountPin, "file_same_owner") + assert renewed_pin is not None + assert renewed_pin.account_id == "acc_same_owner" + assert renewed_pin.expires_at > first_expiry + + +@pytest.mark.asyncio +async def test_file_account_pin_claim_refreshes_a_stale_successful_insert_expiry( + async_client, + monkeypatch, +): + del async_client + stale_insert = text( + """ + INSERT INTO file_account_pins (file_id, account_id, expires_at) + VALUES ( + :file_id, + :account_id, + (strftime('%Y-%m-%d %H:%M:%f', 'now', '-' || :ttl || ' seconds') || '000') + ) + RETURNING account_id + """ + ) + monkeypatch.setattr(file_pin_repository_module, "_SQLITE_CLAIM", stale_insert) + + async with SessionLocal() as session: + await FileAccountPinRepository(session).claim( + "file_stale_insert_candidate", + "acc_stale_insert_candidate", + ttl_seconds=120, + ) + + async with SessionLocal() as session: + database_now = await session.scalar(select(func.now())) + pin = await session.get(FileAccountPin, "file_stale_insert_candidate") + assert isinstance(database_now, datetime) + assert pin is not None + assert pin.expires_at > database_now + timedelta(seconds=100) + + +@pytest.mark.asyncio +async def test_file_account_pin_claim_rolls_back_when_post_claim_refresh_does_not_match( + async_client, + monkeypatch, +): + del async_client + monkeypatch.setattr( + file_pin_repository_module, + "_SQLITE_REFRESH", + text( + """ + UPDATE file_account_pins + SET expires_at = expires_at + WHERE file_id = :file_id + AND account_id = :account_id + AND 0 = 1 + RETURNING account_id + """ + ), + ) + + async with SessionLocal() as session: + with pytest.raises(RuntimeError, match="Failed to refresh file account pin after claim"): + await FileAccountPinRepository(session).claim( + "file_failed_post_claim_refresh", + "acc_failed_post_claim_refresh", + ttl_seconds=120, + ) + + async with SessionLocal() as session: + assert await session.get(FileAccountPin, "file_failed_post_claim_refresh") is None + + +@pytest.mark.asyncio +async def test_reclaimed_file_account_pin_is_observed_without_local_cache(async_client): + from app.dependencies import get_proxy_service_for_app + + first_replica = get_proxy_service_for_app(async_client._transport.app) + second_replica = proxy_module.ProxyService(first_replica._repo_factory) + await first_replica._pin_file_account("file_claim_lifecycle", "acc_claim_a") + assert await first_replica._resolve_file_account("file_claim_lifecycle") == "acc_claim_a" + + async with SessionLocal() as session: + database_now = await session.scalar(select(func.now())) + assert isinstance(database_now, datetime) + pin = await session.get(FileAccountPin, "file_claim_lifecycle") + assert pin is not None + pin.expires_at = database_now - timedelta(seconds=1) + session.add( + FileAccountPin( + file_id="file_cleanup_expired", + account_id="acc_cleanup_expired", + expires_at=database_now - timedelta(seconds=1), + ) + ) + await session.commit() + + await second_replica._pin_file_account("file_claim_lifecycle", "acc_claim_b") + + async with SessionLocal() as session: + assert await session.get(FileAccountPin, "file_cleanup_expired") is None + + assert await first_replica._resolve_file_account("file_claim_lifecycle") == "acc_claim_b" + + +@pytest.mark.asyncio +async def test_concurrent_file_account_pin_claims_choose_one_durable_owner(async_client): + from app.dependencies import get_proxy_service_for_app + + app_service = get_proxy_service_for_app(async_client._transport.app) + start = asyncio.Event() + + async def claim(account_id: str) -> tuple[str, str]: + replica = proxy_module.ProxyService(app_service._repo_factory) + await start.wait() + try: + await replica._pin_file_account("file_concurrent_claim", account_id) + except ProxyResponseError as exc: + return account_id, str(exc.payload["error"]["code"]) + return account_id, "claimed" + + claims = [ + asyncio.create_task(claim("acc_race_a")), + asyncio.create_task(claim("acc_race_b")), + ] + start.set() + results = await asyncio.gather(*claims) + + winners = [account_id for account_id, outcome in results if outcome == "claimed"] + conflicts = [account_id for account_id, outcome in results if outcome == "continuity_owner_conflict"] + assert len(winners) == 1 + assert len(conflicts) == 1 + + observer = proxy_module.ProxyService(app_service._repo_factory) + assert await observer._resolve_file_account("file_concurrent_claim") == winners[0] + + +@pytest.mark.asyncio +async def test_live_file_account_pin_cannot_be_reassigned_by_another_replica(async_client): + await _import_account(async_client, "acc_pin_owner_a", "pin-owner-a@example.com") + await _import_account(async_client, "acc_pin_owner_b", "pin-owner-b@example.com") + + from app.dependencies import get_proxy_service_for_app + + origin = get_proxy_service_for_app(async_client._transport.app) + other_replica = proxy_module.ProxyService(origin._repo_factory) + await origin._pin_file_account("file_immutable_owner", "acc_pin_owner_a") + + with pytest.raises(ProxyResponseError) as exc_info: + await other_replica._pin_file_account("file_immutable_owner", "acc_pin_owner_b") + + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "continuity_owner_conflict" + assert await other_replica._resolve_file_account("file_immutable_owner") == "acc_pin_owner_a" + + @pytest.mark.asyncio async def test_v1_responses_file_id_pin_overrides_prompt_cache_key(async_client, monkeypatch): """A prompt-cache key is locality; an account-scoped file is ownership.""" @@ -621,6 +868,112 @@ async def fake_stream( assert resolved is not None +@pytest.mark.asyncio +async def test_backend_responses_file_pin_does_not_rewrite_existing_thread_row( + async_client, + monkeypatch, +): + from app.dependencies import get_proxy_service_for_app + + thread_owner_chatgpt_id = "acc_file_pin_thread_owner" + file_owner_chatgpt_id = "acc_file_pin_file_owner" + thread_owner_id = await _import_account( + async_client, + thread_owner_chatgpt_id, + "file-pin-thread-owner@example.com", + ) + file_owner_id = await _import_account( + async_client, + file_owner_chatgpt_id, + "file-pin-file-owner@example.com", + ) + process_session = "file-pin-process" + thread_headers = {"session-id": process_session, "thread-id": "file-pin-thread"} + sibling_headers = {"session-id": process_session, "thread-id": "file-pin-sibling"} + thread_key = _codex_backend_identity(thread_headers).thread_selection_key + sibling_key = _codex_backend_identity(sibling_headers).thread_selection_key + process_key = _codex_session_selection_key(process_session) + assert thread_key is not None + assert sibling_key is not None + + async with SessionLocal() as session: + await StickySessionsRepository(session).upsert( + thread_key, + thread_owner_id, + kind=StickySessionKind.PROMPT_CACHE, + ) + + service = get_proxy_service_for_app(async_client._transport.app) + await service._pin_file_account("file_thread_locality", file_owner_id) + seen: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, **kwargs): + del payload, headers, access_token, kwargs + seen.append(account_id) + yield 'data: {"type":"response.completed","response":{"id":"resp_file_pin_thread"}}\n\n' + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + + pinned_response = await async_client.post( + "/backend-api/codex/responses", + headers=thread_headers, + json={ + "model": "gpt-5.2", + "instructions": "You are a helpful assistant.", + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Read the file."}, + {"type": "input_file", "file_id": "file_thread_locality"}, + ], + } + ], + "stream": True, + }, + ) + assert pinned_response.status_code == 200 + assert seen == [file_owner_chatgpt_id] + + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + assert await repo.get_account_id(thread_key, kind=StickySessionKind.PROMPT_CACHE) == thread_owner_id + assert await repo.get_account_id(process_key, kind=StickySessionKind.CODEX_SESSION) == file_owner_id + assert await repo.get_account_id(sibling_key, kind=StickySessionKind.PROMPT_CACHE) is None + + unpinned_response = await async_client.post( + "/backend-api/codex/responses", + headers=thread_headers, + json={ + "model": "gpt-5.2", + "instructions": "You are a helpful assistant.", + "input": "Continue without the file.", + "stream": True, + }, + ) + assert unpinned_response.status_code == 200 + assert seen == [file_owner_chatgpt_id, thread_owner_chatgpt_id] + + sibling_response = await async_client.post( + "/backend-api/codex/responses", + headers=sibling_headers, + json={ + "model": "gpt-5.2", + "instructions": "You are a helpful assistant.", + "input": "Sibling thread without a file.", + "stream": True, + }, + ) + assert sibling_response.status_code == 200 + assert seen == [file_owner_chatgpt_id, thread_owner_chatgpt_id, file_owner_chatgpt_id] + + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + assert await repo.get_account_id(thread_key, kind=StickySessionKind.PROMPT_CACHE) == thread_owner_id + assert await repo.get_account_id(process_key, kind=StickySessionKind.CODEX_SESSION) == file_owner_id + assert await repo.get_account_id(sibling_key, kind=StickySessionKind.PROMPT_CACHE) == file_owner_id + + @pytest.mark.asyncio async def test_derived_prompt_cache_key_does_not_block_file_id_pin(async_client): """Regression: a ``prompt_cache_key`` that the proxy itself derived diff --git a/tests/integration/test_proxy_images.py b/tests/integration/test_proxy_images.py index 494edccc1e..aef0de9119 100644 --- a/tests/integration/test_proxy_images.py +++ b/tests/integration/test_proxy_images.py @@ -9,6 +9,7 @@ from __future__ import annotations +import asyncio import base64 import json import logging @@ -17,6 +18,7 @@ import pytest from httpx import AsyncByteStream +from sqlalchemy import select from starlette.datastructures import UploadFile from starlette.responses import JSONResponse @@ -25,7 +27,9 @@ from app.core.config.settings import Settings from app.core.exceptions import ProxyModelNotAllowed, ProxyRateLimitError from app.core.multipart import MultipartPolicy -from app.db.models import DashboardSettings +from app.db.models import ApiKeyUsageReservation, DashboardSettings +from app.db.session import SessionLocal +from app.modules.api_keys.repository import ApiKeysRepository pytestmark = pytest.mark.integration @@ -1608,12 +1612,30 @@ async def fake_ensure_fresh(self, account, **kwargs): @pytest.mark.asyncio -async def test_images_generations_succeeds_when_reservation_finalize_fails(async_client, monkeypatch): - """A successful image generation must NOT 500 when the post-hoc - API-key reservation finalize raises (e.g. transient DB failure). - The accounting failure is swallowed and logged; the client still - receives the image envelope. - """ +async def test_images_generations_finalize_failure_tracks_release_recovery( + async_client, + monkeypatch, +): + """A successful image keeps tracked ownership after finalization fails.""" + await _enable_api_key_auth(async_client) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "images-finalize-release-recovery", + "limits": [ + { + "limitType": "total_tokens", + "limitWindow": "weekly", + "maxValue": 1_000_000, + }, + ], + }, + ) + assert created.status_code == 200, created.text + key_payload = created.json() + api_key = key_payload["key"] + api_key_id = key_payload["id"] + await _import_account(async_client, "acc_images_finalize_fail", "img-fin-fail@example.com") async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False, **kwargs): @@ -1650,14 +1672,23 @@ async def fake_ensure_fresh(self, account, **kwargs): # Patch finalize to blow up so we can confirm the route still 200s. from app.modules.api_keys.service import ApiKeysService + release_completed = asyncio.Event() + original_release = ApiKeysService.release_usage_reservation + async def fake_finalize(self, *args, **kwargs): del self, args, kwargs raise RuntimeError("simulated DB failure during finalize") + async def tracked_release(self, reservation_id): + await original_release(self, reservation_id) + release_completed.set() + monkeypatch.setattr(ApiKeysService, "finalize_usage_reservation", fake_finalize) + monkeypatch.setattr(ApiKeysService, "release_usage_reservation", tracked_release) response = await async_client.post( "/v1/images/generations", + headers={"Authorization": f"Bearer {api_key}"}, json={ "model": "gpt-image-2", "prompt": "x", @@ -1669,3 +1700,185 @@ async def fake_finalize(self, *args, **kwargs): assert response.status_code == 200, response.text body = response.json() assert body["data"] == [{"b64_json": "B64_FINFAIL"}] + await asyncio.wait_for(release_completed.wait(), timeout=1.0) + + async with SessionLocal() as session: + reservations = ( + ( + await session.execute( + select(ApiKeyUsageReservation).where(ApiKeyUsageReservation.api_key_id == api_key_id) + ) + ) + .scalars() + .all() + ) + assert [reservation.status for reservation in reservations] == ["released"] + + limits = await ApiKeysRepository(session).get_limits_by_key(api_key_id) + assert len(limits) == 1 + assert limits[0].current_value == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route", "stream"), + [ + ("generations", False), + ("generations", True), + ("edits", False), + ("edits", True), + ], +) +async def test_image_routes_handoff_captured_usage_exactly_once( + async_client, + monkeypatch, + route, + stream, +): + await _enable_api_key_auth(async_client) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": f"images-{route}-{'stream' if stream else 'json'}-handoff", + "limits": [ + { + "limitType": "total_tokens", + "limitWindow": "weekly", + "maxValue": 1_000_000, + }, + ], + }, + ) + assert created.status_code == 200, created.text + api_key = created.json()["key"] + + await _import_account( + async_client, + f"acc_images_{route}_{stream}", + f"img-{route}-{stream}@example.com", + ) + + async def fake_stream( + payload, + headers, + access_token, + account_id, + base_url=None, + raise_for_status=False, + **kwargs, + ): + del payload, headers, access_token, account_id, base_url, raise_for_status, kwargs + yield _sse( + { + "type": "response.output_item.done", + "output_index": 0, + "item": { + "type": "image_generation_call", + "id": f"ig_{route}_{stream}", + "status": "completed", + "result": "B64_HANDOFF", + }, + } + ) + yield _sse( + { + "type": "response.completed", + "response": { + "id": f"resp_{route}_{stream}", + "tool_usage": { + "image_gen": { + "input_tokens": 3, + "output_tokens": 4, + } + }, + }, + } + ) + + async def fake_ensure_fresh(self, account, **kwargs): + del self, kwargs + return account + + internal_reservations: list[object] = [] + original_stream_responses = proxy_module.ProxyService.stream_responses + + async def tracked_stream_responses(self, *args, **kwargs): + internal_reservations.append(kwargs.get("api_key_reservation")) + async for chunk in original_stream_responses(self, *args, **kwargs): + yield chunk + + handoffs: list[dict[str, object]] = [] + original_settle_image = proxy_module.ProxyService.settle_image_api_key_usage + + async def tracked_settle_image( + self, + api_key_arg, + reservation_arg, + **kwargs, + ): + handoffs.append( + { + "api_key": api_key_arg, + "reservation": reservation_arg, + **kwargs, + } + ) + return await original_settle_image( + self, + api_key_arg, + reservation_arg, + **kwargs, + ) + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh) + monkeypatch.setattr(proxy_module.ProxyService, "stream_responses", tracked_stream_responses) + monkeypatch.setattr( + proxy_module.ProxyService, + "settle_image_api_key_usage", + tracked_settle_image, + ) + + headers = {"Authorization": f"Bearer {api_key}"} + if route == "generations": + response = await async_client.post( + "/v1/images/generations", + headers=headers, + json={ + "model": "gpt-image-2", + "prompt": "handoff", + "stream": stream, + "size": "1024x1024", + "quality": "low", + }, + ) + else: + response = await async_client.post( + "/v1/images/edits", + headers=headers, + data={ + "model": "gpt-image-2", + "prompt": "handoff", + "stream": str(stream).lower(), + "size": "1024x1024", + "quality": "low", + }, + files={ + "image": ( + "source.png", + b"\x89PNG\r\n\x1a\n" + b"\x00" * 16, + "image/png", + ), + }, + ) + + assert response.status_code == 200, response.text + assert internal_reservations == [None] + assert len(handoffs) == 1 + handoff = handoffs[0] + assert handoff["api_key"] is not None + assert handoff["reservation"] is not None + assert handoff["model"] == "gpt-image-2" + assert handoff["input_tokens"] == 3 + assert handoff["output_tokens"] == 4 + assert handoff["cached_input_tokens"] is None diff --git a/tests/integration/test_proxy_responses.py b/tests/integration/test_proxy_responses.py index 66bed5957f..6d373aeea2 100644 --- a/tests/integration/test_proxy_responses.py +++ b/tests/integration/test_proxy_responses.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import base64 import json from collections.abc import Mapping @@ -20,6 +21,7 @@ from app.core.utils.time import utcnow from app.db.models import Account, DashboardSettings, RequestLog from app.db.session import SessionLocal +from app.modules.api_keys.service import ApiKeyUsageReservationData from app.modules.proxy._service.streaming import retry as streaming_retry_module from app.modules.request_logs.repository import RequestLogsRepository from app.modules.usage.repository import AdditionalUsageRepository @@ -658,6 +660,8 @@ async def fake_compact(payload, headers, access_token, account_id, **kwargs): assert "Image Size: 1512x982." in compact_input_json assert "Omitted inline image bytes that were already observed before compaction" in compact_input_json assert "data:image/png;base64" not in compact_input_json + assert compact_input[-1] == {"type": "compaction_trigger"} + assert sum(1 for item in compact_input if item.get("type") == "compaction_trigger") == 1 assert seen_payload["previous_response_id"] == "resp_compact_anchor" assert seen_payload["account_id"] == raw_account_id compact_payload = cast(Mapping[str, object], seen_payload["payload"]) @@ -804,6 +808,11 @@ async def unexpected_compact(*args, **kwargs): [ [{"type": "compaction_trigger"}, {"role": "user", "content": "hello"}], [{"role": "user", "content": "hello"}, {"type": "compaction_trigger"}, {"type": "compaction_trigger"}], + [ + {"role": "user", "content": "hello"}, + {"type": "compaction_trigger"}, + {"role": "developer", "content": "still trailing"}, + ], ], ) async def test_proxy_responses_rejects_malformed_compaction_trigger(async_client, monkeypatch, input_items): @@ -2055,6 +2064,311 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert log.transport == "http" +@pytest.mark.asyncio +async def test_backend_responses_terminal_disconnect_finalizes_settlement_and_success( + async_client, + app_instance, + monkeypatch, +): + email = "terminal-disconnect@example.com" + raw_account_id = "acc_terminal_disconnect" + expected_account_id = generate_unique_account_id(raw_account_id, email) + auth_json = _make_auth_json(raw_account_id, email) + response = await async_client.post( + "/api/accounts/import", + files={"auth_json": ("auth.json", json.dumps(auth_json), "application/json")}, + ) + assert response.status_code == 200 + + reservation = ApiKeyUsageReservationData( + reservation_id="resv_terminal_disconnect", + key_id="key_terminal_disconnect", + model="gpt-5.1", + ) + stream_closed = asyncio.Event() + settle_calls: list[dict[str, object]] = [] + success_account_ids: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False, **kwargs): + del payload, headers, access_token, account_id, base_url, raise_for_status, kwargs + try: + yield ( + 'data: {"type":"response.completed","response":{"id":"resp_terminal_disconnect",' + '"object":"response","status":"completed","output":[],"usage":' + '{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}\n\n' + ) + await asyncio.Event().wait() + finally: + stream_closed.set() + + async def fake_enforce_request_limits(*_args: object, **_kwargs: object): + return reservation + + async def fake_settle_stream_api_key_usage(self, api_key, api_key_reservation, settlement, request_id, **kwargs): + del self, api_key + settle_calls.append( + { + "reservation": api_key_reservation, + "status": settlement.status, + "request_id": request_id, + "input_tokens": settlement.input_tokens, + "output_tokens": settlement.output_tokens, + "wait_for_settlement": kwargs.get("wait_for_settlement", False), + } + ) + return True + + async def fake_record_success(self, account): + del self + success_account_ids.append(account.id) + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + monkeypatch.setattr(proxy_api_module, "_enforce_request_limits", fake_enforce_request_limits) + monkeypatch.setattr( + proxy_module.ProxyService, + "_settle_stream_api_key_usage", + fake_settle_stream_api_key_usage, + ) + monkeypatch.setattr(proxy_module.LoadBalancer, "record_success", fake_record_success) + + request_id = "req_terminal_disconnect" + request_body = json.dumps( + {"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}, + separators=(",", ":"), + ).encode("utf-8") + request_sent = False + disconnect_allowed = asyncio.Event() + first_terminal_sent = asyncio.Event() + response_started: list[dict[str, object]] = [] + + async def receive() -> dict[str, object]: + nonlocal request_sent + if not request_sent: + request_sent = True + return {"type": "http.request", "body": request_body, "more_body": False} + await disconnect_allowed.wait() + return {"type": "http.disconnect"} + + async def send(message: dict[str, object]) -> None: + if message["type"] == "http.response.start": + response_started.append(message) + return + body = message.get("body") + if ( + message["type"] == "http.response.body" + and isinstance(body, bytes) + and b'"type":"response.completed"' in body + ): + first_terminal_sent.set() + disconnect_allowed.set() + + await app_instance( + { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/backend-api/codex/responses", + "raw_path": b"/backend-api/codex/responses", + "query_string": b"", + "headers": [ + (b"host", b"testserver"), + (b"content-type", b"application/json"), + (b"x-request-id", request_id.encode("ascii")), + ], + "client": ("127.0.0.1", 1234), + "server": ("testserver", 80), + }, + receive, + send, + ) + assert response_started and response_started[0]["status"] == 200 + assert first_terminal_sent.is_set() is True + + await asyncio.wait_for(stream_closed.wait(), timeout=1.0) + await app_instance.state.proxy_service.drain_persistence_tasks(timeout_seconds=5) + + assert settle_calls == [ + { + "reservation": reservation, + "status": "success", + "request_id": request_id, + "input_tokens": 1, + "output_tokens": 1, + "wait_for_settlement": False, + } + ] + assert success_account_ids == [expected_account_id] + + async with SessionLocal() as session: + result = await session.execute(select(RequestLog).where(RequestLog.request_id == "resp_terminal_disconnect")) + log = result.scalars().one() + assert log.archive_request_id == request_id + assert log.account_id == expected_account_id + assert log.status == "success" + + +@pytest.mark.asyncio +async def test_backend_responses_post_refresh_terminal_disconnect_finalizes_settlement( + async_client, + app_instance, + monkeypatch, +): + email = "post-refresh-terminal-disconnect@example.com" + raw_account_id = "acc_post_refresh_terminal_disconnect" + expected_account_id = generate_unique_account_id(raw_account_id, email) + auth_json = _make_auth_json(raw_account_id, email) + response = await async_client.post( + "/api/accounts/import", + files={"auth_json": ("auth.json", json.dumps(auth_json), "application/json")}, + ) + assert response.status_code == 200 + + reservation = ApiKeyUsageReservationData( + reservation_id="resv_post_refresh_terminal_disconnect", + key_id="key_post_refresh_terminal_disconnect", + model="gpt-5.1", + ) + stream_calls: list[int] = [] + stream_closed = asyncio.Event() + settle_calls: list[dict[str, object]] = [] + success_account_ids: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False, **kwargs): + del payload, headers, access_token, account_id, base_url, raise_for_status, kwargs + stream_calls.append(len(stream_calls) + 1) + if len(stream_calls) == 1: + raise proxy_module.ProxyResponseError( + 401, + {"error": {"code": "invalid_api_key", "message": "token invalidated"}}, + ) + try: + yield ( + 'data: {"type":"response.completed","response":{"id":"resp_post_refresh_disconnect",' + '"object":"response","status":"completed","output":[],"usage":' + '{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}\n\n' + ) + await asyncio.Event().wait() + finally: + stream_closed.set() + + async def fake_ensure_fresh(self, account, **kwargs): + del self, kwargs + return account + + async def fake_enforce_request_limits(*_args: object, **_kwargs: object): + return reservation + + async def fake_settle_stream_api_key_usage(self, api_key, api_key_reservation, settlement, request_id, **kwargs): + del self, api_key + settle_calls.append( + { + "reservation": api_key_reservation, + "status": settlement.status, + "request_id": request_id, + "input_tokens": settlement.input_tokens, + "output_tokens": settlement.output_tokens, + "wait_for_settlement": kwargs.get("wait_for_settlement", False), + } + ) + return True + + async def fake_record_success(self, account): + del self + success_account_ids.append(account.id) + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh) + monkeypatch.setattr(proxy_api_module, "_enforce_request_limits", fake_enforce_request_limits) + monkeypatch.setattr( + proxy_module.ProxyService, + "_settle_stream_api_key_usage", + fake_settle_stream_api_key_usage, + ) + monkeypatch.setattr(proxy_module.LoadBalancer, "record_success", fake_record_success) + + request_id = "req_post_refresh_terminal_disconnect" + request_body = json.dumps( + {"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}, + separators=(",", ":"), + ).encode("utf-8") + request_sent = False + disconnect_allowed = asyncio.Event() + first_terminal_sent = asyncio.Event() + response_started: list[dict[str, object]] = [] + + async def receive() -> dict[str, object]: + nonlocal request_sent + if not request_sent: + request_sent = True + return {"type": "http.request", "body": request_body, "more_body": False} + await disconnect_allowed.wait() + return {"type": "http.disconnect"} + + async def send(message: dict[str, object]) -> None: + if message["type"] == "http.response.start": + response_started.append(message) + return + body = message.get("body") + if ( + message["type"] == "http.response.body" + and isinstance(body, bytes) + and b'"type":"response.completed"' in body + ): + first_terminal_sent.set() + disconnect_allowed.set() + + await app_instance( + { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/backend-api/codex/responses", + "raw_path": b"/backend-api/codex/responses", + "query_string": b"", + "headers": [ + (b"host", b"testserver"), + (b"content-type", b"application/json"), + (b"x-request-id", request_id.encode("ascii")), + ], + "client": ("127.0.0.1", 1234), + "server": ("testserver", 80), + }, + receive, + send, + ) + assert response_started and response_started[0]["status"] == 200 + assert stream_calls == [1, 2] + assert first_terminal_sent.is_set() is True + + await asyncio.wait_for(stream_closed.wait(), timeout=1.0) + await app_instance.state.proxy_service.drain_persistence_tasks(timeout_seconds=5) + + assert settle_calls == [ + { + "reservation": reservation, + "status": "success", + "request_id": request_id, + "input_tokens": 1, + "output_tokens": 1, + "wait_for_settlement": False, + } + ] + assert success_account_ids == [expected_account_id] + + async with SessionLocal() as session: + result = await session.execute( + select(RequestLog).where(RequestLog.request_id == "resp_post_refresh_disconnect") + ) + log = result.scalars().one() + assert log.archive_request_id == request_id + assert log.account_id == expected_account_id + assert log.status == "success" + + @pytest.mark.asyncio async def test_proxy_responses_forwards_native_codex_headers(async_client, monkeypatch): email = "stream-headers@example.com" @@ -2637,6 +2951,34 @@ async def test_v1_responses_compact_invalid_messages_returns_openai_400(async_cl assert body["error"]["param"] == "messages" +@pytest.mark.asyncio +async def test_v1_responses_rejects_duplicate_top_level_compaction_trigger(async_client, monkeypatch): + async def fail_stream(*args, **kwargs): + del args, kwargs + pytest.fail("malformed top-level compaction_trigger must fail before upstream streaming") + + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_stream) + + resp = await async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.2", + "input": [ + {"role": "user", "content": "hello"}, + {"type": "compaction_trigger"}, + {"type": "compaction_trigger"}, + ], + "stream": True, + }, + ) + + assert resp.status_code == 400 + body = resp.json() + assert body["error"]["type"] == "invalid_request_error" + assert body["error"]["code"] == "invalid_request_error" + assert body["error"]["param"] == "input" + + @pytest.mark.asyncio async def test_v1_chat_completions_invalid_tool_calls_returns_openai_400(async_client): payload = { diff --git a/tests/integration/test_proxy_sticky_sessions.py b/tests/integration/test_proxy_sticky_sessions.py index a3726b3b89..95e49fe25c 100644 --- a/tests/integration/test_proxy_sticky_sessions.py +++ b/tests/integration/test_proxy_sticky_sessions.py @@ -3,7 +3,7 @@ import asyncio import base64 import json -from datetime import timedelta, timezone +from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import cast @@ -17,12 +17,13 @@ from app.db.models import Account, AccountStatus, StickySessionKind from app.db.session import SessionLocal from app.modules.accounts.repository import AccountsRepository -from app.modules.api_keys.service import ApiKeyData +from app.modules.api_keys.repository import ApiKeysRepository +from app.modules.api_keys.service import ApiKeyCreateData, ApiKeyData, ApiKeysService from app.modules.proxy._service.realtime_live import ( _REALTIME_CALL_AFFINITY_MAX_AGE_SECONDS, realtime_call_affinity_key, ) -from app.modules.proxy.affinity import _codex_session_selection_key +from app.modules.proxy.affinity import _codex_backend_identity, _codex_session_selection_key from app.modules.usage.repository import UsageRepository pytestmark = pytest.mark.integration @@ -93,6 +94,7 @@ def _install_proxy_settings_cache( openai_cache_affinity_max_age_seconds: int = 300, sticky_reallocation_budget_threshold_pct: float = 95.0, openai_prompt_cache_key_derivation_enabled: bool = True, + proxy_request_budget_seconds: float = 75.0, ) -> None: settings = SimpleNamespace( prefer_earlier_reset_accounts=prefer_earlier_reset_accounts, @@ -101,13 +103,14 @@ def _install_proxy_settings_cache( sticky_reallocation_budget_threshold_pct=sticky_reallocation_budget_threshold_pct, openai_prompt_cache_key_derivation_enabled=openai_prompt_cache_key_derivation_enabled, routing_strategy="usage_weighted", - proxy_request_budget_seconds=75.0, + proxy_request_budget_seconds=proxy_request_budget_seconds, compact_request_budget_seconds=75.0, transcription_request_budget_seconds=120.0, upstream_compact_timeout_seconds=None, upstream_stream_transport="auto", trace_channels=frozenset(), http_responses_session_bridge_enabled=False, + http_responses_session_bridge_instance_id="sticky-session-test", http_responses_session_bridge_idle_ttl_seconds=120.0, http_responses_session_bridge_codex_idle_ttl_seconds=900.0, http_responses_session_bridge_max_sessions=128, @@ -242,6 +245,548 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert mapped_account_id == owner_id +@pytest.mark.asyncio +async def test_codex_goal_restart_retires_unavailable_legacy_owner_and_stays_on_replacement( + async_client, + monkeypatch, +): + from sqlalchemy import select + + from app.db.models import StickySession + from app.modules.proxy.sticky_repository import StickySessionsRepository + + _install_proxy_settings_cache(monkeypatch, sticky_threads_enabled=False) + owner_id = await _import_account(async_client, "acc_goal_restart_owner", "goal-restart-owner@example.com") + replacement_id = await _import_account( + async_client, + "acc_goal_restart_replacement", + "goal-restart-replacement@example.com", + ) + raw_session = "goal-restart-session" + selection_key = _codex_session_selection_key(raw_session) + + now_epoch = int(utcnow().replace(tzinfo=timezone.utc).timestamp()) + async with SessionLocal() as session: + usage_repo = UsageRepository(session) + await usage_repo.add_entry( + account_id=owner_id, + used_percent=10.0, + window="primary", + reset_at=now_epoch + 3600, + window_minutes=300, + ) + await usage_repo.add_entry( + account_id=replacement_id, + used_percent=20.0, + window="primary", + reset_at=now_epoch + 3600, + window_minutes=300, + ) + await StickySessionsRepository(session).upsert( + raw_session, + owner_id, + kind=StickySessionKind.CODEX_SESSION, + ) + + seen: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False, **kwargs): + del payload, headers, access_token, base_url, raise_for_status, kwargs + seen.append(account_id) + yield f'data: {{"type":"response.completed","response":{{"id":"resp_goal_{len(seen)}"}}}}\n\n' + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + headers = {"session_id": raw_session} + restart_payload = { + "model": "gpt-5.1", + "instructions": "Continue the existing task.", + "input": [ + { + "role": "developer", + "content": ('\nContinue working toward the active thread goal.'), + }, + {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ], + "stream": True, + } + + # The restart marker is not enough to move a healthy owner. + healthy_response = await async_client.post( + "/backend-api/codex/responses", + json=restart_payload, + headers=headers, + ) + assert healthy_response.status_code == 200 + assert seen == ["acc_goal_restart_owner"] + + async with SessionLocal() as session: + await session.execute(update(Account).where(Account.id == owner_id).values(status=AccountStatus.QUOTA_EXCEEDED)) + await session.commit() + + restart_response = await async_client.post( + "/backend-api/codex/responses", + json=restart_payload, + headers=headers, + ) + assert restart_response.status_code == 200 + assert seen == ["acc_goal_restart_owner", "acc_goal_restart_replacement"] + + # A later ordinary turn uses the replacement's namespaced session affinity. + follow_up_response = await async_client.post( + "/backend-api/codex/responses", + json={"model": "gpt-5.1", "instructions": "continue", "input": [], "stream": True}, + headers=headers, + ) + assert follow_up_response.status_code == 200 + assert seen == [ + "acc_goal_restart_owner", + "acc_goal_restart_replacement", + "acc_goal_restart_replacement", + ] + + # The raw compatibility text can also be a different client's explicit + # turn state. Session-header abandonment must not erase that hard owner or + # dispatch the turn-state continuation on the replacement account. + turn_state_response = await async_client.post( + "/backend-api/codex/responses", + json={"model": "gpt-5.1", "instructions": "continue", "input": [], "stream": True}, + headers={"x-codex-turn-state": raw_session}, + ) + assert turn_state_response.status_code == 502 + assert turn_state_response.json()["error"]["code"] == "turn_state_owner_unavailable" + assert seen == [ + "acc_goal_restart_owner", + "acc_goal_restart_replacement", + "acc_goal_restart_replacement", + ] + + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + rows = { + row.key: row + for row in ( + await session.execute( + select(StickySession).where( + StickySession.key.in_((raw_session, selection_key)), + StickySession.kind == StickySessionKind.CODEX_SESSION, + ) + ) + ).scalars() + } + session_header_lookup = await repo.get_account_id_and_abandonment( + raw_session, + kind=StickySessionKind.CODEX_SESSION, + continuity_source="session_header", + ) + turn_state_owner = await repo.get_account_id( + raw_session, + kind=StickySessionKind.CODEX_SESSION, + continuity_source="turn_state", + ) + assert rows[raw_session].account_id == owner_id + assert rows[raw_session].continuity_abandoned_at is None + assert rows[raw_session].continuity_abandonment_scope == "session_header" + assert rows[selection_key].account_id == replacement_id + assert rows[selection_key].continuity_abandoned_at is None + assert rows[selection_key].continuity_abandonment_scope is None + assert session_header_lookup.account_id is None + assert session_header_lookup.continuity_abandoned is True + assert session_header_lookup.abandoned_account_id == owner_id + assert turn_state_owner == owner_id + # Parent-version readers know only the timestamp tombstone. Keeping it + # NULL makes them fail closed on the retained owner during rollout. + legacy_replica_owner = ( + None if rows[raw_session].continuity_abandoned_at is not None else rows[raw_session].account_id + ) + assert legacy_replica_owner == owner_id + + +@pytest.mark.asyncio +async def test_codex_goal_restart_with_thread_id_retires_unavailable_legacy_owner_and_stays_on_replacement( + async_client, + monkeypatch, +): + from sqlalchemy import select + + from app.db.models import StickySession + from app.modules.proxy.sticky_repository import StickySessionsRepository + + _install_proxy_settings_cache(monkeypatch, sticky_threads_enabled=False) + owner_id = await _import_account( + async_client, + "acc_goal_restart_thread_owner", + "goal-restart-thread-owner@example.com", + ) + replacement_id = await _import_account( + async_client, + "acc_goal_restart_thread_replacement", + "goal-restart-thread-replacement@example.com", + ) + raw_session = "goal-restart-thread-session" + thread_id = "goal-restart-thread" + headers = {"session_id": raw_session, "thread-id": thread_id} + thread_key = _codex_backend_identity(headers).thread_selection_key + assert thread_key is not None + + now_epoch = int(utcnow().replace(tzinfo=timezone.utc).timestamp()) + async with SessionLocal() as session: + usage_repo = UsageRepository(session) + await usage_repo.add_entry( + account_id=owner_id, + used_percent=10.0, + window="primary", + reset_at=now_epoch + 3600, + window_minutes=300, + ) + await usage_repo.add_entry( + account_id=replacement_id, + used_percent=20.0, + window="primary", + reset_at=now_epoch + 3600, + window_minutes=300, + ) + await StickySessionsRepository(session).upsert( + raw_session, + owner_id, + kind=StickySessionKind.CODEX_SESSION, + ) + + seen: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False, **kwargs): + del payload, headers, access_token, base_url, raise_for_status, kwargs + seen.append(account_id) + yield f'data: {{"type":"response.completed","response":{{"id":"resp_goal_thread_{len(seen)}"}}}}\n\n' + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + restart_payload = { + "model": "gpt-5.1", + "instructions": "Continue the existing task.", + "input": [ + { + "role": "developer", + "content": ('\nContinue working toward the active thread goal.'), + }, + {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ], + "stream": True, + } + + healthy_response = await async_client.post( + "/backend-api/codex/responses", + json=restart_payload, + headers=headers, + ) + assert healthy_response.status_code == 200 + assert seen == ["acc_goal_restart_thread_owner"] + + async with SessionLocal() as session: + await session.execute(update(Account).where(Account.id == owner_id).values(status=AccountStatus.QUOTA_EXCEEDED)) + await session.commit() + + restart_response = await async_client.post( + "/backend-api/codex/responses", + json=restart_payload, + headers=headers, + ) + assert restart_response.status_code == 200 + assert seen == ["acc_goal_restart_thread_owner", "acc_goal_restart_thread_replacement"] + + follow_up_response = await async_client.post( + "/backend-api/codex/responses", + json={"model": "gpt-5.1", "instructions": "continue", "input": [], "stream": True}, + headers=headers, + ) + assert follow_up_response.status_code == 200 + assert seen == [ + "acc_goal_restart_thread_owner", + "acc_goal_restart_thread_replacement", + "acc_goal_restart_thread_replacement", + ] + + turn_state_response = await async_client.post( + "/backend-api/codex/responses", + json={"model": "gpt-5.1", "instructions": "continue", "input": [], "stream": True}, + headers={"x-codex-turn-state": raw_session}, + ) + assert turn_state_response.status_code == 502 + assert turn_state_response.json()["error"]["code"] == "turn_state_owner_unavailable" + assert seen == [ + "acc_goal_restart_thread_owner", + "acc_goal_restart_thread_replacement", + "acc_goal_restart_thread_replacement", + ] + + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + raw_row = ( + await session.execute( + select(StickySession).where( + StickySession.key == raw_session, + StickySession.kind == StickySessionKind.CODEX_SESSION, + ) + ) + ).scalar_one() + thread_row = ( + await session.execute( + select(StickySession).where( + StickySession.key == thread_key, + StickySession.kind == StickySessionKind.PROMPT_CACHE, + ) + ) + ).scalar_one() + session_header_lookup = await repo.get_account_id_and_abandonment( + raw_session, + kind=StickySessionKind.CODEX_SESSION, + continuity_source="session_header", + ) + thread_legacy_lookup = await repo.get_account_id_and_abandonment( + raw_session, + kind=StickySessionKind.CODEX_SESSION, + continuity_source="thread_header", + ) + turn_state_owner = await repo.get_account_id( + raw_session, + kind=StickySessionKind.CODEX_SESSION, + continuity_source="turn_state", + ) + assert raw_row.account_id == owner_id + assert raw_row.continuity_abandonment_scope == "session_header" + assert thread_row.account_id == replacement_id + assert session_header_lookup.account_id is None + assert session_header_lookup.continuity_abandoned is True + assert thread_legacy_lookup.account_id == owner_id + assert turn_state_owner == owner_id + + +@pytest.mark.asyncio +async def test_codex_goal_restart_cas_miss_reloads_concurrently_rebound_raw_owner( + async_client, + monkeypatch, +): + from sqlalchemy import select + + from app.db.models import StickySession + from app.modules.proxy.sticky_repository import StickySessionsRepository + + _install_proxy_settings_cache(monkeypatch, sticky_threads_enabled=False) + stale_owner_id = await _import_account( + async_client, + "acc_goal_restart_stale_owner", + "goal-restart-stale-owner@example.com", + ) + rebound_owner_id = await _import_account( + async_client, + "acc_goal_restart_rebound_owner", + "goal-restart-rebound-owner@example.com", + ) + raw_session = "goal-restart-cas-reread" + now_epoch = int(utcnow().replace(tzinfo=timezone.utc).timestamp()) + async with SessionLocal() as session: + usage_repo = UsageRepository(session) + for account_id, used_percent in ( + (stale_owner_id, 10.0), + (rebound_owner_id, 20.0), + ): + await usage_repo.add_entry( + account_id=account_id, + used_percent=used_percent, + window="primary", + reset_at=now_epoch + 3600, + window_minutes=300, + ) + await StickySessionsRepository(session).upsert( + raw_session, + stale_owner_id, + kind=StickySessionKind.CODEX_SESSION, + ) + await session.execute( + update(Account).where(Account.id == stale_owner_id).values(status=AccountStatus.QUOTA_EXCEEDED) + ) + await session.commit() + + original_tombstone = StickySessionsRepository.abandon_legacy_session_header_owner_if_unavailable + race_count = 0 + + async def rebind_before_tombstone( + self, + key: str, + *, + kind: StickySessionKind, + expected_account_id: str, + ) -> bool: + nonlocal race_count + if key == raw_session and race_count == 0: + race_count += 1 + # Simulate another selector establishing a newer raw owner after + # this request cached the stale owner but before its CAS executes. + await self.upsert(key, rebound_owner_id, kind=kind) + return await original_tombstone( + self, + key, + kind=kind, + expected_account_id=expected_account_id, + ) + + seen: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False, **kwargs): + del payload, headers, access_token, base_url, raise_for_status, kwargs + seen.append(account_id) + yield 'data: {"type":"response.completed","response":{"id":"resp_goal_cas_reread"}}\n\n' + + monkeypatch.setattr( + StickySessionsRepository, + "abandon_legacy_session_header_owner_if_unavailable", + rebind_before_tombstone, + ) + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + + response = await async_client.post( + "/backend-api/codex/responses", + headers={"session_id": raw_session}, + json={ + "model": "gpt-5.1", + "instructions": "Continue the existing task.", + "input": [ + { + "role": "developer", + "content": ( + '\nContinue working toward the active thread goal.' + ), + }, + {"role": "user", "content": "continue"}, + ], + "stream": True, + }, + ) + + assert response.status_code == 200 + assert race_count == 1 + assert seen == ["acc_goal_restart_rebound_owner"] + async with SessionLocal() as session: + raw_row = await session.scalar( + select(StickySession).where( + StickySession.key == raw_session, + StickySession.kind == StickySessionKind.CODEX_SESSION, + ) + ) + assert raw_row is not None + assert raw_row.account_id == rebound_owner_id + assert raw_row.continuity_abandoned_at is None + + +@pytest.mark.asyncio +async def test_codex_goal_restart_cannot_retire_owner_outside_api_key_scope( + async_client, + monkeypatch, +): + from sqlalchemy import select + + from app.db.models import StickySession + from app.modules.proxy.sticky_repository import StickySessionsRepository + + settings_response = await async_client.put( + "/api/settings", + json={ + "stickyThreadsEnabled": False, + "preferEarlierResetAccounts": False, + "apiKeyAuthEnabled": True, + }, + ) + assert settings_response.status_code == 200 + owner_id = await _import_account( + async_client, + "acc_goal_restart_out_of_scope_owner", + "goal-restart-out-of-scope-owner@example.com", + ) + replacement_id = await _import_account( + async_client, + "acc_goal_restart_scoped_replacement", + "goal-restart-scoped-replacement@example.com", + ) + raw_session = "goal-restart-scoped-session" + now_epoch = int(utcnow().replace(tzinfo=timezone.utc).timestamp()) + async with SessionLocal() as session: + usage_repo = UsageRepository(session) + await usage_repo.add_entry( + account_id=replacement_id, + used_percent=10.0, + window="primary", + reset_at=now_epoch + 3600, + window_minutes=300, + ) + await StickySessionsRepository(session).upsert( + raw_session, + owner_id, + kind=StickySessionKind.CODEX_SESSION, + ) + await session.execute(update(Account).where(Account.id == owner_id).values(status=AccountStatus.QUOTA_EXCEEDED)) + await session.commit() + async with SessionLocal() as session: + created_key = await ApiKeysService(ApiKeysRepository(session)).create_key( + ApiKeyCreateData( + name="goal restart scoped replacement", + allowed_models=None, + assigned_account_ids=[replacement_id], + ) + ) + + _install_proxy_settings_cache( + monkeypatch, + sticky_threads_enabled=False, + proxy_request_budget_seconds=0.05, + ) + + async def fail_stream(*args, **kwargs): + del args, kwargs + raise AssertionError("an out-of-scope owner must fail closed before upstream dispatch") + if False: + yield "" + + monkeypatch.setattr(proxy_module, "core_stream_responses", fail_stream) + response = await async_client.post( + "/backend-api/codex/responses", + headers={ + "Authorization": f"Bearer {created_key.key}", + "session_id": raw_session, + }, + json={ + "model": "gpt-5.1", + "instructions": "Continue the existing task.", + "input": [ + { + "role": "developer", + "content": ( + '\nContinue working toward the active thread goal.' + ), + }, + {"role": "user", "content": "continue"}, + ], + "stream": True, + }, + ) + + assert response.status_code == 200 + events = [ + json.loads(line.removeprefix("data: ")) + for line in response.text.splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ] + failed_event = next(event for event in events if event.get("type") == "response.failed") + assert failed_event["response"]["error"]["code"] == "hard_affinity_saturated" + async with SessionLocal() as session: + raw_row = await session.scalar( + select(StickySession).where( + StickySession.key == raw_session, + StickySession.kind == StickySessionKind.CODEX_SESSION, + ) + ) + assert raw_row is not None + assert raw_row.account_id == owner_id + assert raw_row.continuity_abandoned_at is None + + @pytest.mark.asyncio async def test_proxy_sticky_switches_when_pinned_rate_limited(async_client, monkeypatch): await _set_routing_settings(async_client, sticky_threads_enabled=True) @@ -494,6 +1039,70 @@ async def fake_compact(payload, headers, access_token, account_id): assert stream_seen == ["acc_sid_a", "acc_sid_a"] +@pytest.mark.asyncio +async def test_backend_thread_rows_route_sibling_responses_and_compact_independently( + async_client, + monkeypatch, +): + from app.modules.proxy.affinity import _codex_backend_identity + from app.modules.proxy.sticky_repository import StickySessionsRepository + + await _set_routing_settings(async_client, sticky_threads_enabled=False) + account_a_id = await _import_account(async_client, "acc_thread_route_a", "thread-route-a@example.com") + account_b_id = await _import_account(async_client, "acc_thread_route_b", "thread-route-b@example.com") + process_session = "process-thread-route-shared" + root_headers = {"session-id": process_session, "thread-id": "thread-route-root"} + child_headers = {"session-id": process_session, "thread-id": "thread-route-child"} + root_key = _codex_backend_identity(root_headers).thread_selection_key + child_key = _codex_backend_identity(child_headers).thread_selection_key + assert root_key is not None + assert child_key is not None + + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + await repo.upsert(root_key, account_a_id, kind=StickySessionKind.PROMPT_CACHE) + await repo.upsert(child_key, account_b_id, kind=StickySessionKind.PROMPT_CACHE) + + observed: list[tuple[str, str, str | None]] = [] + + async def fake_stream(payload, headers, access_token, account_id, **kwargs): + del headers, access_token, kwargs + observed.append(("responses", account_id, payload.prompt_cache_key)) + yield 'data: {"type":"response.completed","response":{"id":"resp_thread_route"}}\n\n' + + async def fake_compact(payload, headers, access_token, account_id): + del headers, access_token + observed.append(("compact", account_id, payload.prompt_cache_key)) + return OpenAIResponsePayload.model_validate({"output": []}) + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + monkeypatch.setattr(proxy_module, "core_compact_responses", fake_compact) + payload = { + "model": "gpt-5.1", + "instructions": "hi", + "input": [], + "prompt_cache_key": process_session, + } + + responses_response = await async_client.post( + "/backend-api/codex/responses", + json={**payload, "stream": True}, + headers=root_headers, + ) + compact_response = await async_client.post( + "/backend-api/codex/responses/compact", + json=payload, + headers=child_headers, + ) + + assert responses_response.status_code == 200 + assert compact_response.status_code == 200 + assert observed == [ + ("responses", "acc_thread_route_a", process_session), + ("compact", "acc_thread_route_b", process_session), + ] + + @pytest.mark.asyncio async def test_proxy_unregistered_turn_state_fails_closed_for_stream_and_compact( async_client, @@ -1665,6 +2274,132 @@ async def test_sticky_insert_if_absent_never_rebinds_existing_owner(db_setup): assert persisted_owner == "acc_live_immutable_a" +@pytest.mark.asyncio +async def test_seeded_sticky_upsert_is_atomic_and_preserves_first_seed_owner(db_setup, monkeypatch): + from sqlalchemy.exc import IntegrityError + from sqlalchemy.sql import Insert + + from app.modules.proxy.sticky_repository import StickySessionsRepository + + encryptor = TokenEncryptor() + async with SessionLocal() as session: + accounts = AccountsRepository(session) + for account_id in ("acc_seeded_a", "acc_seeded_b"): + await accounts.upsert( + Account( + id=account_id, + email=f"{account_id}@example.com", + plan_type="plus", + access_token_encrypted=encryptor.encrypt("access"), + refresh_token_encrypted=encryptor.encrypt("refresh"), + id_token_encrypted=encryptor.encrypt("id"), + last_refresh=utcnow(), + status=AccountStatus.ACTIVE, + deactivation_reason=None, + ) + ) + + seed_key = "seeded-process" + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + first_thread = await repo.upsert_with_seed_if_absent( + "seeded-thread-a", + "acc_seeded_a", + kind=StickySessionKind.PROMPT_CACHE, + seed_key=seed_key, + seed_kind=StickySessionKind.CODEX_SESSION, + ) + second_thread = await repo.upsert_with_seed_if_absent( + "seeded-thread-b", + "acc_seeded_b", + kind=StickySessionKind.PROMPT_CACHE, + seed_key=seed_key, + seed_kind=StickySessionKind.CODEX_SESSION, + ) + + assert first_thread.account_id == "acc_seeded_a" + assert second_thread.account_id == "acc_seeded_b" + assert await repo.get_account_id(seed_key, kind=StickySessionKind.CODEX_SESSION) == "acc_seeded_a" + + original_build_upsert = repo._build_upsert_statement + + def _build_failing_upsert(key: str, account_id: str, kind: StickySessionKind) -> Insert: + del account_id + return original_build_upsert(key, "missing-account", kind) + + monkeypatch.setattr(repo, "_build_upsert_statement", _build_failing_upsert) + with pytest.raises(IntegrityError): + await repo.upsert_with_seed_if_absent( + "seeded-thread-failing", + "acc_seeded_a", + kind=StickySessionKind.PROMPT_CACHE, + seed_key="seeded-process-failing", + seed_kind=StickySessionKind.CODEX_SESSION, + ) + + # The repository rolls back both statements itself, so even a caller + # that catches the failure cannot accidentally commit the seed later. + assert ( + await repo.get_account_id( + "seeded-process-failing", + kind=StickySessionKind.CODEX_SESSION, + ) + is None + ) + + +@pytest.mark.asyncio +async def test_unavailable_owner_tombstone_compare_and_set_preserves_concurrent_rebind(db_setup): + from sqlalchemy import select + + from app.db.models import StickySession + from app.modules.proxy.sticky_repository import StickySessionsRepository + + encryptor = TokenEncryptor() + async with SessionLocal() as session: + accounts = AccountsRepository(session) + for account_id, status in ( + ("acc_restart_cas_old", AccountStatus.QUOTA_EXCEEDED), + ("acc_restart_cas_new", AccountStatus.ACTIVE), + ): + await accounts.upsert( + Account( + id=account_id, + email=f"{account_id}@example.com", + plan_type="plus", + access_token_encrypted=encryptor.encrypt("access"), + refresh_token_encrypted=encryptor.encrypt("refresh"), + id_token_encrypted=encryptor.encrypt("id"), + last_refresh=utcnow(), + status=status, + deactivation_reason=None, + ) + ) + + key = "restart-owner-cas" + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + await repo.upsert(key, "acc_restart_cas_old", kind=StickySessionKind.CODEX_SESSION) + # Simulate a newer claim landing after selection read the old owner. + await repo.upsert(key, "acc_restart_cas_new", kind=StickySessionKind.CODEX_SESSION) + retired = await repo.abandon_legacy_session_header_owner_if_unavailable( + key, + kind=StickySessionKind.CODEX_SESSION, + expected_account_id="acc_restart_cas_old", + ) + row = await session.scalar( + select(StickySession).where( + StickySession.key == key, + StickySession.kind == StickySessionKind.CODEX_SESSION, + ) + ) + + assert retired is False + assert row is not None + assert row.account_id == "acc_restart_cas_new" + assert row.continuity_abandoned_at is None + + @pytest.mark.asyncio async def test_stale_expiry_cleanup_cannot_delete_fresh_rebound_owner(async_client) -> None: from sqlalchemy import select, update @@ -1807,6 +2542,7 @@ async def get_entry(self, key: str, *, kind: StickySessionKind) -> StickySession assert resolved.account_id is None assert resolved.continuity_abandoned is True + assert resolved.abandoned_account_id is None assert persisted_row is not None assert persisted_row.continuity_abandoned_at is not None @@ -2190,3 +2926,198 @@ async def test_seed_hard_sticky_outage_grace_on_startup_refreshes_only_unavailab entry = await repo.get_entry(f"turn_{account_id}", kind=StickySessionKind.CODEX_SESSION) assert entry is not None assert entry.updated_at == long_ago + + +async def _create_account(account_id: str) -> None: + encryptor = TokenEncryptor() + async with SessionLocal() as session: + await AccountsRepository(session).upsert( + Account( + id=account_id, + email=f"{account_id}@example.com", + plan_type="plus", + access_token_encrypted=encryptor.encrypt("access"), + refresh_token_encrypted=encryptor.encrypt("refresh"), + id_token_encrypted=encryptor.encrypt("id"), + last_refresh=utcnow(), + status=AccountStatus.ACTIVE, + deactivation_reason=None, + ) + ) + + +async def _backdate_sticky_row(key: str, kind: StickySessionKind, *, age_seconds: float) -> None: + from app.db.models import StickySession + + async with SessionLocal() as session: + await session.execute( + update(StickySession) + .where(StickySession.key == key, StickySession.kind == kind) + .values(updated_at=utcnow() - timedelta(seconds=age_seconds)) + ) + await session.commit() + + +@pytest.mark.asyncio +async def test_sticky_lookup_refresh_skippable_only_for_fresh_unmarked_rows(db_setup): + """refresh_skip_deadline is set only when a same-owner upsert would be a + pure updated_at rewrite: fresh within min(15s, 1% of TTL), not stamped in + the future, and free of any abandonment marker.""" + from app.db.models import StickySession + from app.modules.proxy.sticky_repository import StickySessionsRepository + + await _create_account("acc_refresh_skip") + key = "key_refresh_skip" + + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + await repo.upsert(key, "acc_refresh_skip", kind=StickySessionKind.PROMPT_CACHE) + + fresh = await repo.get_account_id_and_abandonment( + key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=1800, + ) + assert fresh.account_id == "acc_refresh_skip" + assert isinstance(fresh.refresh_skip_deadline, datetime) + # The deadline is observed_updated_at + window: never further out + # than the full window from now. + assert fresh.refresh_skip_deadline <= utcnow() + timedelta(seconds=15.0) + + # Without a TTL there is no refresh write to skip. + durable = await repo.get_account_id_and_abandonment(key, kind=StickySessionKind.PROMPT_CACHE) + assert durable.account_id == "acc_refresh_skip" + assert durable.refresh_skip_deadline is None + + # 10s old: inside the 15s cap for an 1800s TTL, but outside 1% of a 600s + # TTL (6s) — the window scales with the TTL it protects. + await _backdate_sticky_row(key, StickySessionKind.PROMPT_CACHE, age_seconds=10.0) + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + within_cap = await repo.get_account_id_and_abandonment( + key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=1800, + ) + assert within_cap.account_id == "acc_refresh_skip" + assert isinstance(within_cap.refresh_skip_deadline, datetime) + beyond_fraction = await repo.get_account_id_and_abandonment( + key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=600, + ) + assert beyond_fraction.account_id == "acc_refresh_skip" + assert beyond_fraction.refresh_skip_deadline is None + + # A future updated_at (database clock ahead of the application, or a + # restored row) is never skippable: an upper-bound-only age check would + # otherwise satisfy the window for longer than the documented bound. + await _backdate_sticky_row(key, StickySessionKind.PROMPT_CACHE, age_seconds=-30.0) + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + future_stamped = await repo.get_account_id_and_abandonment( + key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=1800, + ) + assert future_stamped.account_id == "acc_refresh_skip" + assert future_stamped.refresh_skip_deadline is None + + # An abandonment marker disqualifies the skip even on a fresh row: the + # upsert that would be skipped also clears the marker columns. + async with SessionLocal() as session: + await session.execute( + update(StickySession) + .where(StickySession.key == key, StickySession.kind == StickySessionKind.PROMPT_CACHE) + .values(updated_at=utcnow(), continuity_abandonment_scope="session_header") + ) + await session.commit() + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + marked = await repo.get_account_id_and_abandonment( + key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=1800, + continuity_source="turn_state", + ) + # Non-matching source keeps the owner, but the marker still makes a + # same-owner upsert semantic (it would clear the scope). + assert marked.account_id == "acc_refresh_skip" + assert marked.refresh_skip_deadline is None + + +@pytest.mark.asyncio +async def test_sticky_upsert_concurrent_same_key_semantics(db_setup): + """Concurrent upserts on one (key, kind) must each observe their own + write in RETURNING, keep exactly one row, and settle on one of the + written owners.""" + from sqlalchemy import func as sa_func + from sqlalchemy import select + + from app.db.models import StickySession + from app.modules.proxy.sticky_repository import StickySessionsRepository + + await _create_account("acc_conc_a") + await _create_account("acc_conc_b") + key = "key_concurrent_upsert" + started_at = utcnow() + + async def _one_upsert(index: int) -> str: + account_id = "acc_conc_a" if index % 2 == 0 else "acc_conc_b" + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + row = await repo.upsert(key, account_id, kind=StickySessionKind.PROMPT_CACHE) + assert row.key == key + # RETURNING must reflect this statement's own write, not a + # concurrent winner's row. + assert row.account_id == account_id + assert row.continuity_abandoned_at is None + return row.account_id + + results = await asyncio.gather(*(_one_upsert(index) for index in range(12))) + assert set(results) == {"acc_conc_a", "acc_conc_b"} + + async with SessionLocal() as session: + row_count = await session.scalar( + select(sa_func.count()) + .select_from(StickySession) + .where(StickySession.key == key, StickySession.kind == StickySessionKind.PROMPT_CACHE) + ) + assert row_count == 1 + final = await StickySessionsRepository(session).get_entry(key, kind=StickySessionKind.PROMPT_CACHE) + assert final is not None + assert final.account_id in {"acc_conc_a", "acc_conc_b"} + # Backend timestamps may carry second precision only. + assert final.updated_at >= started_at.replace(microsecond=0) + + +@pytest.mark.asyncio +async def test_sticky_refresh_skip_never_clobbers_concurrent_rebind(db_setup): + """A request that observed a fresh same-owner row and skipped its refresh + write must leave a concurrent rebind to another account intact.""" + from app.modules.proxy.sticky_repository import StickySessionsRepository + + await _create_account("acc_skip_old") + await _create_account("acc_skip_new") + key = "key_skip_vs_rebind" + + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + await repo.upsert(key, "acc_skip_old", kind=StickySessionKind.PROMPT_CACHE) + lookup = await repo.get_account_id_and_abandonment( + key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=1800, + ) + assert lookup.account_id == "acc_skip_old" + # The selection layer would skip its same-owner refresh here. + assert isinstance(lookup.refresh_skip_deadline, datetime) + + # Concurrent request rebinds the mapping while the first request is still + # in flight; the first request performs no compensating write. + async with SessionLocal() as session: + await StickySessionsRepository(session).upsert(key, "acc_skip_new", kind=StickySessionKind.PROMPT_CACHE) + + async with SessionLocal() as session: + final = await StickySessionsRepository(session).get_account_id(key, kind=StickySessionKind.PROMPT_CACHE) + assert final == "acc_skip_new" diff --git a/tests/integration/test_proxy_warmup.py b/tests/integration/test_proxy_warmup.py index 19260d713f..9d2d812d11 100644 --- a/tests/integration/test_proxy_warmup.py +++ b/tests/integration/test_proxy_warmup.py @@ -15,7 +15,7 @@ from app.core.clients.proxy import ProxyResponseError from app.core.config.settings import get_settings from app.core.errors import openai_error -from app.core.exceptions import ProxyRateLimitError +from app.core.exceptions import ProxyAuthError, ProxyRateLimitError from app.core.openai.models import CompactResponsePayload from app.core.upstream_proxy import ResolvedProxyEndpoint, ResolvedUpstreamRoute, UpstreamProxyRouteError from app.core.utils.time import utcnow @@ -962,6 +962,62 @@ async def _fake_compact(payload, headers, access_token, account_id, session=None assert peak_compact_calls == 5 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("error_type", "error_code", "error_message"), + [ + (ProxyAuthError, "auth_error", "account unauthorized"), + (ProxyRateLimitError, "rate_limit_exceeded", "account limited"), + ], + ids=["auth", "rate-limit"], +) +async def test_single_account_pre_submit_failure_returns_summary( + async_client, + monkeypatch, + error_type, + error_code, + error_message, +): + await _enable_api_key_auth(async_client) + raw_account_id = "acc-warmup-single-failure" + account_id = await _import_account(async_client, raw_account_id, "warmup-single-failure@example.com") + await _add_primary_usage(account_id, used_percent=0.0, window_minutes=300) + _, key = await _create_api_key(async_client, name="warmup-single-failure") + + async def _fake_ensure_fresh(self, account, *, force=False, timeout_seconds=None): + del self, force, timeout_seconds + return account + + async def _fake_compact(payload, headers, access_token, upstream_account_id, session=None): + del payload, headers, access_token, session + assert upstream_account_id == raw_account_id + raise error_type(error_message) + + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", _fake_ensure_fresh) + monkeypatch.setattr(proxy_module, "core_compact_responses", _fake_compact) + + response = await async_client.post( + "/v1/warmup", + headers={"Authorization": f"Bearer {key}"}, + json={"mode": "force"}, + ) + + assert response.status_code == 200 + assert response.json() == { + "mode": "force", + "total_accounts": 1, + "submitted": [], + "skipped": [], + "failed": [ + { + "account_id": account_id, + "error_code": error_code, + "error_message": error_message, + } + ], + } + + @pytest.mark.asyncio async def test_warmup_account_rate_limit_failure_does_not_abort_summary(async_client, monkeypatch): await _enable_api_key_auth(async_client) diff --git a/tests/integration/test_proxy_websocket_responses.py b/tests/integration/test_proxy_websocket_responses.py index b7c11cebcb..16c5ee2e1e 100644 --- a/tests/integration/test_proxy_websocket_responses.py +++ b/tests/integration/test_proxy_websocket_responses.py @@ -7,16 +7,20 @@ import logging import threading import time +import tomllib from collections import deque from datetime import datetime, timedelta, timezone +from pathlib import Path from types import SimpleNamespace from typing import Any, cast from unittest.mock import AsyncMock import pytest +from fastapi.responses import JSONResponse from fastapi.testclient import TestClient from httpx import Headers from sqlalchemy import select +from starlette.testclient import WebSocketDenialResponse from starlette.websockets import WebSocketDisconnect from websockets.asyncio.client import connect as websocket_connect @@ -28,11 +32,18 @@ UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE, WebsocketsUpstreamWebSocket, ) +from app.core.config.settings_cache import get_settings_cache from app.core.utils.request_id import get_request_id from app.db.models import Account, AccountStatus, ApiKeyUsageReservation, RequestLog from app.db.session import SessionLocal from app.modules.api_keys.repository import ApiKeysRepository -from app.modules.api_keys.service import ApiKeyCreateData, ApiKeyData, ApiKeysService, ApiKeyUsageReservationData +from app.modules.api_keys.service import ( + ApiKeyCreateData, + ApiKeyData, + ApiKeysService, + ApiKeyUsageReservationData, + LimitRuleInput, +) from app.modules.proxy._service.websocket import mixin as websocket_mixin_module from app.modules.proxy.affinity import _codex_session_selection_key from app.modules.proxy.capability_routing import ( @@ -43,6 +54,8 @@ pytestmark = pytest.mark.integration _REAL_WRITE_REQUEST_LOG = proxy_module.ProxyService._write_request_log +_CODEX_CLIENT_CONFIG = Path(__file__).resolve().parents[2] / "docs/examples/codex/config.toml" +_CODEX_DAYBREAK_PROFILE = Path(__file__).resolve().parents[2] / "docs/examples/codex/daybreak-blue.config.toml" @pytest.mark.asyncio @@ -412,12 +425,19 @@ async def prepare_persistence_rows() -> tuple[ApiKeyData, ApiKeyUsageReservation ApiKeyCreateData( name="route drain", allowed_models=None, + # Limit-free keys skip the reservation ledger entirely, and + # this test asserts drain-time settlement ownership of a + # real reservation, so give the key an applicable limit. + limits=[ + LimitRuleInput(limit_type="total_tokens", limit_window="weekly", max_value=1_000_000), + ], ) ) usage_reservation = await service.enforce_limits_for_request( created_key.id, request_model="gpt-5.6-sol", ) + assert usage_reservation is not None return created_key, usage_reservation async def read_persisted_results( @@ -590,6 +610,50 @@ def _capability_test_api_key(key_id: str) -> ApiKeyData: ) +def test_responses_websocket_route_rejects_disallowed_reasoning_before_upstream(app_instance, monkeypatch): + async def create_key(): + async with SessionLocal() as session: + service = ApiKeysService(ApiKeysRepository(session)) + created = await service.create_key( + ApiKeyCreateData( + name="websocket-reasoning-policy", + allowed_models=None, + allowed_reasoning_efforts=["low"], + ) + ) + return created.key, await service.get_key_by_id(created.id) + + with TestClient(app_instance) as client: + assert client.portal is not None + key, api_key = client.portal.call(create_key) + + async def allow_proxy_api_key(_authorization: str | None, *, request: object | None = None): + del request + return api_key + + monkeypatch.setattr(proxy_api_module, "validate_proxy_api_key_authorization", allow_proxy_api_key) + + with client.websocket_connect( + "/backend-api/codex/responses", + headers={"Authorization": f"Bearer {key}"}, + ) as websocket: + websocket.send_text( + json.dumps( + { + "type": "response.create", + "model": "model-alpha", + "input": "hi", + "reasoning": {"effort": "max"}, + } + ) + ) + event = json.loads(websocket.receive_text()) + + assert event["type"] == "error" + assert event["status"] == 403 + assert event["error"]["code"] == "reasoning_effort_not_allowed" + + def _websocket_response_batch( response_id: str, *, @@ -632,6 +696,344 @@ def _websocket_response_create(text: str) -> dict[str, object]: } +def _codex_profile_provider(profile_name: str | None) -> tuple[str, str, dict[str, Any]]: + base_config = tomllib.loads(_CODEX_CLIENT_CONFIG.read_text(encoding="utf-8")) + profile_config = ( + tomllib.loads(_CODEX_DAYBREAK_PROFILE.read_text(encoding="utf-8")) if profile_name is not None else {} + ) + provider_id = profile_config.get("model_provider", base_config["model_provider"]) + model = profile_config.get("model", base_config["model"]) + provider = base_config["model_providers"][provider_id] + return provider_id, model, provider + + +async def _create_profile_authorization(name: str) -> str: + settings = await get_settings_cache().get() + assert settings.api_key_auth_enabled is False + async with SessionLocal() as session: + created_key = await ApiKeysService(ApiKeysRepository(session)).create_key( + ApiKeyCreateData( + name=name, + allowed_models=None, + ) + ) + return f"Bearer {created_key.key}" + + +@pytest.mark.parametrize( + ("profile_name", "expected_provider_id", "expected_security_requirement"), + [ + (None, "codex-lb", False), + ("daybreak-blue", "codex-lb-daybreak-blue", True), + ], + ids=["ordinary", "daybreak-blue"], +) +@pytest.mark.parametrize( + "path", + [ + "ws://localhost/backend-api/codex/responses", + "ws://localhost/backend-api/codex/v1/responses", + ], + ids=["native", "native-v1-alias"], +) +def test_codex_provider_profiles_route_before_first_account_attempt( + app_instance, + monkeypatch, + path, + profile_name, + expected_provider_id, + expected_security_requirement, +): + provider_id, model, provider = _codex_profile_provider(profile_name) + provider_headers = cast(dict[str, str], provider.get("http_headers", {})) + normalized_provider_headers = {name.lower(): value for name, value in provider_headers.items()} + + assert provider_id == expected_provider_id + assert model == "gpt-5.6-sol" + assert provider["name"] == "openai" + assert provider["base_url"].endswith("/backend-api/codex") + assert provider["wire_api"] == "responses" + assert provider["supports_websockets"] is True + assert provider["requires_openai_auth"] is True + if expected_security_requirement: + assert provider["env_key"] == "CODEX_LB_API_KEY" + assert normalized_provider_headers == {REQUIRED_CAPABILITY_HEADER: "trusted_cyber"} + else: + assert "env_key" not in provider + assert REQUIRED_CAPABILITY_HEADER not in normalized_provider_headers + + upstream = _SequencedUpstreamWebSocket( + [], + deferred_message_batches=[_websocket_response_batch(f"resp_profile_{profile_name or 'ordinary'}")], + ) + selection_requirements: list[bool] = [] + opened_account_ids: list[str] = [] + forwarded_capability_headers: list[bool] = [] + + class _FakeSettingsCache: + async def get(self): + return _websocket_settings() + + async def allow_firewall(_websocket): + return None + + async def prepare_profile_authorization() -> str | None: + if not expected_security_requirement: + return None + return await _create_profile_authorization("Inert Daybreak profile") + + async def keep_authenticated_api_key_policy(self, current_api_key): + del self + if expected_security_requirement: + assert current_api_key is not None + assert current_api_key.name == "Inert Daybreak profile" + else: + assert current_api_key is None + return current_api_key + + async def bypass_api_key_usage_reservation(self, current_api_key, **_kwargs): + del self, _kwargs + if expected_security_requirement: + assert current_api_key is not None + assert current_api_key.name == "Inert Daybreak profile" + else: + assert current_api_key is None + return None + + async def fake_select_websocket_connect_account( + self, + deadline, + *, + request_state, + require_security_work_authorized, + **_kwargs, + ): + del self, deadline, request_state, _kwargs + selection_requirements.append(require_security_work_authorized) + account_kind = "cyber" if require_security_work_authorized else "ordinary" + return SimpleNamespace( + id=f"acct_profile_{account_kind}", + security_work_authorized=require_security_work_authorized, + ) + + async def fake_try_open_websocket_connect_attempt(self, account, headers, **_kwargs): + del self, _kwargs + opened_account_ids.append(account.id) + forwarded_capability_headers.append(any(name.lower() == REQUIRED_CAPABILITY_HEADER for name in headers)) + return account, upstream + + monkeypatch.setattr(proxy_api_module, "_websocket_firewall_denial_response", allow_firewall) + monkeypatch.setattr(proxy_module, "get_settings_cache", lambda: _FakeSettingsCache()) + monkeypatch.setattr( + proxy_module.ProxyService, + "_refresh_websocket_api_key_policy", + keep_authenticated_api_key_policy, + ) + monkeypatch.setattr( + proxy_module.ProxyService, + "_reserve_websocket_api_key_usage", + bypass_api_key_usage_reservation, + ) + monkeypatch.setattr( + proxy_module.ProxyService, + "_select_websocket_connect_account", + fake_select_websocket_connect_account, + ) + monkeypatch.setattr( + proxy_module.ProxyService, + "_try_open_websocket_connect_attempt", + fake_try_open_websocket_connect_attempt, + ) + + response_create = _websocket_response_create("inert profile routing check") + response_create["model"] = model + + with TestClient(app_instance, client=("127.0.0.1", 50000)) as client: + assert client.portal is not None + websocket_headers = dict(provider_headers) + authorization = client.portal.call(prepare_profile_authorization) + if authorization is not None: + websocket_headers["Authorization"] = authorization + with client.websocket_connect( + path, + headers=websocket_headers, + ) as websocket: + websocket.send_text(json.dumps(response_create)) + created = json.loads(websocket.receive_text()) + completed = json.loads(websocket.receive_text()) + + assert created["response"]["id"] == f"resp_profile_{profile_name or 'ordinary'}" + assert completed["type"] == "response.completed" + assert selection_requirements == [expected_security_requirement] + expected_account_kind = "cyber" if expected_security_requirement else "ordinary" + assert opened_account_ids == [f"acct_profile_{expected_account_kind}"] + assert forwarded_capability_headers == [False] + + +@pytest.mark.parametrize( + ("path", "payload"), + [ + ( + "/backend-api/codex/responses", + {"model": "gpt-5.6-sol", "input": "inert fallback check", "stream": True}, + ), + ( + "/backend-api/codex/v1/responses", + {"model": "gpt-5.6-sol", "input": "inert fallback check", "stream": True}, + ), + ( + "/v1/responses", + {"model": "gpt-5.6-sol", "input": "inert fallback check", "stream": True}, + ), + ( + "/backend-api/codex/responses/compact", + {"model": "gpt-5.6-sol", "instructions": "", "input": "inert fallback check"}, + ), + ( + "/v1/responses/compact", + {"model": "gpt-5.6-sol", "instructions": "", "input": "inert fallback check"}, + ), + ], + ids=["backend", "backend-v1-alias", "v1", "backend-compact", "v1-compact"], +) +def test_daybreak_profile_http_fallback_fails_closed_before_routing( + app_instance, + monkeypatch, + path, + payload, +): + _provider_id, _model, provider = _codex_profile_provider("daybreak-blue") + provider_headers = cast(dict[str, str], provider["http_headers"]) + + async def fail_before_routing(*_args, **_kwargs): + pytest.fail("capability-bearing HTTP fallback must fail before routing") + + monkeypatch.setattr(proxy_api_module, "_select_responses_model_source", fail_before_routing) + monkeypatch.setattr(proxy_api_module, "_stream_responses", fail_before_routing) + monkeypatch.setattr(proxy_api_module, "_compact_responses", fail_before_routing) + + with TestClient( + app_instance, + base_url="http://lb.example", + client=("203.0.113.10", 50000), + ) as client: + assert client.portal is not None + authorization = client.portal.call(_create_profile_authorization, "Inert Daybreak HTTP fallback") + response = client.post( + path, + json=payload, + headers={"Authorization": authorization, **provider_headers}, + ) + + assert response.status_code == 400 + assert response.json() == { + "error": { + "code": "required_capability_transport_unsupported", + "message": "Required capability routing is only supported over the Responses WebSocket transport.", + "type": "invalid_request_error", + } + } + + +@pytest.mark.parametrize("authorization", [None, "Bearer invalid-profile-key"], ids=["missing", "invalid"]) +def test_daybreak_profile_http_fallback_requires_valid_api_key_before_transport_denial( + app_instance, + monkeypatch, + authorization, +): + _provider_id, _model, provider = _codex_profile_provider("daybreak-blue") + provider_headers = cast(dict[str, str], provider["http_headers"]) + + async def fail_before_routing(*_args, **_kwargs): + pytest.fail("unauthenticated capability-bearing HTTP fallback must not route") + + monkeypatch.setattr(proxy_api_module, "_select_responses_model_source", fail_before_routing) + monkeypatch.setattr(proxy_api_module, "_stream_responses", fail_before_routing) + headers = dict(provider_headers) + if authorization is not None: + headers["Authorization"] = authorization + + with TestClient( + app_instance, + base_url="http://lb.example", + client=("203.0.113.10", 50000), + ) as client: + response = client.post( + "/backend-api/codex/responses", + json={"model": "gpt-5.6-sol", "input": "inert fallback auth check", "stream": True}, + headers=headers, + ) + + assert response.status_code == 401 + assert response.json()["error"]["code"] == "invalid_api_key" + + +@pytest.mark.parametrize("authorization", [None, "Bearer invalid-profile-key"], ids=["missing", "invalid"]) +def test_daybreak_profile_websocket_requires_valid_api_key_before_selection( + app_instance, + monkeypatch, + authorization, +): + _provider_id, _model, provider = _codex_profile_provider("daybreak-blue") + provider_headers = cast(dict[str, str], provider["http_headers"]) + + async def fail_before_selection(*_args, **_kwargs): + pytest.fail("unauthenticated Daybreak WebSocket must not select an account") + + monkeypatch.setattr( + proxy_module.ProxyService, + "_select_websocket_connect_account", + fail_before_selection, + ) + headers = dict(provider_headers) + if authorization is not None: + headers["Authorization"] = authorization + + with TestClient(app_instance, client=("127.0.0.1", 50000)) as client: + with pytest.raises(WebSocketDenialResponse) as denial: + with client.websocket_connect( + "ws://localhost/backend-api/codex/responses", + headers=headers, + ): + pytest.fail("unauthenticated Daybreak WebSocket must not connect") + + assert denial.value.status_code == 401 + assert denial.value.json()["error"]["code"] == "invalid_api_key" + + +def test_ordinary_profile_http_path_remains_unauthenticated_and_unconstrained( + app_instance, + monkeypatch, +): + _provider_id, _model, provider = _codex_profile_provider(None) + provider_headers = cast(dict[str, str], provider.get("http_headers", {})) + normalized_headers = {name.lower(): value for name, value in provider_headers.items()} + assert REQUIRED_CAPABILITY_HEADER not in normalized_headers + + async def no_source(*_args, **_kwargs): + return None + + async def ordinary_stream(_request, _payload, _context, api_key, **_kwargs): + assert api_key is None + return JSONResponse({"ordinary": True}) + + monkeypatch.setattr(proxy_api_module, "_select_responses_model_source", no_source) + monkeypatch.setattr(proxy_api_module, "_stream_responses", ordinary_stream) + + with TestClient( + app_instance, + base_url="http://localhost", + client=("127.0.0.1", 50000), + ) as client: + response = client.post( + "/backend-api/codex/responses", + json={"model": "gpt-5.6-sol", "input": "ordinary path check", "stream": True}, + ) + + assert response.status_code == 200 + assert response.json() == {"ordinary": True} + + def test_backend_responses_websocket_fails_over_confirmed_proxy_connect_before_dispatch( app_instance, monkeypatch, @@ -3586,6 +3988,146 @@ async def fake_connect_proxy_websocket( ] +def test_backend_responses_websocket_goal_restart_retires_reused_socket_and_keeps_full_resend( + app_instance, + monkeypatch, +): + def upstream_messages(response_id: str) -> list[_FakeUpstreamMessage]: + return [ + _FakeUpstreamMessage( + "text", + text=json.dumps( + {"type": "response.created", "response": {"id": response_id, "status": "in_progress"}}, + separators=(",", ":"), + ), + ), + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": response_id, + "status": "completed", + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + }, + separators=(",", ":"), + ), + ), + ] + + owner_upstream = _FakeUpstreamWebSocket(upstream_messages("resp_goal_owner")) + replacement_upstream = _FakeUpstreamWebSocket(upstream_messages("resp_goal_replacement")) + upstreams = deque([owner_upstream, replacement_upstream]) + selections: list[dict[str, object]] = [] + + class _FakeSettingsCache: + async def get(self): + return _websocket_settings() + + async def allow_firewall(_websocket): + return None + + async def allow_proxy_api_key(authorization: str | None, *, request: object | None = None): + del request + assert authorization == "Bearer external-token" + return None + + async def fake_connect_proxy_websocket( + self, + headers, + *, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset, + prefer_earlier_reset_window, + routing_strategy, + model, + request_state, + api_key, + client_send_lock, + websocket, + ): + del ( + self, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset, + prefer_earlier_reset_window, + routing_strategy, + model, + api_key, + client_send_lock, + websocket, + ) + selections.append( + { + "headers": dict(headers), + "sticky_key": sticky_key, + "sticky_kind": sticky_kind, + "abandon_unavailable_legacy_owner": (request_state.affinity_policy.abandon_unavailable_legacy_owner), + } + ) + account_id = "acct_goal_owner" if len(selections) == 1 else "acct_goal_replacement" + return SimpleNamespace(id=account_id), upstreams.popleft() + + monkeypatch.setattr(proxy_api_module, "_websocket_firewall_denial_response", allow_firewall) + monkeypatch.setattr(proxy_api_module, "validate_proxy_api_key_authorization", allow_proxy_api_key) + monkeypatch.setattr(proxy_module, "get_settings_cache", lambda: _FakeSettingsCache()) + monkeypatch.setattr(proxy_module.ProxyService, "_connect_proxy_websocket", fake_connect_proxy_websocket) + + first_input = {"role": "user", "content": [{"type": "input_text", "text": "first"}]} + continued_input = { + "role": "user", + "content": [{"type": "input_text", "text": "continue the goal"}], + } + retained_output = { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "first answer"}], + } + first_payload = { + "type": "response.create", + "model": "gpt-5.4", + "instructions": "Work on the task.", + "input": [first_input], + "stream": True, + } + restart_payload = { + **first_payload, + "instructions": ('\nContinue working toward the active thread goal.'), + "input": [first_input, retained_output, continued_input], + } + + with TestClient(app_instance) as client: + with client.websocket_connect( + "/backend-api/codex/responses", + headers={"Authorization": "Bearer external-token", "session_id": "goal-restart-direct"}, + ) as websocket: + websocket.send_text(json.dumps(first_payload)) + assert json.loads(websocket.receive_text())["type"] == "response.created" + assert json.loads(websocket.receive_text())["type"] == "response.completed" + + websocket.send_text(json.dumps(restart_payload)) + assert json.loads(websocket.receive_text())["type"] == "response.created" + assert json.loads(websocket.receive_text())["type"] == "response.completed" + + assert len(selections) == 2 + assert selections[0]["abandon_unavailable_legacy_owner"] is False + assert selections[1]["abandon_unavailable_legacy_owner"] is True + assert "x-codex-turn-state" not in cast(dict[str, str], selections[1]["headers"]) + assert owner_upstream.closed is True + assert len(owner_upstream.sent_text) == 1 + assert len(replacement_upstream.sent_text) == 1 + replacement_payload = json.loads(replacement_upstream.sent_text[0]) + assert "previous_response_id" not in replacement_payload + assert replacement_payload["input"] == [first_input, retained_output, continued_input] + + def test_backend_responses_websocket_reconnect_keeps_session_affinity_with_fresh_generated_turn_states( app_instance, monkeypatch, @@ -3867,6 +4409,8 @@ def test_v1_responses_websocket_reuses_upstream_for_sequential_requests(app_inst ], ) connect_calls: list[dict[str, object]] = [] + dispatch_owner_snapshots: list[tuple[str | None, str | None]] = [] + original_bind_dispatch_owner = websocket_mixin_module._bind_websocket_request_dispatch_owner class _FakeSettingsCache: async def get(self): @@ -3907,7 +4451,18 @@ async def fake_connect_proxy_websocket( "model": model, } ) - return SimpleNamespace(id=f"acct_ws_proxy_{len(connect_calls)}"), first_upstream + return SimpleNamespace(id="acct_ws_proxy_owner"), first_upstream + + def capture_dispatch_owner(*args, **kwargs): + bound = original_bind_dispatch_owner(*args, **kwargs) + request_state = args[0] if args else kwargs["request_state"] + dispatch_owner_snapshots.append( + ( + request_state.preferred_account_id, + request_state.replay_required_account_id, + ) + ) + return bound async def fake_write_request_log(self, **kwargs): del self, kwargs @@ -3917,6 +4472,11 @@ async def fake_write_request_log(self, **kwargs): monkeypatch.setattr(proxy_module, "get_settings_cache", lambda: _FakeSettingsCache()) monkeypatch.setattr(proxy_module.ProxyService, "_connect_proxy_websocket", fake_connect_proxy_websocket) monkeypatch.setattr(proxy_module.ProxyService, "_write_request_log", fake_write_request_log) + monkeypatch.setattr( + websocket_mixin_module, + "_bind_websocket_request_dispatch_owner", + capture_dispatch_owner, + ) first_request = { "type": "response.create", @@ -3929,6 +4489,7 @@ async def fake_write_request_log(self, **kwargs): "type": "response.create", "model": "gpt-5.5", "input": "second", + "account_bound_probe": "owner-bound", "promptCacheKey": "thread_b", "stream": True, } @@ -3947,6 +4508,10 @@ async def fake_write_request_log(self, **kwargs): assert connect_calls[0]["sticky_key"] == "thread_a" assert connect_calls[0]["sticky_kind"] == proxy_module.StickySessionKind.PROMPT_CACHE assert connect_calls[0]["model"] == "gpt-5.4" + assert dispatch_owner_snapshots == [ + (None, None), + ("acct_ws_proxy_owner", "acct_ws_proxy_owner"), + ] _assert_upstream_payloads( first_upstream.sent_text, [ @@ -3963,6 +4528,7 @@ async def fake_write_request_log(self, **kwargs): "model": "gpt-5.5", "instructions": "", "input": [{"role": "user", "content": [{"type": "input_text", "text": "second"}]}], + "account_bound_probe": "owner-bound", "store": False, "include": [], "prompt_cache_key": "thread_b", @@ -5466,9 +6032,7 @@ def test_responses_websocket_replays_client_full_resend_previous_response_miss_w "status": 400, "error": { "type": "invalid_request_error", - "code": "previous_response_not_found", - "message": "Previous response with id 'resp_ws_prev_anchor' not found.", - "param": "previous_response_id", + "message": "Invalid `previous_response_id`.", }, }, separators=(",", ":"), @@ -5650,9 +6214,7 @@ def test_v1_responses_websocket_masks_invalid_request_previous_response_not_foun "status": 400, "error": { "type": "invalid_request_error", - "code": "invalid_request_error", - "message": ("Previous response with id 'resp_ws_prev_anchor' not found."), - "param": "previous_response_id", + "message": "Invalid `previous_response_id`.", }, }, separators=(",", ":"), @@ -5903,9 +6465,26 @@ async def fake_try_open_websocket_connect_attempt( _assert_previous_response_not_found_error(event["error"]) +@pytest.mark.parametrize( + "upstream_error", + [ + { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": "Previous response with id 'resp_ws_prev_anchor' not found.", + "param": "previous_response_id", + }, + { + "type": "invalid_request_error", + "message": "Invalid `previous_response_id`.", + }, + ], + ids=["canonical-not-found", "parameterless-invalid-id"], +) def test_backend_responses_websocket_masks_short_previous_response_not_found_without_retry( app_instance, monkeypatch, + upstream_error, ): first_upstream = _SequencedUpstreamWebSocket( [], @@ -5939,12 +6518,7 @@ def test_backend_responses_websocket_masks_short_previous_response_not_found_wit { "type": "error", "status": 400, - "error": { - "type": "invalid_request_error", - "code": "previous_response_not_found", - "message": "Previous response with id 'resp_ws_prev_anchor' not found.", - "param": "previous_response_id", - }, + "error": upstream_error, }, separators=(",", ":"), ), @@ -8333,7 +8907,6 @@ def send_oversized_request() -> dict[str, Any]: assert duplicate_event["status"] == 400 assert len(list(tmp_path.glob("*.response-create.json.gz"))) == 1 assert len(list(tmp_path.glob("*.meta.json"))) == 1 - meta_files[0].unlink() with TestClient(app_instance) as client: orphan_retry_event = send_oversized_request() @@ -8346,6 +8919,88 @@ def send_oversized_request() -> dict[str, Any]: assert complete_pairs +def test_backend_responses_websocket_rejects_non_terminal_compaction_trigger_before_upstream( + app_instance, + monkeypatch, +): + class _FakeSettingsCache: + async def get(self): + return _websocket_settings() + + async def allow_firewall(_websocket): + return None + + async def allow_proxy_api_key(_authorization: str | None, *, request: object | None = None): + return None + + async def fail_connect_proxy_websocket( + self, + headers, + *, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset, + prefer_earlier_reset_window, + routing_strategy, + model, + request_state, + api_key, + client_send_lock, + websocket, + ): + del ( + self, + headers, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset, + prefer_earlier_reset_window, + routing_strategy, + model, + request_state, + api_key, + client_send_lock, + websocket, + ) + raise AssertionError("malformed compaction trigger must fail before upstream websocket connect") + + monkeypatch.setattr(proxy_api_module, "_websocket_firewall_denial_response", allow_firewall) + monkeypatch.setattr(proxy_api_module, "validate_proxy_api_key_authorization", allow_proxy_api_key) + monkeypatch.setattr(proxy_module, "get_settings_cache", lambda: _FakeSettingsCache()) + monkeypatch.setattr(proxy_module.ProxyService, "_connect_proxy_websocket", fail_connect_proxy_websocket) + + request_payload = { + "type": "response.create", + "model": "gpt-5.4", + "instructions": "", + "input": [ + {"role": "user", "content": [{"type": "input_text", "text": "hello"}]}, + {"type": "compaction_trigger"}, + {"role": "developer", "content": [{"type": "input_text", "text": "still trailing"}]}, + ], + "stream": True, + } + + with TestClient(app_instance) as client: + with client.websocket_connect("/backend-api/codex/responses") as websocket: + websocket.send_text(json.dumps(request_payload)) + error_event = json.loads(websocket.receive_text()) + + assert error_event["type"] == "error" + assert error_event["status"] == 400 + assert error_event["error"]["code"] == "invalid_request_error" + assert error_event["error"]["type"] == "invalid_request_error" + assert error_event["error"]["param"] == "input" + assert ( + "compaction_trigger must appear exactly once as the final top-level input item" + in error_event["error"]["message"] + ) + + def test_backend_responses_websocket_slims_historical_inline_artifacts_and_succeeds( app_instance, monkeypatch, @@ -8751,7 +9406,10 @@ async def get(self): runtime_settings = _websocket_settings( proxy_downstream_websocket_idle_timeout_seconds=0.1, - stream_idle_timeout_seconds=0.2, + # Keep the upstream stream budget above both delayed messages. The + # assertion targets the downstream idle guard, not an upstream idle + # timeout; a slower CI runner must not turn the fixture into a race. + stream_idle_timeout_seconds=0.5, ) async def allow_firewall(_websocket): @@ -10901,6 +11559,7 @@ async def fake_try_open_websocket_connect_attempt(self, account, headers, **_kwa monkeypatch.setattr(proxy_api_module, "_websocket_firewall_denial_response", allow_firewall) monkeypatch.setattr(proxy_api_module, "validate_proxy_api_key_authorization", allow_proxy_api_key) + monkeypatch.setattr(proxy_api_module, "validate_required_proxy_api_key_authorization", allow_proxy_api_key) monkeypatch.setattr(proxy_module, "get_settings_cache", lambda: _FakeSettingsCache()) monkeypatch.setattr( proxy_module.ProxyService, @@ -11614,6 +12273,7 @@ async def reject_upstream_open(*_args, **_kwargs): monkeypatch.setattr(proxy_api_module, "_websocket_firewall_denial_response", allow_firewall) monkeypatch.setattr(proxy_api_module, "validate_proxy_api_key_authorization", allow_proxy_api_key) + monkeypatch.setattr(proxy_api_module, "validate_required_proxy_api_key_authorization", allow_proxy_api_key) monkeypatch.setattr(proxy_module, "get_settings_cache", lambda: _FakeSettingsCache()) monkeypatch.setattr( proxy_module.ProxyService, @@ -12321,6 +12981,7 @@ async def fake_try_open_websocket_connect_attempt(self, account, headers, **_kwa monkeypatch.setattr(proxy_api_module, "_websocket_firewall_denial_response", allow_firewall) monkeypatch.setattr(proxy_api_module, "validate_proxy_api_key_authorization", allow_proxy_api_key) + monkeypatch.setattr(proxy_api_module, "validate_required_proxy_api_key_authorization", allow_proxy_api_key) monkeypatch.setattr(proxy_module, "get_settings_cache", lambda: _FakeSettingsCache()) monkeypatch.setattr( proxy_module.ProxyService, diff --git a/tests/integration/test_quota_planner_api.py b/tests/integration/test_quota_planner_api.py index 587bc62b37..23e65793ec 100644 --- a/tests/integration/test_quota_planner_api.py +++ b/tests/integration/test_quota_planner_api.py @@ -12,7 +12,15 @@ from app.core.crypto import TokenEncryptor from app.core.utils.time import utcnow -from app.db.models import Account, AccountStatus, QuotaPlannerDecision, QuotaWindowObservation, RequestLog, UsageHistory +from app.db.models import ( + Account, + AccountStatus, + ApiKey, + QuotaPlannerDecision, + QuotaWindowObservation, + RequestLog, + UsageHistory, +) from app.db.session import SessionLocal from app.modules.api_keys.service import ApiKeyInvalidError, ApiKeyNotFoundError, ApiKeyRateLimitExceededError from app.modules.quota_planner.logic import PlannerSettings @@ -933,6 +941,156 @@ async def cancel_probe(self, *, account, model, request_id): assert failed_reservations == [("reservation-cancelled", "gpt-5.4-mini", 0, 0, 0)] +@pytest.mark.asyncio +async def test_quota_planner_warm_now_limit_free_key_probes_without_reservation(monkeypatch, db_setup): + """A key with no applicable limits admits without a reservation; the + warmup probe must execute and never attempt reservation settlement.""" + del db_setup + encryptor = TokenEncryptor() + async with SessionLocal() as session: + account = Account( + id="acc-warm-unlimited-key", + email="warm-unlimited-key@example.test", + plan_type="plus", + access_token_encrypted=encryptor.encrypt("access"), + refresh_token_encrypted=encryptor.encrypt("refresh"), + id_token_encrypted=encryptor.encrypt("id"), + last_refresh=utcnow(), + status=AccountStatus.ACTIVE, + ) + session.add(account) + repo = QuotaPlannerRepository(session) + await repo.upsert_settings( + PlannerSettings( + mode="auto", + allow_synthetic_traffic=True, + dry_run=False, + max_warmup_credits_per_day=1.0, + warmup_model_preference="gpt-5.4-mini", + ) + ) + await repo.add_window_observation( + account_id=account.id, + model="gpt-5.4-mini", + source="warmup_probe", + confidence="observed", + ) + service = QuotaWarmupService(session) + + class FakeApiKeys: + async def enforce_limits_for_request(self, *args, **kwargs): + del args, kwargs + return None + + async def finalize_usage_reservation(self, *args, **kwargs): + del args, kwargs + raise AssertionError("limit-free warmup must not finalize a reservation") + + async def fail_usage_reservation(self, *args, **kwargs): + del args, kwargs + raise AssertionError("limit-free warmup must not fail a reservation") + + async def fake_send(self, *, account, model, request_id): + del self, account, model, request_id + return WarmupUsage(input_tokens=3, output_tokens=1, cached_input_tokens=0, reasoning_tokens=None) + + async def noop_record_effect(self, account, model, *, source, confidence): + del self, account, model, source, confidence + + monkeypatch.setattr(service, "_api_keys", FakeApiKeys()) + monkeypatch.setattr(QuotaWarmupService, "_send_warmup_probe", fake_send) + monkeypatch.setattr(QuotaWarmupService, "_record_warmup_effect", noop_record_effect) + + result = await service.warm_now( + account_id=account.id, + model="gpt-5.4-mini", + api_key_id="api-key-unlimited", + force_probe=True, + ) + + assert result.status == "executed" + assert result.reason == "warmup_executed" + + +@pytest.mark.asyncio +async def test_quota_planner_warm_now_limit_free_admission_keeps_shared_session_state(monkeypatch, db_setup): + """Regression: the limit-free early return closes the admission + transaction on the warmup service's shared session. It must do so + without expiring tracked ORM state (``rollback()`` expires everything + even with ``expire_on_commit=False``): the probe reads + ``account.access_token_encrypted`` and the error path reads + ``decision.id`` after admission, which raised ``MissingGreenlet`` when + the shared ``account``/``decision`` objects were expired. Exercises the + REAL ``ApiKeysService`` with a real limit-free key row — a fake api-key + service returning ``None`` never runs the transaction-closing path.""" + del db_setup + encryptor = TokenEncryptor() + async with SessionLocal() as session: + account = Account( + id="acc-warm-limit-free-real", + email="warm-limit-free-real@example.test", + plan_type="plus", + access_token_encrypted=encryptor.encrypt("access"), + refresh_token_encrypted=encryptor.encrypt("refresh"), + id_token_encrypted=encryptor.encrypt("id"), + last_refresh=utcnow(), + status=AccountStatus.ACTIVE, + ) + session.add(account) + # Real API key with no configured limits: admission takes the + # limit-free early return inside the real ApiKeysService. + session.add( + ApiKey( + id="api-key-limit-free-real", + name="limit-free warmup key", + key_hash="limit-free-warmup-hash", + key_prefix="sk-lfw", + ) + ) + repo = QuotaPlannerRepository(session) + await repo.upsert_settings( + PlannerSettings( + mode="auto", + allow_synthetic_traffic=True, + dry_run=False, + max_warmup_credits_per_day=1.0, + warmup_model_preference="gpt-5.4-mini", + ) + ) + service = QuotaWarmupService(session) + + probed_tokens: list[str] = [] + + async def fake_send(self, *, account, model, request_id): + del model, request_id + # Mirror the real probe's first attribute access on the shared + # session's tracked account: raises MissingGreenlet if admission + # expired it. + probed_tokens.append(self._encryptor.decrypt(account.access_token_encrypted)) + return WarmupUsage(input_tokens=3, output_tokens=1, cached_input_tokens=0, reasoning_tokens=None) + + async def noop_record_effect(self, account, model, *, source, confidence): + del self, account, model, source, confidence + + monkeypatch.setattr(QuotaWarmupService, "_send_warmup_probe", fake_send) + monkeypatch.setattr(QuotaWarmupService, "_record_warmup_effect", noop_record_effect) + + result = await service.warm_now( + account_id=account.id, + model="gpt-5.4-mini", + api_key_id="api-key-limit-free-real", + force_probe=True, + ) + + # The tracked objects must remain readable after admission (the + # failure-handling path reads ``decision.id``-style attributes too). + assert account.access_token_encrypted is not None + + assert probed_tokens == ["access"] + assert result.status == "executed" + assert result.reason == "warmup_executed" + + @pytest.mark.asyncio async def test_quota_planner_warm_now_api_key_not_found_is_skipped(monkeypatch, db_setup): del db_setup diff --git a/tests/integration/test_reports_api.py b/tests/integration/test_reports_api.py index a078a7c89c..80b92db9a7 100644 --- a/tests/integration/test_reports_api.py +++ b/tests/integration/test_reports_api.py @@ -84,6 +84,7 @@ async def test_reports_api_returns_null_account_bucket(async_client, db_setup): "requests": 2, "inputTokens": 15, "outputTokens": 5, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -105,6 +106,194 @@ async def test_reports_api_returns_null_account_bucket(async_client, db_setup): ] +async def test_reports_api_aggregates_reasoning_tokens_for_unfiltered_window(async_client, db_setup): + async with SessionLocal() as session: + session.add(_make_account("acc_reports_reasoning", "reports-reasoning@example.com")) + session.add_all( + [ + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-day-1", + requested_at=datetime(2026, 6, 1, 10, 0), + model="gpt-5.1", + status="success", + input_tokens=10, + output_tokens=40, + reasoning_tokens=30, + ), + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-missing", + requested_at=datetime(2026, 6, 1, 11, 0), + model="gpt-5.1", + status="success", + input_tokens=20, + output_tokens=20, + reasoning_tokens=None, + ), + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-zero", + requested_at=datetime(2026, 6, 1, 12, 0), + model="gpt-5.1", + status="success", + input_tokens=5, + output_tokens=10, + reasoning_tokens=0, + ), + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-day-2", + requested_at=datetime(2026, 6, 2, 10, 0), + model="gpt-5.1", + status="success", + input_tokens=30, + output_tokens=80, + reasoning_tokens=70, + ), + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-only-output-fallback", + requested_at=datetime(2026, 6, 2, 11, 0), + model="gpt-5.1", + status="success", + input_tokens=5, + output_tokens=None, + reasoning_tokens=5, + ), + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-all-unknown-day", + requested_at=datetime(2026, 6, 3, 10, 0), + model="gpt-5.1", + status="success", + input_tokens=10, + output_tokens=20, + reasoning_tokens=None, + ), + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-outside-window", + requested_at=datetime(2026, 6, 4, 10, 0), + model="gpt-5.1", + status="success", + input_tokens=100, + output_tokens=1000, + reasoning_tokens=900, + ), + ] + ) + await session.commit() + + response = await async_client.get( + "/api/reports", + params={"start_date": "2026-06-01", "end_date": "2026-06-03"}, + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["summary"]["totalReasoningTokens"] == 105 + assert payload["summary"]["reasoningUsageKnownRequests"] == 4 + assert payload["summary"]["totalOutputTokens"] == 175 + assert [(row["date"], row["outputTokens"], row["reasoningTokens"]) for row in payload["daily"]] == [ + ("2026-06-01", 70, 30), + ("2026-06-02", 85, 75), + ("2026-06-03", 20, None), + ] + + +async def test_reports_api_reasoning_tokens_honor_filters_without_double_counting_comparison( + async_client, + db_setup, +): + async with SessionLocal() as session: + session.add_all( + [ + _make_account("acc_reports_reasoning_filter", "reports-reasoning-filter@example.com"), + _make_account("acc_reports_reasoning_other", "reports-reasoning-other@example.com"), + ] + ) + session.add_all( + [ + RequestLog( + account_id="acc_reports_reasoning_filter", + request_id="report-reasoning-filter-previous", + requested_at=datetime(2026, 5, 31, 10, 0), + model="gpt-5.1", + useragent_group="opencode", + status="success", + input_tokens=50, + output_tokens=None, + reasoning_tokens=70, + ), + RequestLog( + account_id="acc_reports_reasoning_filter", + request_id="report-reasoning-filter-selected", + requested_at=datetime(2026, 6, 1, 10, 0), + model="gpt-5.1", + useragent_group="opencode", + status="success", + input_tokens=10, + output_tokens=40, + reasoning_tokens=30, + ), + RequestLog( + account_id="acc_reports_reasoning_filter", + request_id="report-reasoning-filter-other-model", + requested_at=datetime(2026, 6, 1, 11, 0), + model="gpt-5.2", + useragent_group="opencode", + status="success", + input_tokens=10, + output_tokens=110, + reasoning_tokens=100, + ), + RequestLog( + account_id="acc_reports_reasoning_other", + request_id="report-reasoning-filter-other-account", + requested_at=datetime(2026, 6, 1, 12, 0), + model="gpt-5.1", + useragent_group="opencode", + status="success", + input_tokens=10, + output_tokens=210, + reasoning_tokens=200, + ), + RequestLog( + account_id="acc_reports_reasoning_filter", + request_id="report-reasoning-filter-other-useragent", + requested_at=datetime(2026, 6, 1, 13, 0), + model="gpt-5.1", + useragent_group="CodexCLI", + status="success", + input_tokens=10, + output_tokens=310, + reasoning_tokens=300, + ), + ] + ) + await session.commit() + + response = await async_client.get( + "/api/reports", + params={ + "start_date": "2026-06-01", + "end_date": "2026-06-01", + "account_id": "acc_reports_reasoning_filter", + "model": "gpt-5.1", + "useragent_group": "opencode", + }, + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["summary"]["totalReasoningTokens"] == 30 + assert payload["summary"]["reasoningUsageKnownRequests"] == 1 + assert payload["summary"]["totalOutputTokens"] == 40 + assert payload["daily"][0]["reasoningTokens"] == 30 + assert payload["comparison"]["previous"]["totalTokens"] == 120 + + async def test_reports_api_returns_distinct_nonblank_conversation_counts(async_client, db_setup): async with SessionLocal() as session: session.add(_make_account("acc_reports_conversations", "reports-conversations@example.com")) @@ -306,6 +495,7 @@ async def test_reports_api_includes_preserved_deleted_account_history(async_clie "requests": 1, "inputTokens": 13, "outputTokens": 7, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -450,6 +640,7 @@ async def test_reports_api_interprets_dates_in_requested_timezone(async_client, "requests": 2, "inputTokens": 5, "outputTokens": 2, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -701,6 +892,7 @@ async def test_reports_api_default_range_uses_last_seven_calendar_days_in_reques "requests": 1, "inputTokens": 5, "outputTokens": 1, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -716,6 +908,7 @@ async def test_reports_api_default_range_uses_last_seven_calendar_days_in_reques "requests": 1, "inputTokens": 5, "outputTokens": 1, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -806,6 +999,7 @@ async def test_reports_api_uses_dst_aware_boundaries_for_requested_timezone(asyn "requests": 2, "inputTokens": 5, "outputTokens": 2, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -1578,6 +1772,7 @@ async def test_reports_api_summary_uses_sql_range_totals_not_rounded_daily_rows( "requests": 1, "inputTokens": 1, "outputTokens": 1, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -1593,6 +1788,7 @@ async def test_reports_api_summary_uses_sql_range_totals_not_rounded_daily_rows( "requests": 1, "inputTokens": 1, "outputTokens": 1, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -1608,8 +1804,88 @@ async def test_reports_api_summary_uses_sql_range_totals_not_rounded_daily_rows( "requests": 1, "inputTokens": 1, "outputTokens": 1, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, }, ] + + +async def test_reports_api_filters_by_api_key_id(async_client, db_setup): + start_at = _naive_utc(datetime(2026, 6, 1, 10, 0, 0, tzinfo=timezone.utc)) + async with SessionLocal() as session: + session.add(_make_account("acc_key_filter", "key-filter@example.com")) + session.add_all( + [ + RequestLog( + account_id="acc_key_filter", + api_key_id="key-1", + request_id="req-key-1", + requested_at=start_at, + model="gpt-5.1", + status="success", + input_tokens=10, + output_tokens=5, + cached_input_tokens=0, + cost_usd=0.50, + ), + RequestLog( + account_id="acc_key_filter", + api_key_id="key-2", + request_id="req-key-2", + requested_at=start_at, + model="gpt-5.1", + status="success", + input_tokens=20, + output_tokens=10, + cached_input_tokens=0, + cost_usd=1.00, + ), + ] + ) + await session.commit() + + # Test filtering by single key-1 + response1 = await async_client.get( + "/api/reports", + params={ + "start_date": "2026-06-01", + "end_date": "2026-06-01", + "api_key_id": ["key-1"], + }, + ) + assert response1.status_code == 200 + payload1 = response1.json() + assert payload1["summary"]["totalRequests"] == 1 + assert payload1["summary"]["totalCostUsd"] == 0.50 + assert payload1["daily"][0]["requests"] == 1 + assert payload1["byAccount"][0]["requests"] == 1 + + # Test filtering by multiple keys (key-1 and key-2) + response2 = await async_client.get( + "/api/reports", + params={ + "start_date": "2026-06-01", + "end_date": "2026-06-01", + "api_key_id": ["key-1", "key-2"], + }, + ) + assert response2.status_code == 200 + payload2 = response2.json() + assert payload2["summary"]["totalRequests"] == 2 + assert payload2["summary"]["totalCostUsd"] == 1.50 + + # Test filtering by non-matching key returns 0 metrics + response3 = await async_client.get( + "/api/reports", + params={ + "start_date": "2026-06-01", + "end_date": "2026-06-01", + "api_key_id": ["nonexistent-key"], + }, + ) + assert response3.status_code == 200 + payload3 = response3.json() + assert payload3["summary"]["totalRequests"] == 0 + assert payload3["summary"]["totalCostUsd"] == 0.0 diff --git a/tests/integration/test_repositories.py b/tests/integration/test_repositories.py index 9d5e639015..08983ba27d 100644 --- a/tests/integration/test_repositories.py +++ b/tests/integration/test_repositories.py @@ -1240,6 +1240,40 @@ async def test_accounts_upsert_merge_by_chatgpt_identity_skips_without_upstream_ assert saved.id.startswith("acc_no_id__copy") +@pytest.mark.asyncio +async def test_identity_reconciliation_does_not_select_identityless_local_row_as_duplicate(db_setup): + async with SessionLocal() as session: + repo = AccountsRepository(session) + canonical = _make_account_with_chatgpt_id( + "acc_identity_canonical", + "identity-invariant@example.com", + "chatgpt_identity_invariant", + ) + identityless = _make_account("acc_identityless_local", "identity-invariant@example.com") + await repo.upsert(canonical, merge_by_email=False) + await repo.upsert(identityless, merge_by_email=False) + + saved = await repo.upsert( + _make_account_with_chatgpt_id( + "acc_identity_reauth", + "identity-invariant@example.com", + "chatgpt_identity_invariant", + ), + merge_by_email=False, + merge_by_chatgpt_identity=True, + ) + + assert saved.id == canonical.id + remaining = { + account.id: account.chatgpt_account_id + for account in (await session.execute(select(Account).order_by(Account.id))).scalars().all() + } + assert remaining == { + canonical.id: "chatgpt_identity_invariant", + identityless.id: None, + } + + @pytest.mark.asyncio async def test_usage_repository_aggregate(db_setup): async with SessionLocal() as session: diff --git a/tests/integration/test_request_logs_api.py b/tests/integration/test_request_logs_api.py index a949247293..beb6427829 100644 --- a/tests/integration/test_request_logs_api.py +++ b/tests/integration/test_request_logs_api.py @@ -136,6 +136,57 @@ async def test_request_logs_api_returns_recent(async_client, db_setup): assert older["connectionRequestKind"] is None +@pytest.mark.asyncio +async def test_request_logs_api_returns_upstream_proxy_route_metadata(async_client, db_setup): + del db_setup + async with SessionLocal() as session: + logs_repo = RequestLogsRepository(session) + now = utcnow() + await logs_repo.add_log( + account_id=None, + request_id="req_route_success", + model="gpt-5.1", + input_tokens=10, + output_tokens=20, + latency_ms=100, + status="success", + error_code=None, + requested_at=now - timedelta(seconds=1), + upstream_proxy_route_mode="account_bound", + upstream_proxy_pool_id="pool_route", + upstream_proxy_endpoint_id="endpoint_route", + upstream_proxy_fallback_used=True, + ) + await logs_repo.add_log( + account_id=None, + request_id="req_route_fail_closed", + model="gpt-5.1", + input_tokens=None, + output_tokens=None, + latency_ms=0, + status="error", + error_code="upstream_proxy_unavailable", + requested_at=now, + upstream_proxy_route_mode="account_bound", + upstream_proxy_pool_id="pool_route", + upstream_proxy_fail_closed_reason="no_healthy_endpoint", + ) + + response = await async_client.get("/api/request-logs?limit=2") + assert response.status_code == 200 + fail_closed, success = response.json()["requests"] + assert fail_closed["upstreamProxyRouteMode"] == "account_bound" + assert fail_closed["upstreamProxyPoolId"] == "pool_route" + assert fail_closed["upstreamProxyEndpointId"] is None + assert fail_closed["upstreamProxyFallbackUsed"] is None + assert fail_closed["upstreamProxyFailClosedReason"] == "no_healthy_endpoint" + assert success["upstreamProxyRouteMode"] == "account_bound" + assert success["upstreamProxyPoolId"] == "pool_route" + assert success["upstreamProxyEndpointId"] == "endpoint_route" + assert success["upstreamProxyFallbackUsed"] is True + assert success["upstreamProxyFailClosedReason"] is None + + @pytest.mark.asyncio async def test_request_logs_api_returns_model_source_metadata(async_client, db_setup): del db_setup diff --git a/tests/integration/test_request_logs_filters.py b/tests/integration/test_request_logs_filters.py index ab85f47750..bff0f27dd2 100644 --- a/tests/integration/test_request_logs_filters.py +++ b/tests/integration/test_request_logs_filters.py @@ -47,6 +47,81 @@ def _cost( ) +async def _seed_cancelled_and_error_logs() -> None: + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + await accounts_repo.upsert(_make_account("acc_cancelled_filter", "cancelled-filter@example.com")) + + await logs_repo.add_log( + account_id="acc_cancelled_filter", + request_id="req_cancelled_filter", + model="gpt-5.1", + input_tokens=1, + output_tokens=0, + latency_ms=10, + status="cancelled", + error_code="client_disconnected", + requested_at=now - timedelta(minutes=1), + ) + await logs_repo.add_log( + account_id="acc_cancelled_filter", + request_id="req_cancelled_error_control", + model="gpt-5.1", + input_tokens=1, + output_tokens=0, + latency_ms=10, + status="error", + error_code="upstream_error", + error_message="upstream failure", + requested_at=now, + ) + + +@pytest.mark.asyncio +async def test_request_logs_unfiltered_includes_cancelled(async_client, db_setup): + await _seed_cancelled_and_error_logs() + + response = await async_client.get("/api/request-logs?limit=10") + + assert response.status_code == 200 + payload = response.json() + assert payload["total"] == 2 + assert {request["requestId"]: request["status"] for request in payload["requests"]} == { + "req_cancelled_error_control": "error", + "req_cancelled_filter": "cancelled", + } + + +@pytest.mark.asyncio +async def test_request_logs_status_cancelled_filters_cancelled(async_client, db_setup): + await _seed_cancelled_and_error_logs() + + response = await async_client.get("/api/request-logs?status=cancelled&limit=10") + + assert response.status_code == 200 + payload = response.json() + assert payload["total"] == 1 + assert [(request["requestId"], request["status"]) for request in payload["requests"]] == [ + ("req_cancelled_filter", "cancelled") + ] + + +@pytest.mark.asyncio +async def test_request_logs_status_error_excludes_cancelled(async_client, db_setup): + await _seed_cancelled_and_error_logs() + + response = await async_client.get("/api/request-logs?status=error&limit=10") + + assert response.status_code == 200 + payload = response.json() + assert payload["total"] == 1 + assert [(request["requestId"], request["status"]) for request in payload["requests"]] == [ + ("req_cancelled_error_control", "error") + ] + + @pytest.mark.asyncio async def test_request_logs_status_ok_filters_success(async_client, db_setup): now = utcnow() diff --git a/tests/integration/test_request_usage_rollup_parity.py b/tests/integration/test_request_usage_rollup_parity.py index 563736887f..42e5dd75f4 100644 --- a/tests/integration/test_request_usage_rollup_parity.py +++ b/tests/integration/test_request_usage_rollup_parity.py @@ -35,7 +35,6 @@ ) from app.db.session import SessionLocal from app.modules.accounts.repository import AccountsRepository -from app.modules.accounts.usage_rollup import FOLD_LAG from app.modules.accounts.usage_time_rollup import ( floor_to_hour, run_conversation_fold_pass, @@ -50,10 +49,24 @@ _EPOCH = datetime(1970, 1, 1) +# The 10-day corpus geometry below (TARGET_W at BASE + 9d, prune floor at +# BASE + 8d, unaligned windows and boundary rows placed between them) was +# authored against a 24h fold lag. The parity semantics under test are +# lag-independent, so the lag is pinned here to keep the corpus exercising +# every boundary it was designed around; the production FOLD_LAG's own +# tail/absorption behavior is covered in test_account_usage_rollup.py. +CORPUS_FOLD_LAG = timedelta(hours=24) + + +@pytest.fixture(autouse=True) +def _pin_corpus_fold_lag(monkeypatch): + monkeypatch.setattr("app.modules.accounts.usage_time_rollup.FOLD_LAG", CORPUS_FOLD_LAG) + + # Fixed 10-day corpus timeline (all naive UTC, matching requested_at). BASE = datetime(2025, 7, 1) NOW = BASE + timedelta(days=10, minutes=37) -TARGET_W = floor_to_hour(NOW - FOLD_LAG) # BASE + 9d +TARGET_W = floor_to_hour(NOW - CORPUS_FOLD_LAG) # BASE + 9d MID_W = BASE + timedelta(days=5, hours=3) # whole hour mid-history SINCE_ALIGNED = BASE + timedelta(days=2) @@ -140,9 +153,9 @@ def _corpus() -> list[RequestLog]: for offset in (timedelta(hours=30), timedelta(days=5, hours=1), timedelta(days=9, hours=20)): rows.append(_log(BASE + offset, request_id_suffix="w", request_kind="warmup", cost_usd=0.002)) rows.append(_log(BASE + offset + timedelta(minutes=20), request_id_suffix="lw", request_kind="limit_warmup")) - # Cancelled rows on both sides of every candidate watermark: the listing - # count's default status split (success+error) must exclude them from - # the folded sum and the raw tail alike. + # Cancelled rows on both sides of every candidate watermark: the default + # listing must include them as a distinct status in both the folded sum + # and the raw tail. for offset in (timedelta(days=1, hours=5), timedelta(days=9, hours=23)): rows.append( _log( @@ -330,9 +343,18 @@ async def _total(**kwargs) -> int: return { "default": await _total(), - "success_only": await _total(include_error_other=False), - "error_only": await _total(include_success=False), - "no_status_filter": await _total(include_success=False, include_error_other=False), + "success_only": await _total(include_cancelled=False, include_error_other=False), + "cancelled_only": await _total( + include_success=False, + include_cancelled=True, + include_error_other=False, + ), + "error_only": await _total(include_success=False, include_cancelled=False), + "no_status_filter": await _total( + include_success=False, + include_cancelled=False, + include_error_other=False, + ), "windowed": await _total(since=lead_since, until=UNTIL_UNALIGNED), "folded_only": await _total(since=FOLDED_ONLY_WINDOW[0], until=FOLDED_ONLY_WINDOW[1]), "tail_only": await _total(since=TAIL_ONLY_WINDOW[0], until=TAIL_ONLY_WINDOW[1]), @@ -453,10 +475,10 @@ async def test_switched_readers_match_legacy_across_watermark_states(db_setup): # Watermark state 2 — mid-history whole hour. The hourly and conversation # folds advance separately (mixed-watermark states in between must hold # parity too: each satellite degrades on its own watermark). - await run_hourly_fold_pass(now=MID_W + FOLD_LAG) + await run_hourly_fold_pass(now=MID_W + CORPUS_FOLD_LAG) assert await _watermark() == MID_W _assert_snapshots_equal(await _snapshot(), reference) - await run_conversation_fold_pass(now=MID_W + FOLD_LAG) + await run_conversation_fold_pass(now=MID_W + CORPUS_FOLD_LAG) assert await _conversation_watermark() == MID_W _assert_snapshots_equal(await _snapshot(), reference) @@ -477,8 +499,8 @@ async def test_reader_is_consistent_under_concurrent_fold_commit(db_setup, monke came from, and folding never deletes raw rows.""" await _seed_corpus() reference = await _snapshot() - await run_hourly_fold_pass(now=MID_W + FOLD_LAG) - await run_conversation_fold_pass(now=MID_W + FOLD_LAG) + await run_hourly_fold_pass(now=MID_W + CORPUS_FOLD_LAG) + await run_conversation_fold_pass(now=MID_W + CORPUS_FOLD_LAG) real_read_hourly_window = request_logs_repository_module.read_hourly_window fold_injections = {"count": 0} @@ -559,7 +581,7 @@ async def test_escape_hatch_reset_degrades_to_legacy_then_rebackfills(db_setup): @pytest.mark.asyncio async def test_statistics_survive_retention_pruning_folded_raw(db_setup, monkeypatch): """The headline guarantee: after raw rows below the retention gate - (watermark - FOLD_LAG) are physically deleted, every rollup-served + (watermark - fold lag) are physically deleted, every rollup-served statistic is unchanged — INCLUDING the distinct-conversation metrics, which the conversation presence satellite now serves for folded history (they used to be raw-bound and shrink here). earliest_activity_at falls @@ -575,7 +597,7 @@ async def test_statistics_survive_retention_pruning_folded_raw(db_setup, monkeyp lead_ceil = floor_to_hour(SINCE_UNALIGNED) + timedelta(hours=1) leadless = await _snapshot(lead_since=lead_ceil) - prune_cutoff = TARGET_W - FOLD_LAG + prune_cutoff = TARGET_W - CORPUS_FOLD_LAG async with SessionLocal() as session: await session.execute(delete(RequestLog).where(RequestLog.requested_at < prune_cutoff)) await session.commit() @@ -684,7 +706,7 @@ async def test_dashboard_overview_json_is_identical_before_and_after_fold(async_ before = await async_client.get("/api/dashboard/overview?timeframe=7d") assert before.status_code == 200 - folded_slices = await run_hourly_fold_pass() # real now: rows are > FOLD_LAG old + folded_slices = await run_hourly_fold_pass() # real now: rows are > fold lag old assert folded_slices > 0 assert await run_conversation_fold_pass() > 0 async with SessionLocal() as session: diff --git a/tests/integration/test_settings_api.py b/tests/integration/test_settings_api.py index 7ef4250346..d4113812ec 100644 --- a/tests/integration/test_settings_api.py +++ b/tests/integration/test_settings_api.py @@ -1228,3 +1228,46 @@ async def test_retention_override_tri_state_echo_capture_and_clear(async_client, settings = await session.get(DashboardSettings, 1) assert settings is not None assert settings.request_log_retention_days is None + + +@pytest.mark.asyncio +async def test_auto_redeem_opt_in_rejected_while_reset_credit_polling_disabled(async_client, monkeypatch): + from types import SimpleNamespace + + disabled = SimpleNamespace(rate_limit_reset_credits_refresh_enabled=False) + monkeypatch.setattr("app.modules.settings.api.get_app_settings", lambda: disabled) + + response = await async_client.get("/api/settings") + assert response.status_code == 200 + payload = response.json() + assert payload["autoRedeemResetCreditsBeforeExpiry"] is False + payload["autoRedeemResetCreditsBeforeExpiry"] = True + + response = await async_client.put("/api/settings", json=payload) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "reset_credit_polling_disabled" + + +@pytest.mark.asyncio +async def test_full_put_with_persisted_auto_redeem_allowed_while_polling_disabled(async_client, monkeypatch): + from types import SimpleNamespace + + async with SessionLocal() as session: + await session.execute( + text("UPDATE dashboard_settings SET auto_redeem_reset_credits_before_expiry = 1 WHERE id = 1") + ) + await session.commit() + + disabled = SimpleNamespace(rate_limit_reset_credits_refresh_enabled=False) + monkeypatch.setattr("app.modules.settings.api.get_app_settings", lambda: disabled) + + response = await async_client.get("/api/settings") + assert response.status_code == 200 + payload = response.json() + assert payload["autoRedeemResetCreditsBeforeExpiry"] is True + + response = await async_client.put("/api/settings", json=payload) + + assert response.status_code == 200 + assert response.json()["autoRedeemResetCreditsBeforeExpiry"] is True diff --git a/tests/integration/test_usage_refresh_scheduler_scope.py b/tests/integration/test_usage_refresh_scheduler_scope.py index f657b0cd3a..8b9ae51a4a 100644 --- a/tests/integration/test_usage_refresh_scheduler_scope.py +++ b/tests/integration/test_usage_refresh_scheduler_scope.py @@ -2,9 +2,11 @@ import time from collections.abc import Awaitable, Callable, Collection +from datetime import datetime, timezone from typing import cast import pytest +from sqlalchemy.orm.exc import DetachedInstanceError from app.core.crypto import TokenEncryptor from app.core.usage import refresh_scheduler as refresh_scheduler_module @@ -44,6 +46,24 @@ def _account( ) +@pytest.mark.asyncio +async def test_background_accounts_repo_keeps_loaded_account_usable_after_context(db_setup) -> None: + del db_setup + account = _account("acc_detached", status=AccountStatus.RATE_LIMITED) + async with SessionLocal() as session: + await AccountsRepository(session).upsert(account) + + async with refresh_scheduler_module._background_accounts_repo() as accounts_repo: + loaded = await accounts_repo.get_by_id_fresh(account.id) + + assert loaded is not None + try: + actual = (loaded.status, loaded.limit_warmup_enabled) + except DetachedInstanceError: + pytest.fail("background account expired when its read transaction closed") + assert actual == (AccountStatus.RATE_LIMITED, True) + + @pytest.mark.asyncio async def test_scheduler_repository_path_scopes_selected_account_history_and_followups( db_setup, @@ -150,6 +170,7 @@ async def run_after_usage_refresh(self, **kwargs: object) -> None: selected.id, unrelated.id, } + assert warmup_calls[0]["previous_plan_types"] == {selected.id: "plus"} for snapshot_name in ("before_primary", "before_secondary", "after_primary", "after_secondary"): assert set(cast("dict[str, UsageHistory]", warmup_calls[0][snapshot_name])) <= {selected.id} @@ -163,22 +184,42 @@ async def run_after_usage_refresh(self, **kwargs: object) -> None: @pytest.mark.asyncio -async def test_scheduler_free_monthly_reset_creates_monthly_warmup_attempt( +async def test_scheduler_recovers_rate_limited_free_before_monthly_reset_warmup( db_setup, monkeypatch: pytest.MonkeyPatch, ) -> None: del db_setup - before_reset_at = int(time.time()) + 3600 - after_reset_at = before_reset_at + 43_200 * 60 - account = _account("acc_free", status=AccountStatus.ACTIVE) + now = int(time.time()) + primary_reset_at = now + 8 * 24 * 60 * 60 + before_reset_at = primary_reset_at + blocked_at = now - 5 * 24 * 60 * 60 + before_recorded_at = datetime.fromtimestamp(now - 120, timezone.utc).replace(tzinfo=None) + after_recorded_epoch = now - 60 + after_recorded_at = datetime.fromtimestamp(after_recorded_epoch, timezone.utc).replace(tzinfo=None) + after_reset_at = after_recorded_epoch + 43_200 * 60 + account = _account( + "acc_free", + status=AccountStatus.RATE_LIMITED, + reset_at=primary_reset_at, + blocked_at=blocked_at, + ) account.plan_type = "free" async with SessionLocal() as session: await AccountsRepository(session).upsert(account) + await UsageRepository(session).add_entry( + account.id, + 100.0, + window="primary", + recorded_at=datetime.fromtimestamp(blocked_at, timezone.utc).replace(tzinfo=None), + reset_at=primary_reset_at, + window_minutes=300, + ) await UsageRepository(session).add_entry( account.id, 100.0, window="monthly", + recorded_at=before_recorded_at, reset_at=before_reset_at, window_minutes=43_200, ) @@ -199,12 +240,13 @@ async def refresh_accounts( latest_usage: dict[str, UsageHistory], ) -> bool: assert [candidate.id for candidate in accounts] == [account.id] - assert latest_usage == {} + assert set(latest_usage) == {account.id} async with SessionLocal() as session: await UsageRepository(session).add_entry( account.id, 0.0, window="monthly", + recorded_at=after_recorded_at, reset_at=after_reset_at, window_minutes=43_200, ) @@ -214,7 +256,21 @@ class _Sender: def __init__(self) -> None: self.calls: list[tuple[str, str]] = [] - async def send(self, target: Account, *, model: str, prompt: str) -> LimitWarmupSendResult: + async def send( + self, + target: Account, + *, + model: str, + prompt: str, + ) -> LimitWarmupSendResult: + async with SessionLocal() as session: + persisted = await AccountsRepository(session).get_by_id(target.id) + assert persisted is not None + assert (persisted.status, persisted.reset_at, persisted.blocked_at) == ( + AccountStatus.ACTIVE, + None, + None, + ) self.calls.append((target.id, model)) return LimitWarmupSendResult( request_id="warmup-monthly", @@ -232,5 +288,250 @@ async def send(self, target: Account, *, model: str, prompt: str) -> LimitWarmup assert await scheduler._refresh_once() == 60.0 assert sender.calls == [(account.id, "gpt-5.1-codex-mini")] async with SessionLocal() as session: + persisted_account = await AccountsRepository(session).get_by_id(account.id) attempt = (await LimitWarmupRepository(session).latest_by_account([account.id]))[account.id] + assert persisted_account is not None + assert (persisted_account.status, persisted_account.reset_at, persisted_account.blocked_at) == ( + AccountStatus.ACTIVE, + None, + None, + ) assert (attempt.window, attempt.reset_at, attempt.status) == ("monthly", after_reset_at, "succeeded") + + +@pytest.mark.asyncio +async def test_scheduler_warms_confirmed_paid_to_free_plan_transition( + db_setup, + monkeypatch: pytest.MonkeyPatch, +) -> None: + del db_setup + account = _account("acc_paid_to_free", status=AccountStatus.ACTIVE) + prior_reset_at = int(time.time()) + 7 * 24 * 60 * 60 + monthly_reset_at = int(time.time()) + 30 * 24 * 60 * 60 + + async with SessionLocal() as session: + await AccountsRepository(session).upsert(account) + await UsageRepository(session).add_entry( + account.id, + 100.0, + window="secondary", + recorded_at=utcnow(), + reset_at=prior_reset_at, + window_minutes=10_080, + ) + await SettingsRepository(session).update( + limit_warmup_enabled=True, + limit_warmup_windows="secondary", + limit_warmup_model="gpt-5.1-codex-mini", + ) + + class _Leader: + async def run_if_leader(self, fn: Callable[[], Awaitable[object]]) -> object: + return await fn() + + class _Updater: + async def refresh_accounts( + self, + accounts: list[Account], + latest_usage: dict[str, UsageHistory], + ) -> bool: + assert [candidate.id for candidate in accounts] == [account.id] + assert accounts[0].plan_type == "plus" + accounts[0].plan_type = "free" + async with SessionLocal() as session: + persisted = await AccountsRepository(session).get_by_id(account.id) + assert persisted is not None + persisted.plan_type = "free" + await session.commit() + await UsageRepository(session).add_entry( + account.id, + 0.0, + window="monthly", + recorded_at=utcnow(), + reset_at=monthly_reset_at, + window_minutes=43_200, + ) + return True + + class _Sender: + def __init__(self) -> None: + self.calls: list[tuple[str, str]] = [] + + async def send( + self, + target: Account, + *, + model: str, + prompt: str, + ) -> LimitWarmupSendResult: + self.calls.append((target.id, model)) + return LimitWarmupSendResult(request_id="warmup-plan-transition", success=True, latency_ms=12) + + sender = _Sender() + monkeypatch.setattr(refresh_scheduler_module, "_get_leader_election", lambda: _Leader()) + monkeypatch.setattr(refresh_scheduler_module, "build_background_usage_updater", lambda: _Updater()) + monkeypatch.setattr(refresh_scheduler_module, "StreamingLimitWarmupSender", lambda *_args, **_kwargs: sender) + + scheduler = refresh_scheduler_module.UsageRefreshScheduler(interval_seconds=60, enabled=True) + + assert await scheduler._refresh_once() == 60.0 + assert sender.calls == [(account.id, "gpt-5.1-codex-mini")] + async with SessionLocal() as session: + persisted_account = await AccountsRepository(session).get_by_id(account.id) + attempt = (await LimitWarmupRepository(session).latest_by_account([account.id]))[account.id] + assert persisted_account is not None + assert persisted_account.plan_type == "free" + assert (attempt.window, attempt.reset_at, attempt.status) == ("monthly", monthly_reset_at, "succeeded") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("existing_attempt", [False, True], ids=["warmup-new", "warmup-deduped"]) +@pytest.mark.parametrize( + "current_pair_confirms_unanchored", + [False, True], + ids=["persisted-only", "unanchored-current"], +) +async def test_scheduler_restart_uses_anchored_persisted_evidence( + db_setup, + monkeypatch: pytest.MonkeyPatch, + existing_attempt: bool, + current_pair_confirms_unanchored: bool, +) -> None: + del db_setup + now = int(time.time()) + blocked_at = now - 5 * 24 * 60 * 60 + legacy_reset_at = now + 8 * 24 * 60 * 60 + transition_recorded_at = now - 60 * 60 + transition_reset_at = transition_recorded_at + 43_200 * 60 + current_before_recorded_at = now - 120 + current_before_reset_at = now - 90 + latest_recorded_at = now - 60 + latest_reset_at = latest_recorded_at + 43_200 * 60 + expected_attempt_reset_at = latest_reset_at if current_pair_confirms_unanchored else transition_reset_at + account = _account( + "acc_free_restart", + status=AccountStatus.RATE_LIMITED, + reset_at=legacy_reset_at, + blocked_at=blocked_at, + ) + account.plan_type = "free" + + async with SessionLocal() as session: + await AccountsRepository(session).upsert(account) + usage_repo = UsageRepository(session) + await usage_repo.add_entry( + account.id, + 100.0, + window="monthly", + recorded_at=datetime.fromtimestamp(blocked_at, timezone.utc).replace(tzinfo=None), + reset_at=legacy_reset_at, + window_minutes=43_200, + ) + await usage_repo.add_entry( + account.id, + 100.0, + window="monthly", + recorded_at=datetime.fromtimestamp(blocked_at + 60, timezone.utc).replace(tzinfo=None), + reset_at=legacy_reset_at, + window_minutes=43_200, + ) + sliding_sample_count = 240 + sliding_span = transition_recorded_at - blocked_at - 120 + for index in range(1, sliding_sample_count + 1): + recorded_at = blocked_at + 60 + (sliding_span * index // sliding_sample_count) + await usage_repo.add_entry( + account.id, + 100.0, + window="monthly", + recorded_at=datetime.fromtimestamp(recorded_at, timezone.utc).replace(tzinfo=None), + reset_at=legacy_reset_at + index * 60, + window_minutes=43_200, + ) + await usage_repo.add_entry( + account.id, + 0.0, + window="monthly", + recorded_at=datetime.fromtimestamp(transition_recorded_at, timezone.utc).replace(tzinfo=None), + reset_at=transition_reset_at, + window_minutes=43_200, + ) + if current_pair_confirms_unanchored: + await usage_repo.add_entry( + account.id, + 40.0, + window="monthly", + recorded_at=datetime.fromtimestamp(current_before_recorded_at, timezone.utc).replace(tzinfo=None), + reset_at=current_before_reset_at, + window_minutes=43_200, + ) + await SettingsRepository(session).update( + limit_warmup_enabled=True, + limit_warmup_windows="secondary", + limit_warmup_model="gpt-5.1-codex-mini", + ) + if existing_attempt: + attempt = await LimitWarmupRepository(session).try_create_attempt( + account_id=account.id, + window="monthly", + reset_at=expected_attempt_reset_at, + model="gpt-5.1-codex-mini", + attempted_at=utcnow(), + status="succeeded", + reset_at_tolerance_seconds=5, + ) + assert attempt is not None + + class _Leader: + async def run_if_leader(self, fn: Callable[[], Awaitable[object]]) -> object: + return await fn() + + class _Updater: + async def refresh_accounts( + self, + accounts: list[Account], + latest_usage: dict[str, UsageHistory], + ) -> bool: + assert [candidate.id for candidate in accounts] == [account.id] + async with SessionLocal() as session: + await UsageRepository(session).add_entry( + account.id, + 0.0, + window="monthly", + recorded_at=datetime.fromtimestamp(latest_recorded_at, timezone.utc).replace(tzinfo=None), + reset_at=latest_reset_at, + window_minutes=43_200, + ) + return True + + class _Sender: + def __init__(self) -> None: + self.calls: list[str] = [] + + async def send(self, target: Account, *, model: str, prompt: str) -> LimitWarmupSendResult: + del model, prompt + async with SessionLocal() as session: + persisted = await AccountsRepository(session).get_by_id(target.id) + assert persisted is not None + assert persisted.status == AccountStatus.ACTIVE + self.calls.append(target.id) + return LimitWarmupSendResult(request_id="warmup-restart", success=True, latency_ms=12) + + sender = _Sender() + monkeypatch.setattr(refresh_scheduler_module, "_get_leader_election", lambda: _Leader()) + monkeypatch.setattr(refresh_scheduler_module, "build_background_usage_updater", lambda: _Updater()) + monkeypatch.setattr(refresh_scheduler_module, "StreamingLimitWarmupSender", lambda *_args, **_kwargs: sender) + + scheduler = refresh_scheduler_module.UsageRefreshScheduler(interval_seconds=60, enabled=True) + + assert await scheduler._refresh_once() == 60.0 + assert sender.calls == ([] if existing_attempt else [account.id]) + async with SessionLocal() as session: + persisted = await AccountsRepository(session).get_by_id(account.id) + attempt = (await LimitWarmupRepository(session).latest_by_account([account.id]))[account.id] + assert persisted is not None + assert (persisted.status, persisted.reset_at, persisted.blocked_at) == (AccountStatus.ACTIVE, None, None) + assert (attempt.window, attempt.reset_at, attempt.status) == ( + "monthly", + expected_attempt_reset_at, + "succeeded", + ) diff --git a/tests/integration/test_usage_repository.py b/tests/integration/test_usage_repository.py index a0b2be0db9..6ac9455378 100644 --- a/tests/integration/test_usage_repository.py +++ b/tests/integration/test_usage_repository.py @@ -1298,6 +1298,245 @@ async def test_bulk_history_since_per_account_cutoffs_parity(db_setup): assert [snapshot.used_percent for snapshot in trimmed] == [20.0] +@pytest.mark.asyncio +async def test_bulk_history_since_per_account_row_cap_keeps_newest_rows(db_setup): + """The PostgreSQL row cap keeps each account's newest in-cutoff rows in + oldest-first order; under-cap accounts are unaffected and SQLite ignores + the cap entirely (snapshot-cache path, like ``cutoffs``).""" + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + repo = UsageRepository(session) + await accounts_repo.upsert(_make_account("acc-dense")) + await accounts_repo.upsert(_make_account("acc-sparse")) + + for offset in range(8): + await repo.add_entry( + "acc-dense", + 10.0 + offset, + window="secondary", + recorded_at=now - timedelta(minutes=8 - offset), + ) + await repo.add_entry("acc-sparse", 90.0, window="secondary", recorded_at=now - timedelta(hours=2)) + await repo.add_entry("acc-sparse", 95.0, window="secondary", recorded_at=now - timedelta(hours=1)) + + since = now - timedelta(days=7) + capped = await repo.bulk_history_since( + ["acc-dense", "acc-sparse"], + "secondary", + since, + per_account_row_cap=3, + ) + uncapped = await repo.bulk_history_since(["acc-dense", "acc-sparse"], "secondary", since) + + dialect = "postgresql" if str(engine.url).startswith("postgresql") else "sqlite" + if dialect == "postgresql": + # Newest three rows, still oldest-first. + assert [snapshot.used_percent for snapshot in capped["acc-dense"]] == [15.0, 16.0, 17.0] + assert capped["acc-dense"] == uncapped["acc-dense"][-3:] + else: + # SQLite serves the shared-floor snapshot cache; the cap is ignored. + assert [snapshot.used_percent for snapshot in capped["acc-dense"]] == [ + snapshot.used_percent for snapshot in uncapped["acc-dense"] + ] + # Under-cap accounts return their full in-cutoff slice on every backend. + assert [snapshot.used_percent for snapshot in capped["acc-sparse"]] == [90.0, 95.0] + + +@pytest.mark.asyncio +async def test_bulk_history_since_row_cap_respects_per_account_cutoffs_postgresql(db_setup): + """The cap composes with per-account cutoffs: the cutoff bounds the + lookback first, then the cap keeps the newest rows inside it.""" + now = utcnow() + async with SessionLocal() as session: + if _dialect_name(session) != "postgresql": + pytest.skip("PostgreSQL-only row-cap test") + + accounts_repo = AccountsRepository(session) + repo = UsageRepository(session) + await accounts_repo.upsert(_make_account("acc-short")) + await accounts_repo.upsert(_make_account("acc-wide")) + + await repo.add_entry("acc-short", 10.0, window="primary", recorded_at=now - timedelta(hours=20)) + await repo.add_entry("acc-short", 20.0, window="primary", recorded_at=now - timedelta(hours=1)) + for offset in range(4): + await repo.add_entry( + "acc-wide", + 30.0 + offset, + window="primary", + recorded_at=now - timedelta(hours=20 - offset), + ) + + grouped = await repo.bulk_history_since( + ["acc-short", "acc-wide"], + "primary", + now - timedelta(days=7), + cutoffs={ + "acc-short": now - timedelta(hours=5), + "acc-wide": now - timedelta(days=7), + }, + per_account_row_cap=3, + ) + + # acc-short's 20h-old row falls outside its cutoff even though the cap + # alone would have kept it. + assert [snapshot.used_percent for snapshot in grouped["acc-short"]] == [20.0] + # acc-wide keeps only the newest three of its four in-cutoff rows. + assert [snapshot.used_percent for snapshot in grouped["acc-wide"]] == [31.0, 32.0, 33.0] + + +@pytest.mark.asyncio +async def test_bulk_history_since_row_cap_exempts_uncapped_recent_floor_postgresql(db_setup): + """Rows at or after ``uncapped_recent_floor`` bypass the row cap. + + Live ingestion writes per proxied request whenever the usage fingerprint + moves, so a burst can put more rows inside the pace-smoothing window than + any fixed cap; the smoothing mean weighs those samples equally, so they + must all come back. The cap still bounds the older remainder. + """ + now = utcnow() + async with SessionLocal() as session: + if _dialect_name(session) != "postgresql": + pytest.skip("PostgreSQL-only row-cap test") + + accounts_repo = AccountsRepository(session) + repo = UsageRepository(session) + await accounts_repo.upsert(_make_account("acc-burst")) + + # Six rows inside the floor window (a burst denser than the cap) and + # four older rows between the cutoff and the floor. + for offset in range(6): + await repo.add_entry( + "acc-burst", + 50.0 + offset, + window="secondary", + recorded_at=now - timedelta(minutes=30 - offset), + ) + for offset in range(4): + await repo.add_entry( + "acc-burst", + 10.0 + offset, + window="secondary", + recorded_at=now - timedelta(hours=10 - offset), + ) + + grouped = await repo.bulk_history_since( + ["acc-burst"], + "secondary", + now - timedelta(days=7), + per_account_row_cap=3, + uncapped_recent_floor=now - timedelta(minutes=60), + ) + + # All six in-floor rows survive despite cap=3; the older tail keeps only + # its newest three rows; the slice stays oldest-first. + assert [snapshot.used_percent for snapshot in grouped["acc-burst"]] == [ + 11.0, + 12.0, + 13.0, + 50.0, + 51.0, + 52.0, + 53.0, + 54.0, + 55.0, + ] + + +@pytest.mark.asyncio +async def test_bulk_history_since_capped_query_plan_is_index_only_postgresql(db_setup): + """The capped lateral probes must stay heap-free on the covering indexes. + + Each per-account probe descends the covering index backward and stops at + the cap or cutoff; a plain Index Scan here would mean the probe shape + lost the covering payload and fetches the heap per row. + """ + async with SessionLocal() as session: + if _dialect_name(session) != "postgresql": + pytest.skip("PostgreSQL-only query plan test") + + await _seed_bulk_history_plan_fixture(session) + + await session.execute(text("SET enable_seqscan = off")) + await session.execute(text("SET enable_bitmapscan = off")) + plan = ( + await session.execute( + text( + """ + EXPLAIN (FORMAT JSON) + SELECT recent.* + FROM (VALUES ('acc1', now() - interval '5 hours'), + ('acc2', now() - interval '7 days')) + AS account_cutoffs (account_id, cutoff) + JOIN LATERAL ( + SELECT id, account_id, used_percent, recorded_at, reset_at, window_minutes + FROM usage_history + WHERE account_id = account_cutoffs.account_id + AND recorded_at >= account_cutoffs.cutoff + AND "window" = 'secondary' + ORDER BY recorded_at DESC, id DESC + LIMIT 100 + ) AS recent ON true + """ + ) + ) + ).scalar_one() + + plan_json = json.dumps(plan) + assert "Index Only Scan" in plan_json + assert "idx_usage_window_raw_account_time_covering" in plan_json + assert "Seq Scan on usage_history" not in plan_json + + +@pytest.mark.asyncio +async def test_bulk_history_since_capped_floor_query_plan_is_index_only_postgresql(db_setup): + """The floor-exempt probe shape (uncapped recent branch UNION ALL capped + older branch) must keep both branches heap-free on the covering index.""" + async with SessionLocal() as session: + if _dialect_name(session) != "postgresql": + pytest.skip("PostgreSQL-only query plan test") + + await _seed_bulk_history_plan_fixture(session) + + await session.execute(text("SET enable_seqscan = off")) + await session.execute(text("SET enable_bitmapscan = off")) + plan = ( + await session.execute( + text( + """ + EXPLAIN (FORMAT JSON) + SELECT recent.* + FROM (VALUES ('acc1', now() - interval '7 days', now() - interval '4 hours'), + ('acc2', now() - interval '7 days', now() - interval '4 hours')) + AS account_cutoffs (account_id, cutoff, uncapped_floor) + JOIN LATERAL ( + (SELECT id, account_id, used_percent, recorded_at, reset_at, window_minutes + FROM usage_history + WHERE account_id = account_cutoffs.account_id + AND recorded_at >= account_cutoffs.uncapped_floor + AND "window" = 'secondary') + UNION ALL + (SELECT id, account_id, used_percent, recorded_at, reset_at, window_minutes + FROM usage_history + WHERE account_id = account_cutoffs.account_id + AND recorded_at >= account_cutoffs.cutoff + AND recorded_at < account_cutoffs.uncapped_floor + AND "window" = 'secondary' + ORDER BY recorded_at DESC, id DESC + LIMIT 100) + ) AS recent ON true + """ + ) + ) + ).scalar_one() + + plan_json = json.dumps(plan) + assert "Index Only Scan" in plan_json + assert "idx_usage_window_raw_account_time_covering" in plan_json + assert "Seq Scan on usage_history" not in plan_json + assert "Index Scan using" not in plan_json + + def _legacy_additional_entry( account_id: str, *, diff --git a/tests/integration/test_v1_models.py b/tests/integration/test_v1_models.py index dee695dd43..917b77872f 100644 --- a/tests/integration/test_v1_models.py +++ b/tests/integration/test_v1_models.py @@ -261,6 +261,18 @@ async def test_v1_models_uses_bootstrap_models_when_registry_not_populated(async assert ids == BOOTSTRAP_MODEL_SLUGS assert "gpt-5.5-pro" not in ids + # The raised GPT-5.6 ceiling is a Codex-native field. /v1 input budgets + # stay on ``context_window`` so OpenAI-compatible clients keep packing to + # the 272k default instead of the 872k ``max_context_window`` ceiling. + entries = {item["id"]: item for item in payload["data"]} + for slug in ("gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"): + entry = entries[slug] + assert entry["metadata"]["context_window"] == 272_000 + assert entry["metadata"]["input_context_window"] == 272_000 + assert entry["capabilities"]["context_length"] == 272_000 + assert entry["context_length"] == 272_000 + assert entry["contextLength"] == 272_000 + @pytest.mark.asyncio async def test_backend_codex_models_uses_bootstrap_upstream_metadata(async_client): @@ -281,7 +293,7 @@ async def test_backend_codex_models_uses_bootstrap_upstream_metadata(async_clien sol = entries["gpt-5.6-sol"] assert sol["display_name"] == "GPT-5.6-Sol" - assert sol["context_window"] == 372_000 + assert sol["context_window"] == 272_000 assert sol["default_reasoning_level"] == "low" assert {level["effort"] for level in sol["supported_reasoning_levels"]} == { "low", @@ -314,10 +326,14 @@ async def test_backend_codex_models_uses_bootstrap_upstream_metadata(async_clien "max", } - # Upstream-exact GPT-5.6 metadata as served on the Codex catalog wire - # (codex-rs/models-manager/models.json at rust-v0.144.1). + # Reproducible upstream catalog evidence: + # codex-rs/models-manager/models.json at rust-v0.145.0, except + # ``max_context_window``: raised to 872000 in openai/codex commit + # 2eee483e49f88b868f67364134a658b3298e6c14 (openai/codex#39102), which no + # rust-v* release tag carries yet. for gpt56 in (sol, terra, luna): assert gpt56["minimal_client_version"] == "0.144.0" + assert gpt56["context_window"] == 272_000 assert gpt56["tool_mode"] == "code_mode_only" assert gpt56["use_responses_lite"] is True assert gpt56["apply_patch_tool_type"] == "freeform" @@ -327,7 +343,8 @@ async def test_backend_codex_models_uses_bootstrap_upstream_metadata(async_clien assert gpt56["reasoning_summary_format"] == "experimental" assert gpt56["comp_hash"] == "3000" assert gpt56["experimental_supported_tools"] == [] - assert gpt56["max_context_window"] == 372_000 + assert gpt56["max_context_window"] == 872_000 + assert gpt56["max_context_window"] > gpt56["context_window"] assert gpt56["service_tiers"] == [ {"id": "priority", "name": "Fast", "description": "1.5x speed, increased usage"} ] @@ -797,6 +814,37 @@ async def test_backend_codex_models_unions_service_tiers_across_accounts(async_c assert "fast" in (model.get("additional_speed_tiers") or []) +@pytest.mark.asyncio +async def test_backend_codex_models_filters_unknown_reasoning_efforts(async_client): + registry = get_model_registry() + model = replace( + _make_upstream_model( + "source-gpt", + raw={ + "shell_type": "shell_command", + "visibility": "list", + }, + ), + supported_reasoning_levels=tuple( + ReasoningLevel(effort=effort, description=effort) + for effort in ("none", "high", "provider-specific", "ultra") + ), + default_reasoning_level="provider-specific", + ) + await registry.update({"plus": [model], "pro": [model]}) + + resp = await async_client.get("/backend-api/codex/models") + + assert resp.status_code == 200 + entry = next(m for m in resp.json()["models"] if m["slug"] == "source-gpt") + assert [level["effort"] for level in entry["supported_reasoning_levels"]] == [ + "none", + "high", + "ultra", + ] + assert entry["default_reasoning_level"] is None + + @pytest.mark.asyncio async def test_backend_codex_models_does_not_reunion_stale_global_service_tiers(async_client): registry = get_model_registry() @@ -1363,10 +1411,74 @@ async def test_model_context_window_override(async_client, monkeypatch): v1_entry = next(m for m in resp_v1.json()["data"] if m["id"] == "gpt-5.4") metadata = v1_entry["metadata"] assert metadata["context_window"] == 515000 - assert metadata["input_context_window"] == 272000 - assert v1_entry["capabilities"]["context_length"] == 272000 - assert v1_entry["contextLength"] == 272000 - assert v1_entry["context_length"] == 272000 + # An explicit operator override is the reported input budget too: generic + # OpenAI-compatible clients read `context_length`/`contextLength` and would + # otherwise cap themselves at the un-overridden upstream window. + assert metadata["input_context_window"] == 515000 + assert v1_entry["capabilities"]["context_length"] == 515000 + assert v1_entry["contextLength"] == 515000 + assert v1_entry["context_length"] == 515000 + + +@pytest.mark.asyncio +async def test_model_context_window_override_clamped_to_max_context_window(async_client, monkeypatch): + registry = get_model_registry() + models = [_make_upstream_model("gpt-5.4", raw=_raw_with_max_context_window(872_000))] + await registry.update({"pro": models}) + + from app.core.config.settings import get_settings + from app.modules.proxy import api as proxy_api_module + + patched = get_settings().model_copy(update={"model_context_window_overrides": {"gpt-5.4": 1_000_000}}) + monkeypatch.setattr(proxy_api_module, "get_settings", lambda: patched) + + resp_v1 = await async_client.get("/v1/models") + assert resp_v1.status_code == 200 + v1_entry = next(m for m in resp_v1.json()["data"] if m["id"] == "gpt-5.4") + + # The reported input budget never exceeds the upstream-declared ceiling, and + # `metadata.context_window` reports the same clamped value: the override is + # resolved once, so the clamp cannot reintroduce a dual-budget split. + assert v1_entry["metadata"]["context_window"] == 872_000 + assert v1_entry["metadata"]["input_context_window"] == 872_000 + assert v1_entry["capabilities"]["context_length"] == 872_000 + assert v1_entry["contextLength"] == 872_000 + assert v1_entry["context_length"] == 872_000 + + # The Codex-native catalog shares the same single resolution. + resp_codex = await async_client.get("/backend-api/codex/models") + assert resp_codex.status_code == 200 + native_entry = next(m for m in resp_codex.json()["models"] if m["slug"] == "gpt-5.4") + assert native_entry["context_window"] == 872_000 + assert native_entry["max_context_window"] == 872_000 + + +@pytest.mark.asyncio +async def test_model_context_window_override_applies_to_codex_models_data_alias(async_client, monkeypatch): + registry = get_model_registry() + models = [_make_upstream_model("gpt-5.4")] + await registry.update({"pro": models}) + + from app.core.config.settings import get_settings + from app.modules.proxy import api as proxy_api_module + + patched = get_settings().model_copy(update={"model_context_window_overrides": {"gpt-5.4": 515_000}}) + monkeypatch.setattr(proxy_api_module, "get_settings", lambda: patched) + + # The OpenAI-compatible `data` alias on /backend-api/codex/models is built + # from the same list-item shape as /v1/models, so the override reaches its + # context_length-family fields too (pinned: both views advertise one budget). + resp = await async_client.get("/backend-api/codex/models") + assert resp.status_code == 200 + payload = resp.json() + native_entry = next(m for m in payload["models"] if m["slug"] == "gpt-5.4") + assert native_entry["context_window"] == 515_000 + alias_item = next(m for m in payload["data"] if m["id"] == "gpt-5.4") + assert alias_item["metadata"]["context_window"] == 515_000 + assert alias_item["metadata"]["input_context_window"] == 515_000 + assert alias_item["capabilities"]["context_length"] == 515_000 + assert alias_item["contextLength"] == 515_000 + assert alias_item["context_length"] == 515_000 @pytest.mark.asyncio diff --git a/tests/test_request_logs_options_api.py b/tests/test_request_logs_options_api.py index a307d2dcc0..876092b013 100644 --- a/tests/test_request_logs_options_api.py +++ b/tests/test_request_logs_options_api.py @@ -29,6 +29,31 @@ def _make_account(account_id: str, email: str) -> Account: ) +@pytest.mark.asyncio +async def test_request_logs_options_include_cancelled_status(async_client, db_setup): + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + await accounts_repo.upsert(_make_account("acc_opt_cancelled", "cancelled@example.com")) + await logs_repo.add_log( + account_id="acc_opt_cancelled", + request_id="req_opt_cancelled", + model="gpt-5.1", + input_tokens=1, + output_tokens=0, + latency_ms=10, + status="cancelled", + error_code="client_disconnected", + requested_at=now, + ) + + response = await async_client.get("/api/request-logs/options") + + assert response.status_code == 200 + assert response.json()["statuses"] == ["cancelled"] + + @pytest.mark.asyncio async def test_request_logs_options_returns_distinct_accounts_and_models(async_client, db_setup): now = utcnow() diff --git a/tests/unit/hypothesis_strategies.py b/tests/unit/hypothesis_strategies.py new file mode 100644 index 0000000000..ab843a9a34 --- /dev/null +++ b/tests/unit/hypothesis_strategies.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from hypothesis import strategies as st + +json_scalars = st.one_of( + st.none(), + st.booleans(), + st.integers(min_value=-10_000, max_value=10_000), + st.floats(allow_nan=False, allow_infinity=False, width=32), + st.text(max_size=80), +) + +json_values = st.recursive( + json_scalars, + lambda children: st.one_of( + st.lists(children, max_size=5), + st.dictionaries(st.text(max_size=20), children, max_size=5), + ), + max_leaves=20, +) + +json_directive_types = json_values.filter(lambda value: value is not None and value != "message") + +json_objects = st.dictionaries( + st.text(max_size=20), + json_values, + max_size=8, +) + +json_arrays = st.lists(json_values, max_size=8) diff --git a/tests/unit/test_accounts_repository_locks.py b/tests/unit/test_accounts_repository_locks.py index e4c6238abc..ce61f9ba7a 100644 --- a/tests/unit/test_accounts_repository_locks.py +++ b/tests/unit/test_accounts_repository_locks.py @@ -1,6 +1,7 @@ from __future__ import annotations from contextlib import asynccontextmanager +from typing import Any, cast from unittest.mock import AsyncMock, MagicMock import pytest @@ -8,8 +9,9 @@ import app.modules.accounts.repository as repository_module from app.core.crypto import TokenEncryptor from app.core.utils.time import utcnow +from app.db.account_identity_lock import account_identity_lock_key, lock_postgresql_account_identities from app.db.models import Account, AccountStatus -from app.modules.accounts.repository import AccountsRepository +from app.modules.accounts.repository import AccountIdentityRelockError, AccountsRepository def _stub_account(account_id: str, email: str, chatgpt_id: str | None = None) -> Account: @@ -41,18 +43,40 @@ def _make_postgres_repo(monkeypatch: pytest.MonkeyPatch) -> tuple[AccountsReposi session.execute = AsyncMock() session.commit = AsyncMock() session.refresh = AsyncMock() + session.rollback = AsyncMock() session.add = MagicMock() session.get = AsyncMock(return_value=None) repo = AccountsRepository(session) - recorded: dict[str, list[str]] = {"identity": [], "email": []} + recorded: dict[str, list[str]] = {"upstream": [], "identity": [], "email": [], "order": []} async def fake_identity_lock(key: str) -> None: recorded["identity"].append(key) + recorded["order"].append(f"identity:{key}") async def fake_email_lock(email: str) -> None: recorded["email"].append(email) + recorded["order"].append(f"email:{email}") + + async def fake_upstream_identity_locks(account: Account, *, include_email: bool) -> frozenset[str]: + del include_email + if account.chatgpt_account_id: + recorded["upstream"].append(account.chatgpt_account_id) + recorded["order"].append(f"upstream:{account.chatgpt_account_id}") + return frozenset((account.chatgpt_account_id,)) + return frozenset() + + async def fake_candidates_are_locked( + account: Account, + *, + include_email: bool, + locked_identities: frozenset[str], + ) -> bool: + del account + del include_email + del locked_identities + return True async def fake_merge_by_email_enabled() -> bool: # only used when merge_by_email is None return True @@ -76,9 +100,13 @@ async def fake_next_available_account_id(account_id: str) -> str: monkeypatch.setattr(repo, "_dialect_name", lambda: "postgresql") monkeypatch.setattr(repo, "_acquire_postgresql_identity_lock", fake_identity_lock) monkeypatch.setattr(repo, "_acquire_postgresql_merge_lock", fake_email_lock) + monkeypatch.setattr(repo, "_lock_postgresql_upsert_identity_candidates", fake_upstream_identity_locks) + monkeypatch.setattr(repo, "_postgresql_upsert_identity_candidates_are_locked", fake_candidates_are_locked) monkeypatch.setattr(repo, "_merge_by_email_enabled", fake_merge_by_email_enabled) monkeypatch.setattr(repo, "_account_by_chatgpt_identity", fake_account_by_chatgpt_identity) + monkeypatch.setattr(repo, "_account_by_slot_identity", AsyncMock(return_value=None)) monkeypatch.setattr(repo, "_single_account_by_email", fake_single_account_by_email) + monkeypatch.setattr(repo, "_single_unknown_workspace_account_by_email", fake_single_account_by_email) monkeypatch.setattr(repo, "_next_available_account_id", fake_next_available_account_id) return repo, recorded @@ -90,6 +118,35 @@ def _make_result(value: str | None = "acc") -> MagicMock: return result +@pytest.mark.asyncio +async def test_postgresql_upstream_identity_locks_use_existing_namespace_in_sorted_order() -> None: + session = MagicMock() + session.get_bind.return_value.dialect.name = "postgresql" + session.execute = AsyncMock() + + lock_keys = await lock_postgresql_account_identities(session, ("workspace-z", None, "workspace-a", "workspace-z")) + + expected = tuple(sorted((account_identity_lock_key("workspace-a"), account_identity_lock_key("workspace-z")))) + assert lock_keys == expected + assert session.execute.await_args_list[0].args[1] == {"timeout": "30000ms"} + assert [call.args[1]["lock_key"] for call in session.execute.await_args_list[1:]] == list(expected) + + +@pytest.mark.asyncio +async def test_postgresql_upstream_identity_lock_failure_rolls_back_and_propagates() -> None: + session = MagicMock() + session.get_bind.return_value.dialect.name = "postgresql" + lock_error = RuntimeError("injected lock timeout") + session.execute = AsyncMock(side_effect=[MagicMock(), lock_error]) + session.rollback = AsyncMock() + + with pytest.raises(RuntimeError) as exc_info: + await lock_postgresql_account_identities(session, ("workspace-timeout",)) + + assert exc_info.value is lock_error + session.rollback.assert_awaited_once() + + @pytest.mark.asyncio async def test_account_update_status_uses_sqlite_writer_section(monkeypatch): session = MagicMock() @@ -183,9 +240,10 @@ async def test_upsert_takes_identity_lock_even_when_merge_by_email_enabled(monke await repo.upsert(account, merge_by_email=True, merge_by_chatgpt_identity=True) - assert recorded["identity"] == ["chatgpt:chatgpt_xyz"], ( - "identity lock must be acquired even when merge_by_email is True" + assert recorded["upstream"] == ["chatgpt_xyz"], ( + "upstream identity lock must be acquired even when merge_by_email is True" ) + assert recorded["identity"] == [] assert recorded["email"] == ["a@example.com"], "email lock must still be acquired when merge_by_email is True" @@ -200,7 +258,8 @@ async def test_upsert_takes_identity_lock_when_merge_by_email_disabled(monkeypat await repo.upsert(account, merge_by_email=False, merge_by_chatgpt_identity=True) - assert recorded["identity"] == ["chatgpt:chatgpt_zzz"] + assert recorded["upstream"] == ["chatgpt_zzz"] + assert recorded["identity"] == [] assert recorded["email"] == [] @@ -216,6 +275,7 @@ async def test_upsert_falls_back_to_id_lock_without_identity(monkeypatch): await repo.upsert(account, merge_by_email=False, merge_by_chatgpt_identity=False) + assert recorded["upstream"] == [] assert recorded["identity"] == ["acc_c"] assert recorded["email"] == [] @@ -231,5 +291,148 @@ async def test_upsert_email_only_when_identity_not_in_play(monkeypatch): await repo.upsert(account, merge_by_email=True, merge_by_chatgpt_identity=False) - assert recorded["identity"] == [], "no identity lock when merge_by_chatgpt_identity is False" + assert recorded["upstream"] == ["chatgpt_qqq"] + assert recorded["identity"] == [] assert recorded["email"] == ["d@example.com"] + assert recorded["order"] == ["upstream:chatgpt_qqq", "email:d@example.com"] + + +@pytest.mark.asyncio +async def test_ordinary_identity_upsert_uses_upstream_membership_lock(monkeypatch): + repo, recorded = _make_postgres_repo(monkeypatch) + account = _stub_account("acc_e", "e@example.com", chatgpt_id="chatgpt_ordinary") + + await repo.upsert(account, merge_by_email=False, merge_by_chatgpt_identity=False) + + assert recorded["upstream"] == ["chatgpt_ordinary"] + assert recorded["order"] == ["upstream:chatgpt_ordinary"] + + +@pytest.mark.asyncio +async def test_account_slot_upsert_locks_upstream_before_slot_keys(monkeypatch): + repo, recorded = _make_postgres_repo(monkeypatch) + account = _stub_account("acc_slot", "slot@example.com", chatgpt_id="chatgpt_slot") + account.workspace_id = "workspace-slot" + + await repo.upsert_account_slot(account, preserve_unknown_workspace_duplicates=False) + + assert recorded["upstream"] == ["chatgpt_slot"] + assert recorded["order"][0] == "upstream:chatgpt_slot" + assert all(item.startswith("identity:") for item in recorded["order"][1:]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("slot_upsert", [False, True]) +async def test_identity_candidate_revalidation_restarts_once_then_succeeds(monkeypatch, slot_upsert: bool): + repo, _recorded = _make_postgres_repo(monkeypatch) + account = _stub_account("acc_retry", "retry@example.com", chatgpt_id="chatgpt_retry") + candidates_are_locked = AsyncMock(side_effect=[False, True]) + monkeypatch.setattr(repo, "_postgresql_upsert_identity_candidates_are_locked", candidates_are_locked) + + if slot_upsert: + saved = await repo.upsert_account_slot(account, preserve_unknown_workspace_duplicates=False) + else: + saved = await repo.upsert(account, merge_by_email=False) + + assert saved is account + assert candidates_are_locked.await_count == 2 + assert cast(Any, repo.session.rollback).await_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("slot_upsert", [False, True]) +async def test_identity_candidate_revalidation_raises_typed_error_after_second_change( + monkeypatch, + slot_upsert: bool, +): + repo, _recorded = _make_postgres_repo(monkeypatch) + account = _stub_account("acc_terminal", "terminal@example.com", chatgpt_id="chatgpt_terminal") + monkeypatch.setattr( + repo, + "_postgresql_upsert_identity_candidates_are_locked", + AsyncMock(side_effect=[False, False]), + ) + + with pytest.raises(AccountIdentityRelockError): + if slot_upsert: + await repo.upsert_account_slot(account, preserve_unknown_workspace_duplicates=False) + else: + await repo.upsert(account, merge_by_email=False) + + assert cast(Any, repo.session.rollback).await_count == 2 + + +@pytest.mark.asyncio +async def test_local_identity_membership_relocks_after_observed_identity_changes(monkeypatch): + session = MagicMock() + changed = _stub_account("acc_relock", "relock@example.com", chatgpt_id="chatgpt_changed") + session.scalar = AsyncMock(side_effect=["chatgpt_old", changed, "chatgpt_changed", changed]) + session.rollback = AsyncMock() + repo = AccountsRepository(session) + identity_locks = AsyncMock() + monkeypatch.setattr(repository_module, "lock_postgresql_account_identities", identity_locks) + + locked = await repo._lock_postgresql_account_identity_membership("acc_relock", "chatgpt_incoming") + + assert locked is changed + assert [call.args[1] for call in identity_locks.await_args_list] == [ + ("chatgpt_old", "chatgpt_incoming"), + ("chatgpt_changed", "chatgpt_incoming"), + ] + session.rollback.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_local_identity_membership_raises_typed_error_after_second_change(monkeypatch): + session = MagicMock() + changed_once = _stub_account("acc_relock", "relock@example.com", chatgpt_id="chatgpt_changed") + changed_twice = _stub_account("acc_relock", "relock@example.com", chatgpt_id="chatgpt_changed_again") + session.scalar = AsyncMock(side_effect=["chatgpt_old", changed_once, "chatgpt_changed", changed_twice]) + session.rollback = AsyncMock() + repo = AccountsRepository(session) + monkeypatch.setattr(repository_module, "lock_postgresql_account_identities", AsyncMock()) + + with pytest.raises(AccountIdentityRelockError): + await repo._lock_postgresql_account_identity_membership("acc_relock", "chatgpt_incoming") + + assert session.rollback.await_count == 2 + + +@pytest.mark.asyncio +async def test_local_identity_writers_lock_old_and_incoming_membership(monkeypatch): + repo, _recorded = _make_postgres_repo(monkeypatch) + existing = _stub_account("acc_writer", "writer@example.com", chatgpt_id="chatgpt_old") + membership_locks: list[tuple[str, str | None]] = [] + cast(Any, repo.session.execute).return_value = _make_result("acc_writer") + + async def fake_membership_lock(account_id: str, incoming: str | None) -> Account: + membership_locks.append((account_id, incoming)) + return existing + + monkeypatch.setattr(repo, "_lock_postgresql_account_identity_membership", fake_membership_lock) + monkeypatch.setattr(repo, "_apply_account_replacement", AsyncMock()) + monkeypatch.setattr(repository_module, "lock_fold_state", AsyncMock()) + monkeypatch.setattr(repository_module, "mirror_account_soft_delete_into_time_rollups", AsyncMock()) + + await repo.replace_reauthorized( + existing.id, + _stub_account("incoming", existing.email, chatgpt_id="chatgpt_new"), + ) + assert await repo.rotate_tokens( + existing.id, + b"access", + b"refresh", + b"id", + utcnow(), + expected_refresh_token_encrypted=b"expected", + chatgpt_account_id="chatgpt_new", + ) + assert await repo.update_account_metadata(existing.id, chatgpt_account_id="chatgpt_new") + assert await repo.delete(existing.id) + + assert membership_locks == [ + (existing.id, "chatgpt_new"), + (existing.id, "chatgpt_new"), + (existing.id, "chatgpt_new"), + (existing.id, None), + ] diff --git a/tests/unit/test_accounts_service_transitions.py b/tests/unit/test_accounts_service_transitions.py index db7f318863..10888c1eeb 100644 --- a/tests/unit/test_accounts_service_transitions.py +++ b/tests/unit/test_accounts_service_transitions.py @@ -26,6 +26,7 @@ def _account( deactivation_reason=deactivation_reason, reset_at=reset_at, blocked_at=blocked_at, + delete_requested_at=None, ) diff --git a/tests/unit/test_api_keys_service.py b/tests/unit/test_api_keys_service.py index f31476ec0a..9a33fd47f1 100644 --- a/tests/unit/test_api_keys_service.py +++ b/tests/unit/test_api_keys_service.py @@ -6,7 +6,7 @@ from typing import Any, cast import pytest -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import IntegrityError, OperationalError from app.core.utils.time import utcnow from app.db.models import ( @@ -94,6 +94,9 @@ async def get_by_id(self, key_id: str) -> ApiKey | None: row.source_assignments = self._source_assignments.get(key_id, []) return row + async def get_for_limit_enforcement(self, key_id: str) -> ApiKey | None: + return await self.get_by_id(key_id) + async def get_by_hash(self, key_hash: str) -> ApiKey | None: for row in self.rows.values(): if row.key_hash == key_hash: @@ -146,6 +149,7 @@ async def update( apply_to_codex_model: bool | _Unset = _UNSET, enforced_model: str | None | _Unset = _UNSET, enforced_reasoning_effort: str | None | _Unset = _UNSET, + allowed_reasoning_efforts: str | None | _Unset = _UNSET, enforced_service_tier: str | None | _Unset = _UNSET, traffic_class: str | _Unset = _UNSET, transport_policy_override: str | None | _Unset = _UNSET, @@ -168,6 +172,7 @@ async def update( "apply_to_codex_model": apply_to_codex_model, "enforced_model": enforced_model, "enforced_reasoning_effort": enforced_reasoning_effort, + "allowed_reasoning_efforts": allowed_reasoning_efforts, "enforced_service_tier": enforced_service_tier, "traffic_class": traffic_class, "transport_policy_override": transport_policy_override, @@ -703,6 +708,108 @@ async def test_create_key_normalizes_enforced_reasoning_effort() -> None: assert created.enforced_reasoning_effort == "high" +@pytest.mark.asyncio +async def test_create_key_normalizes_allowed_reasoning_efforts() -> None: + repo = _FakeApiKeysRepository() + service = ApiKeysService(repo) + + created = await service.create_key( + ApiKeyCreateData( + name="selectable-reasoning-policy", + allowed_models=None, + allowed_reasoning_efforts=["XHIGH", "low", "high", "low"], + ) + ) + + assert created.allowed_reasoning_efforts == ["low", "high", "xhigh"] + stored = await repo.get_by_id(created.id) + assert stored is not None + assert stored.allowed_reasoning_efforts == '["low", "high", "xhigh"]' + assert (await service.validate_key(created.key)).id == created.id + + +@pytest.mark.asyncio +async def test_create_key_rejects_empty_or_conflicting_reasoning_policies() -> None: + service = ApiKeysService(_FakeApiKeysRepository()) + + with pytest.raises(ApiKeyValidationError, match="must not be empty"): + await service.create_key( + ApiKeyCreateData(name="empty-reasoning-policy", allowed_models=None, allowed_reasoning_efforts=[]) + ) + + with pytest.raises(ApiKeyValidationError, match="cannot be configured together"): + await service.create_key( + ApiKeyCreateData( + name="conflicting-reasoning-policy", + allowed_models=None, + enforced_reasoning_effort="low", + allowed_reasoning_efforts=["low"], + ) + ) + + +@pytest.mark.asyncio +async def test_update_key_validates_effective_reasoning_policy() -> None: + repo = _FakeApiKeysRepository() + service = ApiKeysService(repo) + created = await service.create_key( + ApiKeyCreateData(name="switchable-reasoning-policy", allowed_models=None, enforced_reasoning_effort="low") + ) + + with pytest.raises(ApiKeyValidationError, match="cannot be configured together"): + await service.update_key( + created.id, + ApiKeyUpdateData( + allowed_reasoning_efforts=["low", "medium"], + allowed_reasoning_efforts_set=True, + ), + ) + + updated = await service.update_key( + created.id, + ApiKeyUpdateData( + enforced_reasoning_effort=None, + enforced_reasoning_effort_set=True, + allowed_reasoning_efforts=["low", "medium"], + allowed_reasoning_efforts_set=True, + ), + ) + + assert updated.enforced_reasoning_effort is None + assert updated.allowed_reasoning_efforts == ["low", "medium"] + + +@pytest.mark.asyncio +async def test_update_key_translates_reasoning_policy_constraint_race() -> None: + class ConstraintFailingRepository(_FakeApiKeysRepository): + fail_reasoning_policy_commit = False + + async def commit(self) -> None: + if self.fail_reasoning_policy_commit: + raise IntegrityError( + "UPDATE api_keys", + {}, + Exception("CHECK constraint failed: ck_api_keys_reasoning_policy_exclusive"), + ) + await super().commit() + + repo = ConstraintFailingRepository() + service = ApiKeysService(repo) + created = await service.create_key(ApiKeyCreateData(name="concurrent-reasoning-policy", allowed_models=None)) + repo.fail_reasoning_policy_commit = True + + with pytest.raises(ApiKeyValidationError, match="cannot be configured together"): + await service.update_key( + created.id, + ApiKeyUpdateData( + allowed_reasoning_efforts=["low"], + allowed_reasoning_efforts_set=True, + ), + ) + + assert repo.rollback_calls == 1 + + @pytest.mark.asyncio async def test_create_key_persists_apply_to_codex_model_flag() -> None: repo = _FakeApiKeysRepository() @@ -741,6 +848,23 @@ async def test_create_key_normalizes_fast_service_tier_alias() -> None: assert created.enforced_service_tier == "priority" +@pytest.mark.asyncio +async def test_create_key_preserves_ultrafast_service_tier() -> None: + repo = _FakeApiKeysRepository() + service = ApiKeysService(repo) + + created = await service.create_key( + ApiKeyCreateData( + name="ultrafast-service-tier-policy", + allowed_models=None, + enforced_service_tier=" ULTRAFAST ", + expires_at=None, + ) + ) + + assert created.enforced_service_tier == "ultrafast" + + @pytest.mark.asyncio async def test_update_key_normalizes_service_tier_alias() -> None: repo = _FakeApiKeysRepository() @@ -1392,6 +1516,7 @@ async def test_enforce_limits_reserves_tier_aware_cost_budget() -> None: request_service_tier="priority", request_usage_budget=ApiKeyRequestUsageBudget(input_tokens=8192, output_tokens=8192), ) + assert priority_reservation is not None assert priority_reservation.key_id == priority_created.id priority_limits = await repo.get_limits_by_key(priority_created.id) @@ -1414,6 +1539,7 @@ async def test_enforce_limits_reserves_tier_aware_cost_budget() -> None: request_service_tier=None, request_usage_budget=ApiKeyRequestUsageBudget(input_tokens=8192, output_tokens=8192), ) + assert standard_reservation is not None assert standard_reservation.key_id == standard_created.id standard_limits = await repo.get_limits_by_key(standard_created.id) @@ -1453,7 +1579,9 @@ async def test_enforce_limits_default_budget_allows_eight_priority_lanes_under_f ) assert len(reservations) == 8 - assert {reservation.key_id for reservation in reservations} == {created.id} + granted = [reservation for reservation in reservations if reservation is not None] + assert len(granted) == 8 + assert {reservation.key_id for reservation in granted} == {created.id} limits = await repo.get_limits_by_key(created.id) cost_limit = next(lim for lim in limits if lim.limit_type == LimitType.COST_USD) assert 0 < cost_limit.current_value < 5_000_000 @@ -1510,6 +1638,7 @@ async def test_finalize_usage_reservation_accounts_for_zero_reserved_limit_item( request_model="gpt-5.5", request_usage_budget=ApiKeyRequestUsageBudget(input_tokens=0, output_tokens=0), ) + assert reservation is not None limits = await repo.get_limits_by_key(created.id) output_limit = next(limit for limit in limits if limit.limit_type == LimitType.OUTPUT_TOKENS) @@ -1553,16 +1682,110 @@ async def create_usage_reservation( repo = _BusyRepo() service = ApiKeysService(repo) monkeypatch.setattr("app.modules.api_keys.service.asyncio.sleep", _async_noop) - created = await service.create_key(ApiKeyCreateData(name="busy-retry-key", allowed_models=None, expires_at=None)) + created = await service.create_key( + ApiKeyCreateData( + name="busy-retry-key", + allowed_models=None, + expires_at=None, + limits=[LimitRuleInput(limit_type="total_tokens", limit_window="weekly", max_value=1_000_000)], + ) + ) initial_commit_count = repo.commit_count reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None assert reservation.key_id == created.id assert repo.create_usage_reservation_calls == 3 assert repo.commit_count == initial_commit_count + 1 +@pytest.mark.asyncio +async def test_enforce_limits_without_limits_skips_reservation_and_commit() -> None: + repo = _FakeApiKeysRepository() + coalescer = ApiKeyLastUsedCoalescer() + service = ApiKeysService(repo, last_used_coalescer=coalescer) + created = await service.create_key(ApiKeyCreateData(name="unlimited-key", allowed_models=None, expires_at=None)) + initial_commit_count = repo.commit_count + initial_rollback_calls = repo.rollback_calls + + reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + + assert reservation is None + assert repo._reservations == {} + # The implicit transaction opened by the admission SELECTs must be closed + # on the limit-free path (long-lived sessions would otherwise idle in + # transaction across the upstream round-trip) — via commit(), never + # rollback(): rollback expires every ORM object tracked by a shared + # session even with expire_on_commit=False, which broke quota-planner + # warmup probes. The commit is read-only (no reservation was inserted). + assert repo.commit_count == initial_commit_count + 1 + assert repo.rollback_calls == initial_rollback_calls + # Settlement never runs without a reservation, so admission itself must + # record the last-used touch for limit-free keys. + pending = coalescer.pending_snapshot() + assert set(pending) == {created.id} + assert pending[created.id] <= utcnow() + + +@pytest.mark.asyncio +async def test_enforce_limits_with_no_applicable_limits_skips_reservation_and_leaves_limits_untouched() -> None: + repo = _FakeApiKeysRepository() + service = ApiKeysService(repo) + created = await service.create_key( + ApiKeyCreateData( + name="filtered-limits-key", + allowed_models=None, + expires_at=None, + limits=[ + LimitRuleInput( + limit_type="total_tokens", + limit_window="weekly", + max_value=10_000, + model_filter="gpt-5.1", + ), + ], + ) + ) + initial_commit_count = repo.commit_count + initial_rollback_calls = repo.rollback_calls + + reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.5") + + assert reservation is None + assert repo._reservations == {} + # One read-only commit closes the admission transaction; no rollback + # (rollback would expire shared-session ORM state — see the limit-free + # transaction-close comment in _enforce_limits_for_request_once). + assert repo.commit_count == initial_commit_count + 1 + assert repo.rollback_calls == initial_rollback_calls + limits = await repo.get_limits_by_key(created.id) + assert limits[0].current_value == 0 + + +@pytest.mark.asyncio +async def test_enforce_limits_with_applicable_limit_still_creates_reservation() -> None: + repo = _FakeApiKeysRepository() + service = ApiKeysService(repo) + created = await service.create_key( + ApiKeyCreateData( + name="limited-regression-key", + allowed_models=None, + expires_at=None, + limits=[LimitRuleInput(limit_type="total_tokens", limit_window="weekly", max_value=1_000_000)], + ) + ) + initial_commit_count = repo.commit_count + + reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + + assert reservation is not None + assert reservation.key_id == created.id + assert reservation.has_applicable_limits is True + assert reservation.reservation_id in repo._reservations + assert repo.commit_count == initial_commit_count + 1 + + @pytest.mark.asyncio async def test_enforce_limits_retries_sqlite_busy_during_lazy_reset_rolls_back(monkeypatch: pytest.MonkeyPatch) -> None: class _BusyRepo(_FakeApiKeysRepository): @@ -1608,6 +1831,7 @@ async def rollback(self) -> None: reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5") + assert reservation is not None assert reservation.key_id == created.id assert repo.reset_limit_calls == 3 assert repo.rollback_calls >= 2 @@ -1854,6 +2078,7 @@ async def test_usage_reservation_uses_gpt_5_6_personality_pricing( request_model=model, request_usage_budget=ApiKeyRequestUsageBudget(input_tokens=8_192, output_tokens=8_192), ) + assert reservation is not None limits = await repo.get_limits_by_key(created.id) cost_limit = next(lim for lim in limits if lim.limit_type == LimitType.COST_USD) @@ -1885,6 +2110,7 @@ async def test_release_usage_reservation_restores_reserved_counter() -> None: ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None limits = await repo.get_limits_by_key(created.id) assert limits[0].current_value == 100 @@ -1909,6 +2135,7 @@ async def test_touch_usage_reservation_only_updates_reserved_reservation() -> No ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None assert await service.touch_usage_reservation(reservation.reservation_id) is True await service.release_usage_reservation(reservation.reservation_id) @@ -1932,6 +2159,7 @@ async def test_finalize_usage_reservation_is_idempotent() -> None: ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None await service.finalize_usage_reservation( reservation.reservation_id, model="gpt-5.1", @@ -1969,6 +2197,7 @@ async def test_finalize_usage_reservation_records_last_used_in_coalescer() -> No initial_commit_count = repo.commit_count reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None assert repo.commit_count == initial_commit_count + 1 await service.finalize_usage_reservation( @@ -2019,6 +2248,7 @@ async def get_usage_reservation(self, reservation_id: str) -> UsageReservationDa ) ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None await service.finalize_usage_reservation( reservation.reservation_id, @@ -2066,6 +2296,7 @@ async def get_usage_reservation(self, reservation_id: str) -> UsageReservationDa ) ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None await service.release_usage_reservation(reservation.reservation_id) @@ -2094,6 +2325,7 @@ async def test_fail_usage_reservation_preserves_failed_request_record() -> None: ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None await service.fail_usage_reservation( reservation.reservation_id, model="gpt-5.1", @@ -2126,6 +2358,7 @@ async def test_release_after_finalize_is_noop() -> None: ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None limits = await repo.get_limits_by_key(created.id) assert limits[0].current_value == 100 # reserved @@ -2164,6 +2397,7 @@ async def test_finalize_after_release_is_noop() -> None: ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None await service.release_usage_reservation(reservation.reservation_id) diff --git a/tests/unit/test_app_version_middleware.py b/tests/unit/test_app_version_middleware.py index 80d2075f92..e45f7ccafe 100644 --- a/tests/unit/test_app_version_middleware.py +++ b/tests/unit/test_app_version_middleware.py @@ -1,14 +1,10 @@ from __future__ import annotations import asyncio -from collections.abc import Awaitable, Callable -from typing import cast import pytest -from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse, Response +from fastapi import FastAPI, Response from httpx import ASGITransport, AsyncClient -from starlette.types import Message import app.main as main from app import __version__ @@ -17,98 +13,43 @@ pytestmark = pytest.mark.unit -_Dispatch = Callable[[Request, Callable[[Request], Awaitable[Response]]], Awaitable[Response]] - -@pytest.mark.asyncio -async def test_app_version_middleware_adds_header_to_2xx_response(): +def _build_app(status_code: int, *, headers: dict[str, str] | None = None) -> FastAPI: app = FastAPI() add_app_version_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) - - request = Request( - { - "type": "http", - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/health", - "raw_path": b"/health", - "query_string": b"", - "root_path": "", - "headers": [], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=_empty_receive, - ) - async def call_next(_: Request) -> JSONResponse: - return JSONResponse({"ok": True}, status_code=204) + @app.get("/probe") + async def probe() -> Response: + return Response(status_code=status_code, headers=headers) + + return app + - response = await dispatch(request, call_next) +@pytest.mark.asyncio +async def test_app_version_middleware_adds_header_to_2xx_response(): + transport = ASGITransport(app=_build_app(204)) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + response = await client.get("/probe") + assert response.status_code == 204 assert response.headers["X-App-Version"] == __version__ @pytest.mark.asyncio async def test_app_version_middleware_skips_header_on_5xx_response(): - app = FastAPI() - add_app_version_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) - - request = Request( - { - "type": "http", - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/health", - "raw_path": b"/health", - "query_string": b"", - "root_path": "", - "headers": [], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=_empty_receive, - ) - - async def call_next(_: Request) -> JSONResponse: - return JSONResponse({"error": "boom"}, status_code=503) - - response = await dispatch(request, call_next) + transport = ASGITransport(app=_build_app(503)) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + response = await client.get("/probe") + assert response.status_code == 503 assert "X-App-Version" not in response.headers @pytest.mark.asyncio async def test_app_version_middleware_preserves_existing_header_value(): - app = FastAPI() - add_app_version_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) - - request = Request( - { - "type": "http", - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/health", - "raw_path": b"/health", - "query_string": b"", - "root_path": "", - "headers": [], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=_empty_receive, - ) - - async def call_next(_: Request) -> Response: - return Response(status_code=200, headers={"X-App-Version": "route-owned-version"}) - - response = await dispatch(request, call_next) + transport = ASGITransport(app=_build_app(200, headers={"X-App-Version": "route-owned-version"})) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + response = await client.get("/probe") assert response.headers["X-App-Version"] == "route-owned-version" @@ -145,7 +86,3 @@ async def work(): assert overloaded.status_code == 429 assert overloaded.headers["X-App-Version"] == __version__ - - -async def _empty_receive() -> Message: - return {"type": "http.request", "body": b"", "more_body": False} diff --git a/tests/unit/test_auth_dependencies_upstream_proxy.py b/tests/unit/test_auth_dependencies_upstream_proxy.py index 918243611a..75ccbf30a8 100644 --- a/tests/unit/test_auth_dependencies_upstream_proxy.py +++ b/tests/unit/test_auth_dependencies_upstream_proxy.py @@ -5,8 +5,10 @@ from typing import Any, cast import pytest +from starlette.datastructures import Headers from app.core.auth import dependencies as auth_dependencies +from app.core.clients.proxy import CODEX_LB_REQUIRED_CAPABILITY_HEADER from app.core.upstream_proxy import ResolvedProxyEndpoint, ResolvedUpstreamRoute, UpstreamProxyRouteError from app.core.usage.models import UsagePayload from app.db.models import Account, AccountStatus @@ -14,6 +16,114 @@ pytestmark = pytest.mark.unit +@pytest.mark.asyncio +async def test_validate_proxy_api_key_requires_carrier_authentication( + monkeypatch: pytest.MonkeyPatch, +) -> None: + principal = object() + request = cast( + Any, + SimpleNamespace(headers=Headers({CODEX_LB_REQUIRED_CAPABILITY_HEADER: "trusted_cyber"})), + ) + + async def required_auth(authorization: str | None) -> object: + assert authorization == "Bearer inert-key" + return principal + + async def fail_ordinary_auth(*_args: object, **_kwargs: object) -> None: + pytest.fail("capability carrier must use required per-request authentication") + + monkeypatch.setattr(auth_dependencies, "validate_required_proxy_api_key_authorization", required_auth) + monkeypatch.setattr(auth_dependencies, "validate_proxy_api_key_authorization", fail_ordinary_auth) + + resolved = await auth_dependencies.validate_proxy_api_key( + request, + cast(Any, SimpleNamespace(credentials="inert-key")), + ) + + assert resolved is principal + + +@pytest.mark.asyncio +async def test_validate_proxy_api_key_preserves_headerless_authentication( + monkeypatch: pytest.MonkeyPatch, +) -> None: + principal = object() + request = cast(Any, SimpleNamespace(headers=Headers())) + + async def fail_required_auth(_authorization: str | None) -> None: + pytest.fail("headerless provider request must retain ordinary authentication") + + async def ordinary_auth(authorization: str | None, *, request: object | None = None) -> object: + assert authorization == "Bearer ordinary-key" + assert request is not None + return principal + + monkeypatch.setattr(auth_dependencies, "validate_required_proxy_api_key_authorization", fail_required_auth) + monkeypatch.setattr(auth_dependencies, "validate_proxy_api_key_authorization", ordinary_auth) + + resolved = await auth_dependencies.validate_proxy_api_key( + request, + cast(Any, SimpleNamespace(credentials="ordinary-key")), + ) + + assert resolved is principal + + +@pytest.mark.asyncio +async def test_validate_codex_provider_usage_identity_authenticates_carrier_before_usage_io( + monkeypatch: pytest.MonkeyPatch, +) -> None: + principal = object() + request = cast( + Any, + SimpleNamespace( + headers=Headers( + { + CODEX_LB_REQUIRED_CAPABILITY_HEADER: "trusted_cyber", + "Authorization": "Bearer inert-key", + } + ) + ), + ) + + async def required_auth(authorization: str | None) -> object: + assert authorization == "Bearer inert-key" + return principal + + async def fail_usage_identity(*_args: object, **_kwargs: object) -> None: + pytest.fail("capability carrier must not enter ChatGPT usage identity validation") + + monkeypatch.setattr(auth_dependencies, "validate_required_proxy_api_key_authorization", required_auth) + monkeypatch.setattr(auth_dependencies, "validate_codex_usage_identity", fail_usage_identity) + + resolved = await auth_dependencies.validate_codex_provider_usage_identity(request) + + assert resolved is principal + + +@pytest.mark.asyncio +async def test_validate_codex_provider_usage_identity_preserves_headerless_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + principal = object() + request = cast(Any, SimpleNamespace(headers=Headers({"Authorization": "Bearer ordinary-token"}))) + + async def fail_required_auth(_authorization: str | None) -> None: + pytest.fail("headerless usage identity must retain ordinary validation") + + async def ordinary_usage_identity(current_request: object) -> object: + assert current_request is request + return principal + + monkeypatch.setattr(auth_dependencies, "validate_required_proxy_api_key_authorization", fail_required_auth) + monkeypatch.setattr(auth_dependencies, "validate_codex_usage_identity", ordinary_usage_identity) + + resolved = await auth_dependencies.validate_codex_provider_usage_identity(request) + + assert resolved is principal + + def _account() -> Account: return Account( id="acc_1", @@ -59,7 +169,7 @@ async def resolve_route(*args: object, **kwargs: object) -> ResolvedUpstreamRout async def fetch_usage(*args: object, **kwargs: object) -> None: calls["fetch_kwargs"] = kwargs - monkeypatch.setattr(auth_dependencies, "get_request_session", session_context) + monkeypatch.setattr(auth_dependencies, "get_background_session", session_context) monkeypatch.setattr(auth_dependencies, "AccountsRepository", Repo) monkeypatch.setattr(auth_dependencies, "resolve_upstream_route", resolve_route) monkeypatch.setattr(auth_dependencies, "fetch_usage", fetch_usage) @@ -128,7 +238,7 @@ async def fetch_usage(*args: object, **kwargs: object) -> UsagePayload: assert kwargs["route"] is owner_route return UsagePayload(workspace_id="ws_1", workspace_label="Team") - monkeypatch.setattr(auth_dependencies, "get_request_session", session_context) + monkeypatch.setattr(auth_dependencies, "get_background_session", session_context) monkeypatch.setattr(auth_dependencies, "AccountsRepository", Repo) monkeypatch.setattr(auth_dependencies, "resolve_upstream_route", resolve_route) monkeypatch.setattr(auth_dependencies, "fetch_usage", fetch_usage) @@ -195,7 +305,7 @@ async def fetch_usage(*args: object, **kwargs: object) -> UsagePayload: assert kwargs["route"] is owner_route return UsagePayload(workspace_id="ws_1", workspace_label="Team") - monkeypatch.setattr(auth_dependencies, "get_request_session", session_context) + monkeypatch.setattr(auth_dependencies, "get_background_session", session_context) monkeypatch.setattr(auth_dependencies, "AccountsRepository", Repo) monkeypatch.setattr(auth_dependencies, "resolve_upstream_route", resolve_route) monkeypatch.setattr(auth_dependencies, "fetch_usage", fetch_usage) @@ -254,7 +364,7 @@ async def fetch_usage(*args: object, **kwargs: object) -> UsagePayload: assert kwargs["route"] is owner_route return UsagePayload(workspace_id="ws_1", workspace_label="Team") - monkeypatch.setattr(auth_dependencies, "get_request_session", session_context) + monkeypatch.setattr(auth_dependencies, "get_background_session", session_context) monkeypatch.setattr(auth_dependencies, "AccountsRepository", Repo) monkeypatch.setattr(auth_dependencies, "resolve_upstream_route", resolve_route) monkeypatch.setattr(auth_dependencies, "fetch_usage", fetch_usage) @@ -287,7 +397,7 @@ async def session_context(): async def resolve_route(*args: object, **kwargs: object) -> ResolvedUpstreamRoute: raise UpstreamProxyRouteError("default_pool_unconfigured", account_id="acc_1") - monkeypatch.setattr(auth_dependencies, "get_request_session", session_context) + monkeypatch.setattr(auth_dependencies, "get_background_session", session_context) monkeypatch.setattr(auth_dependencies, "AccountsRepository", Repo) monkeypatch.setattr(auth_dependencies, "resolve_upstream_route", resolve_route) diff --git a/tests/unit/test_auth_guardian.py b/tests/unit/test_auth_guardian.py index 377309b9a9..5f0f3bd787 100644 --- a/tests/unit/test_auth_guardian.py +++ b/tests/unit/test_auth_guardian.py @@ -15,9 +15,8 @@ from app.core.auth.refresh import RefreshError from app.core.config import settings as settings_module from app.db.models import Account, AccountStatus, Base -from app.db.session import close_session, detach_session_objects +from app.db.session import close_session from app.modules.accounts.auth_manager import AuthManager -from app.modules.accounts.background_repository import BackgroundAccountsRepository from app.modules.accounts.repository import AccountsRepository pytestmark = pytest.mark.unit @@ -102,25 +101,14 @@ def test_select_auth_guardian_candidates_returns_stale_eligible_accounts_only() assert [account.id for account in batched] == ["oldest-active", "stale-paused"] -def test_default_auth_manager_factory_uses_per_operation_background_repo() -> None: - candidate_repo = _Repo([]) +def test_default_auth_manager_factory_uses_owned_refresh_repo() -> None: + repo = _Repo([]) - manager = cast(AuthManager, guardian_module._default_auth_manager_factory(candidate_repo)) + manager = cast(AuthManager, guardian_module._default_auth_manager_factory(repo)) - assert isinstance(manager._repo, BackgroundAccountsRepository) - assert manager._repo is not candidate_repo assert manager._refresh_repo_factory is guardian_module._default_accounts_repo_factory -@pytest.mark.asyncio -async def test_default_accounts_repo_factory_returns_independent_background_repositories() -> None: - async with guardian_module._default_accounts_repo_factory() as first: - async with guardian_module._default_accounts_repo_factory() as second: - assert isinstance(first, BackgroundAccountsRepository) - assert isinstance(second, BackgroundAccountsRepository) - assert first is not second - - def test_build_auth_guardian_scheduler_allows_single_replica_without_leader_election( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -315,7 +303,6 @@ async def repo_factory() -> AsyncIterator[AccountsRepository]: try: yield AccountsRepository(session) finally: - detach_session_objects(session) await close_session(session) calls: list[str] = [] @@ -653,7 +640,7 @@ async def repo_factory() -> AsyncIterator[_Repo]: @pytest.mark.asyncio -async def test_auth_guardian_exits_candidate_repo_before_waiting_for_cancelled_refresh() -> None: +async def test_auth_guardian_waits_for_refresh_before_cancelled_candidate_exits() -> None: now = datetime(2026, 1, 2, 12, 0, 0) account = _account("stale-active", status=AccountStatus.ACTIVE, last_refresh=now - timedelta(hours=13)) repo = _Repo([account]) @@ -667,7 +654,6 @@ async def ensure_fresh(self, account: Account, *, force: bool = False) -> Accoun nonlocal completed assert force is True assert account.id == "stale-active" - assert repo_exited is True started.set() await allow_finish.wait() completed = True @@ -680,7 +666,8 @@ async def repo_factory() -> AsyncIterator[_Repo]: try: yield repo finally: - repo_exited = True + if started.is_set(): + repo_exited = True scheduler = AuthGuardianScheduler( interval_seconds=21600, @@ -703,7 +690,7 @@ async def repo_factory() -> AsyncIterator[_Repo]: await asyncio.sleep(0) assert completed is False - assert repo_exited is True + assert repo_exited is False allow_finish.set() with pytest.raises(asyncio.CancelledError): diff --git a/tests/unit/test_auth_manager.py b/tests/unit/test_auth_manager.py index a1967c6d2a..1b9da7aa1c 100644 --- a/tests/unit/test_auth_manager.py +++ b/tests/unit/test_auth_manager.py @@ -593,6 +593,87 @@ async def _fake_refresh(_: str, **_kwargs: object) -> TokenRefreshResult: assert refresh_calls == 1 +@pytest.mark.asyncio +async def test_ensure_fresh_old_failure_cannot_replace_successor(monkeypatch): + """A delayed failed completion must not evict a newer refresh task.""" + encryptor = TokenEncryptor() + stale_refresh = utcnow().replace(year=utcnow().year - 1) + account = Account( + id="acc_sf_successor", + email="user@example.com", + plan_type="plus", + access_token_encrypted=encryptor.encrypt("access-old"), + refresh_token_encrypted=encryptor.encrypt("refresh-old"), + id_token_encrypted=encryptor.encrypt("id-old"), + last_refresh=stale_refresh, + status=AccountStatus.ACTIVE, + deactivation_reason=None, + ) + refreshed_payload = {column.name: getattr(account, column.name) for column in Account.__table__.columns} + refreshed_payload.update( + access_token_encrypted=encryptor.encrypt("access-new"), + refresh_token_encrypted=encryptor.encrypt("refresh-new"), + ) + refreshed = Account(**refreshed_payload) + repo = _DummyRepo() + manager = AuthManager(cast(AccountsRepositoryPort, repo)) + monkeypatch.setattr(manager, "_ensure_chatgpt_account_id", lambda value: _identity(value)) + + mode = {"calls": 0} + successor_started = asyncio.Event() + release_successor = asyncio.Event() + + async def fake_run(_account): + mode["calls"] += 1 + if mode["calls"] == 1: + raise RefreshError("invalid_grant", "old refresh failed", False) + successor_started.set() + await release_successor.wait() + return refreshed + + monkeypatch.setattr(manager, "_run_refresh", fake_run) + singleflight = auth_manager_module._REFRESH_SINGLEFLIGHT + old_completion_started = asyncio.Event() + old_completion_finished = asyncio.Event() + release_old_completion = asyncio.Event() + original_complete = singleflight._complete + + async def hold_old_completion(key, task): + old_completion_started.set() + await release_old_completion.wait() + await original_complete(key, task) + old_completion_finished.set() + + monkeypatch.setattr(singleflight, "_complete", hold_old_completion) + + with pytest.raises(RefreshError, match="old refresh failed"): + await manager.ensure_fresh(account, force=True) + await old_completion_started.wait() + + successor = asyncio.create_task(manager.ensure_fresh(account, force=True)) + await successor_started.wait() + joined_successor = asyncio.create_task(manager.ensure_fresh(account, force=True)) + await asyncio.sleep(0) + assert not joined_successor.done() + assert mode["calls"] == 2 + + # The failed task's callback settles after the successor is installed. + release_old_completion.set() + await old_completion_finished.wait() + late_caller = asyncio.create_task(manager.ensure_fresh(account, force=True)) + await asyncio.sleep(0) + assert not late_caller.done() + release_successor.set() + assert await successor is refreshed + assert await joined_successor is refreshed + assert await late_caller is refreshed + assert mode["calls"] == 2 + + +async def _identity(value): + return value + + @pytest.mark.asyncio async def test_ensure_fresh_singleflights_refresh_admission_for_same_account(monkeypatch): started = asyncio.Event() diff --git a/tests/unit/test_bridge_ring_lifecycle.py b/tests/unit/test_bridge_ring_lifecycle.py index 744cd6c51f..6cead41406 100644 --- a/tests/unit/test_bridge_ring_lifecycle.py +++ b/tests/unit/test_bridge_ring_lifecycle.py @@ -22,6 +22,7 @@ AccountStatus, Base, BridgeRingMember, + HttpBridgeOperationRecord, HttpBridgeRetryCircuit, HttpBridgeSessionAlias, HttpBridgeSessionRecord, @@ -29,14 +30,19 @@ ) from app.modules.proxy import service as proxy_service from app.modules.proxy._service.http_bridge.helpers import ( + _http_bridge_allow_durable_takeover, + _http_bridge_claim_allows_takeover, _http_bridge_durable_lookup_allows_turn_state_takeover, ) from app.modules.proxy.continuity import make_http_bridge_account_neutral_replay_key +from app.modules.proxy.durable_bridge_coordinator import DurableBridgeSessionCoordinator from app.modules.proxy.durable_bridge_repository import ( DurableBridgeAliasRegistration, DurableBridgeRepository, durable_bridge_hash, + durable_bridge_operation_id, ) +from app.modules.proxy.http_bridge_event_batcher import HttpBridgeOperationEventBatcher from app.modules.proxy.ring_membership import RingMembershipService pytestmark = pytest.mark.unit @@ -565,6 +571,1126 @@ async def test_recovery_attempt_pre_dispatch_claim_can_be_rolled_back( ) assert restored is not None assert restored.state.value == "unknown" + assert await repository.rollback_recovery_attempt_before_dispatch( + session_id=claim.id, + instance_id="inst-recovery-rollback", + owner_epoch=claim.owner_epoch, + request_fingerprint="fingerprint-recovery-rollback", + ) + assert ( + await repository.lookup_recovery_attempt( + session_id=claim.id, + request_fingerprint="fingerprint-recovery-rollback", + ) + is None + ) + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_operation_ledger_is_fenced_and_idempotent( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-operation-ledger", session_key_value="sid-operation") + fingerprint = durable_bridge_hash("continuation-body") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + created = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-ledger", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + request_text='{"model":"gpt-5.6","input":"turn"}', + ) + assert created is not None + assert created.created is True + assert created.state == "submitted" + assert created.request_text == '{"model":"gpt-5.6","input":"turn"}' + assert created.event_spool_complete is False + + existing = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-ledger", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert existing is not None + assert existing.created is False + assert existing.operation_id == operation_id + + assert await repository.update_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-ledger", + owner_epoch=claim.owner_epoch, + state="completed", + response_id="resp-completed", + ) + completed = await repository.get_latest_completed_operation( + session_id=claim.id, + parent_response_id="resp-parent", + ) + assert completed is not None + assert completed.response_id == "resp-completed" + by_fingerprint = await repository.get_operation_by_fingerprint(request_fingerprint=fingerprint) + assert by_fingerprint is not None + assert by_fingerprint.operation_id == operation_id + cross_session_completed = await repository.get_latest_completed_operation_any_session( + parent_response_id="resp-parent", + ) + assert cross_session_completed is not None + assert cross_session_completed.response_id == "resp-completed" + + assert await repository.append_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-ledger", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.completed"}\n\n', + max_bytes=1024, + ) + # Repeated identical SSE blocks are distinct downstream occurrences, + # so replay must preserve both copies rather than hash-deduplicating. + assert await repository.append_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-ledger", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.completed"}\n\n', + max_bytes=1024, + ) + assert await repository.get_operation_events(operation_id=operation_id) == [ + 'data: {"type":"response.completed"}\n\n', + 'data: {"type":"response.completed"}\n\n', + ] + # A missing parent turn makes the chain ineligible rather than + # silently constructing an incomplete conversation. + assert await repository.get_replayable_transcript(response_id="resp-completed") is None + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_operation_retry_reset_clears_partial_spool( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-operation-reset", session_key_value="sid-operation-reset") + fingerprint = durable_bridge_hash("continuation-reset") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-reset", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + assert await repository.append_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-reset", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.output_text.delta"}\n\n', + max_bytes=1024, + ) + assert await repository.reset_operation_event_spool( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-reset", + owner_epoch=claim.owner_epoch, + ) + assert await repository.get_operation_events(operation_id=operation_id) == [] + reset = await repository.get_operation(operation_id=operation_id) + assert reset is not None + assert reset.event_spool_complete is False + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_terminal_operation_event_exposes_failure_after_spooling( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-terminal-event", session_key_value="sid-terminal-event") + fingerprint = durable_bridge_hash("terminal-event") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-event", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-terminal-event", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + event_text = 'data: {"type":"response.failed"}\n\n' + + assert await repository.append_terminal_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-event", + owner_epoch=claim.owner_epoch, + event_text=event_text, + max_bytes=1024, + state="failed", + ) + failed = await repository.get_operation(operation_id=operation_id) + assert failed is not None + assert failed.state == "failed" + assert await repository.get_operation_events(operation_id=operation_id) == [event_text] + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_terminal_failure_exposes_state_when_spool_overflows( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim( + repository, + instance_id="inst-terminal-overflow", + session_key_value="sid-terminal-overflow", + ) + fingerprint = durable_bridge_hash("terminal-overflow") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-overflow", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-terminal-overflow", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + + persisted = await repository.append_terminal_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-overflow", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.failed"}\n\n', + max_bytes=1, + state="failed", + ) + + assert persisted is False + failed = await repository.get_operation(operation_id=operation_id) + assert failed is not None + assert failed.state == "failed" + assert failed.event_spool_complete is False + assert await repository.get_operation_events(operation_id=operation_id) == [] + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_terminal_append_failure_settlement_is_visible_to_recovery( + async_session_factory: Callable[[], AsyncSession], + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim( + repository, + instance_id="inst-terminal-recovery", + session_key_value="sid-terminal-recovery", + ) + fingerprint = durable_bridge_hash("terminal-recovery") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-terminal-recovery", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + assert await repository.append_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.created"}\n\n', + max_bytes=1024, + ) + assert await repository.update_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="acknowledged", + response_id="resp-terminal-recovery", + ) + + replay_fingerprint = durable_bridge_hash("terminal-recovery-replay-alias") + replay_operation_id = durable_bridge_operation_id(claim.id, replay_fingerprint) + assert await repository.record_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + request_fingerprint=replay_fingerprint, + account_id="account-terminal-recovery", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert await repository.update_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="acknowledged", + response_id="resp-upstream-replay", + ) + assert await repository.settle_terminal_append_failure( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-upstream-replay", + response_id="resp-client-visible-replay", + ) + replay_operation = await repository.get_operation(operation_id=replay_operation_id) + assert replay_operation is not None + assert replay_operation.state == "failed" + assert replay_operation.response_id == "resp-client-visible-replay" + assert replay_operation.event_spool_complete is False + + assert await repository.update_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + response_id="resp-upstream-replay", + ) + assert await repository.settle_terminal_append_failure( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-upstream-replay", + response_id="resp-client-visible-replay", + ) + pre_settled_replay = await repository.get_operation(operation_id=replay_operation_id) + assert pre_settled_replay is not None + assert pre_settled_replay.state == "failed" + assert pre_settled_replay.response_id == "resp-client-visible-replay" + + assert await repository.update_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="acknowledged", + response_id="resp-persisted-before-replacement", + ) + assert await repository.settle_terminal_append_failure( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-unpersisted-replacement", + alternate_expected_response_id="resp-persisted-before-replacement", + response_id="resp-client-visible-replay", + ) + partially_persisted_replay = await repository.get_operation(operation_id=replay_operation_id) + assert partially_persisted_replay is not None + assert partially_persisted_replay.state == "failed" + assert partially_persisted_replay.response_id == "resp-client-visible-replay" + + assert await repository.update_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="acknowledged", + response_id="resp-upstream-replay", + ) + assert await repository.settle_terminal_append_failure( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-upstream-replay", + response_id=None, + ) + null_alias_settlement = await repository.get_operation(operation_id=replay_operation_id) + assert null_alias_settlement is not None + assert null_alias_settlement.state == "failed" + assert null_alias_settlement.response_id == "resp-upstream-replay" + + assert await repository.update_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="unknown", + ) + assert await repository.claim_unknown_operation_for_recovery( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + ) + assert await repository.update_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="acknowledged", + response_id="resp-upstream-replay", + ) + assert not await repository.append_terminal_operation_event( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.failed"}\n\n', + max_bytes=1024, + state="failed", + expected_recovery_dispatch_count=0, + response_id="resp-client-visible-replay", + ) + assert not await repository.settle_terminal_append_failure( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-upstream-replay", + expected_recovery_dispatch_count=0, + response_id="resp-client-visible-replay", + ) + newer_attempt = await repository.get_operation(operation_id=replay_operation_id) + assert newer_attempt is not None + assert newer_attempt.state == "acknowledged" + assert newer_attempt.recovery_dispatch_count == 1 + assert newer_attempt.event_spool_complete is False + finally: + await session.close() + + coordinator = DurableBridgeSessionCoordinator(async_session_factory) + append_terminal_operation_event = coordinator.append_terminal_operation_event + settle_terminal_append_failure = coordinator.settle_terminal_append_failure + settlement_finished = asyncio.Event() + + async def fail_terminal_append(**kwargs: Any) -> bool: + assert await append_terminal_operation_event(**kwargs) + raise RuntimeError("injected post-commit terminal append failure") + + async def track_terminal_settlement(**kwargs: Any) -> bool: + try: + return await settle_terminal_append_failure(**kwargs) + finally: + settlement_finished.set() + + monkeypatch.setattr(coordinator, "append_terminal_operation_event", fail_terminal_append) + monkeypatch.setattr(coordinator, "settle_terminal_append_failure", track_terminal_settlement) + batcher = HttpBridgeOperationEventBatcher( + coordinator, + max_bytes=1024, + flush_interval_seconds=60.0, + ) + + append_result = await batcher.append_terminal_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.failed"}\n\n', + max_bytes=1024, + state="failed", + response_id="resp-terminal-recovery", + ) + assert append_result.persisted is False + assert append_result.settlement_required is True + await batcher.settle_terminal_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-terminal-recovery", + response_id="resp-terminal-recovery", + ) + await asyncio.wait_for(settlement_finished.wait(), timeout=1.0) + + recovery = DurableBridgeSessionCoordinator(async_session_factory) + observed = await recovery.get_operation_by_fingerprint(request_fingerprint=fingerprint) + assert observed is not None + assert observed.operation_id == operation_id + assert observed.session_id == claim.id + assert observed.account_id == "account-terminal-recovery" + assert observed.state == "failed" + assert observed.event_spool_complete is False + assert await recovery.get_operation_events(operation_id=operation_id) == [ + 'data: {"type":"response.created"}\n\n', + 'data: {"type":"response.failed"}\n\n', + ] + + retry = await recovery.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-terminal-recovery", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert retry is not None + assert retry.state == "submitted" + await batcher.settle_terminal_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-terminal-recovery", + response_id="resp-terminal-recovery", + ) + after_stale_settlement = await recovery.get_operation(operation_id=operation_id) + assert after_stale_settlement is not None + assert after_stale_settlement.state == "submitted" + assert after_stale_settlement.response_id is None + await batcher.close() + + +@pytest.mark.asyncio +async def test_consumed_recovery_checkpoint_does_not_rebind_failed_operation( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + original = await _claim( + repository, + instance_id="inst-consumed-original", + session_key_value="sid-consumed-original", + ) + replacement = await _claim( + repository, + instance_id="inst-consumed-replacement", + session_key_value="sid-consumed-replacement", + ) + fingerprint = durable_bridge_hash("consumed-failed-operation") + operation_id = durable_bridge_operation_id(original.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=original.id, + instance_id="inst-consumed-original", + owner_epoch=original.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-consumed", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + assert await repository.append_terminal_operation_event( + operation_id=operation_id, + session_id=original.id, + instance_id="inst-consumed-original", + owner_epoch=original.owner_epoch, + event_text='data: {"type":"response.failed"}\n\n', + max_bytes=1024, + state="failed", + ) + + existing = await repository.record_operation( + operation_id=operation_id, + session_id=replacement.id, + instance_id="inst-consumed-replacement", + owner_epoch=replacement.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-replacement", + model="gpt-5.6", + parent_response_id="resp-parent", + recovery_attempt_consumed=True, + ) + + assert existing is not None + assert existing.created is False + assert existing.session_id == original.id + assert existing.state == "failed" + persisted = await repository.get_operation(operation_id=operation_id) + assert persisted is not None + assert persisted.session_id == original.id + assert persisted.state == "failed" + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_unknown_operation_recovery_claim_is_atomic_and_single_use( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-operation-claim", session_key_value="sid-operation-claim") + fingerprint = durable_bridge_hash("continuation-claim") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-claim", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + assert await repository.append_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-claim", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.output_text.delta"}\n\n', + max_bytes=1024, + ) + assert await repository.mark_operation_unknown( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-claim", + owner_epoch=claim.owner_epoch, + ) + + assert await repository.claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-claim", + owner_epoch=claim.owner_epoch, + ) + claimed = await repository.get_operation(operation_id=operation_id) + assert claimed is not None + assert claimed.state == "submitted" + assert claimed.response_id is None + assert claimed.event_spool_complete is False + assert await repository.get_operation_events(operation_id=operation_id) == [] + + # The state transition is the claim: a concurrent reconnect that gets + # the write lock later cannot reset and submit the same operation. + assert not await repository.claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-claim", + owner_epoch=claim.owner_epoch, + ) + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_one_shot_recovery_budget_survives_unknown_reset( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim( + repository, + instance_id="inst-operation-one-shot", + session_key_value="sid-operation-one-shot", + ) + fingerprint = durable_bridge_hash("continuation-one-shot") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-one-shot", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + assert await repository.mark_operation_unknown( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-one-shot", + owner_epoch=claim.owner_epoch, + ) + + assert await repository.claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-one-shot", + owner_epoch=claim.owner_epoch, + max_recovery_dispatches=1, + ) + # A failed or ambiguous dispatch may return the operation to UNKNOWN, + # but that must not refund the durable one-shot recovery budget. + assert await repository.mark_operation_unknown( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-one-shot", + owner_epoch=claim.owner_epoch, + ) + assert not await repository.claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-one-shot", + owner_epoch=claim.owner_epoch, + max_recovery_dispatches=1, + ) + persisted = await repository.get_operation(operation_id=operation_id) + assert persisted is not None + assert persisted.state == "unknown" + assert persisted.recovery_dispatch_count == 1 + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_pre_dispatch_recovery_claim_restores_one_shot_budget( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim( + repository, + instance_id="inst-operation-refund", + session_key_value="sid-operation-refund", + ) + fingerprint = durable_bridge_hash("continuation-refund") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-refund", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + assert await repository.mark_operation_unknown( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-refund", + owner_epoch=claim.owner_epoch, + ) + assert await repository.claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-refund", + owner_epoch=claim.owner_epoch, + max_recovery_dispatches=1, + ) + + # A cancellation before send_text() is proven pre-dispatch and must + # refund the claim so the next reconnect can make the one safe retry. + assert await repository.mark_operation_unknown( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-refund", + owner_epoch=claim.owner_epoch, + restore_recovery_dispatch_claim=True, + ) + assert await repository.claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-refund", + owner_epoch=claim.owner_epoch, + max_recovery_dispatches=1, + ) + persisted = await repository.get_operation(operation_id=operation_id) + assert persisted is not None + assert persisted.recovery_dispatch_count == 1 + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_pre_dispatch_operation_rollback_removes_only_empty_new_row( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim( + repository, + instance_id="inst-operation-rollback", + session_key_value="sid-operation-rollback", + ) + fingerprint = durable_bridge_hash("operation-rollback") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-rollback", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None and operation.created is True + assert await repository.rollback_operation_before_dispatch( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-rollback", + owner_epoch=claim.owner_epoch, + ) + assert await repository.get_operation(operation_id=operation_id) is None + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_operation_spool_purge_expires_stale_nonterminal_rows( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-stale-operation", session_key_value="sid-stale-operation") + fingerprint = durable_bridge_hash("stale-operation") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + assert await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-stale-operation", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id=None, + request_text='{"input":"stale"}', + ) + stale_at = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=8) + assert await repository.update_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-stale-operation", + owner_epoch=claim.owner_epoch, + state="unknown", + ) + await session.execute( + update(HttpBridgeOperationRecord) + .where(HttpBridgeOperationRecord.operation_id == operation_id) + .values(updated_at=stale_at) + ) + await session.commit() + + # A stale timestamp alone must not delete an UNKNOWN operation whose + # session is still owned and leased; it may be a long-running recovery + # request whose duplicate-suppression fence must remain intact. + assert await repository.purge_operation_spool(cutoff=datetime.now(timezone.utc).replace(tzinfo=None)) == 0 + await session.execute( + update(HttpBridgeSessionRecord) + .where(HttpBridgeSessionRecord.id == claim.id) + .values(owner_instance_id=None, lease_expires_at=None) + ) + await session.commit() + assert await repository.purge_operation_spool(cutoff=datetime.now(timezone.utc).replace(tzinfo=None)) == 1 + assert await repository.get_operation(operation_id=operation_id) is None + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_nonterminal_operation_rebinds_before_cross_session_recovery_reset( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + original = await _claim( + repository, + instance_id="inst-original-operation", + session_key_value="sid-original-operation", + ) + replacement = await _claim( + repository, + instance_id="inst-replacement-operation", + session_key_value="sid-replacement-operation", + ) + fingerprint = durable_bridge_hash("cross-session-operation") + operation_id = durable_bridge_operation_id(original.id, fingerprint) + assert await repository.record_operation( + operation_id=operation_id, + session_id=original.id, + instance_id="inst-original-operation", + owner_epoch=original.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + request_text='{"input":"cross-session"}', + ) + await session.execute( + update(HttpBridgeSessionRecord) + .where(HttpBridgeSessionRecord.id == original.id) + .values(owner_instance_id=None, lease_expires_at=None) + ) + await session.commit() + rebound = await repository.record_operation( + operation_id=operation_id, + session_id=replacement.id, + instance_id="inst-replacement-operation", + owner_epoch=replacement.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-replacement", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert rebound is not None + assert rebound.session_id == replacement.id + assert await repository.reset_operation_event_spool( + operation_id=operation_id, + session_id=replacement.id, + instance_id="inst-replacement-operation", + owner_epoch=replacement.owner_epoch, + ) + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_nonterminal_operation_does_not_rebind_from_live_prior_owner( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + original = await _claim( + repository, + instance_id="inst-live-original-operation", + session_key_value="sid-live-original-operation", + ) + replacement = await _claim( + repository, + instance_id="inst-live-replacement-operation", + session_key_value="sid-live-replacement-operation", + ) + fingerprint = durable_bridge_hash("live-cross-session-operation") + operation_id = durable_bridge_operation_id(original.id, fingerprint) + assert await repository.record_operation( + operation_id=operation_id, + session_id=original.id, + instance_id="inst-live-original-operation", + owner_epoch=original.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + + existing = await repository.record_operation( + operation_id=operation_id, + session_id=replacement.id, + instance_id="inst-live-replacement-operation", + owner_epoch=replacement.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-replacement", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + + assert existing is not None + assert existing.session_id == original.id + persisted = await repository.get_operation(operation_id=operation_id) + assert persisted is not None + assert persisted.session_id == original.id + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_recovery_handoff_rebinds_operation_while_origin_journal_stays_fenced( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + instance_id = "inst-recovery-handoff" + original = await _claim( + repository, + instance_id=instance_id, + session_key_value="sid-recovery-origin", + ) + replacement = await _claim( + repository, + instance_id=instance_id, + session_key_value="sid-recovery-replacement", + ) + operation_fingerprint = durable_bridge_hash("recovery-handoff-operation") + operation_id = durable_bridge_operation_id(original.id, operation_fingerprint) + assert await repository.record_operation( + operation_id=operation_id, + session_id=original.id, + instance_id=instance_id, + owner_epoch=original.owner_epoch, + request_fingerprint=operation_fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + recovery_fingerprint = durable_bridge_hash("recovery-handoff-request") + attempt = await repository.record_recovery_attempt( + session_id=original.id, + instance_id=instance_id, + owner_epoch=original.owner_epoch, + request_fingerprint=recovery_fingerprint, + request_id="request-recovery-handoff", + account_id="account-operation", + model="gpt-5.6", + replay_safe=True, + ) + assert attempt is not None + assert await repository.mark_recovery_attempt_replayed( + session_id=original.id, + instance_id=instance_id, + owner_epoch=original.owner_epoch, + request_fingerprint=recovery_fingerprint, + ) + + rebound = await repository.record_operation( + operation_id=operation_id, + session_id=replacement.id, + instance_id=instance_id, + owner_epoch=replacement.owner_epoch, + request_fingerprint=operation_fingerprint, + account_id="account-replacement", + model="gpt-5.6", + parent_response_id="resp-parent", + recovery_attempt_session_id=original.id, + recovery_attempt_owner_epoch=original.owner_epoch, + recovery_attempt_fingerprint=recovery_fingerprint, + ) + assert rebound is not None + assert rebound.session_id == replacement.id + origin = await repository.get_session_by_id(original.id) + assert origin is not None + assert origin.owner_instance_id == instance_id + assert await repository.rollback_recovery_attempt_replayed( + session_id=original.id, + instance_id=instance_id, + owner_epoch=original.owner_epoch, + request_fingerprint=recovery_fingerprint, + ) + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_startup_retains_completed_operation_session( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-completed-retain", session_key_value="sid-completed-retain") + fingerprint = durable_bridge_hash("completed-retain") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + assert await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-completed-retain", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert await repository.update_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-completed-retain", + owner_epoch=claim.owner_epoch, + state="completed", + response_id="resp-completed", + ) + assert await repository.purge_owned_sessions_on_startup(instance_id="inst-completed-retain") == 0 + retained = await repository.get_operation(operation_id=operation_id) + assert retained is not None + owner = await repository.get_session_by_id(claim.id) + assert owner is not None + assert owner.owner_instance_id is None + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_startup_retains_completed_operation_session_across_process_epoch( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-epoch-retain", session_key_value="sid-epoch-retain") + fingerprint = durable_bridge_hash("epoch-retain") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + assert await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-epoch-retain", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert await repository.update_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-epoch-retain", + owner_epoch=claim.owner_epoch, + state="completed", + response_id="resp-completed", + ) + assert ( + await repository.purge_owned_sessions_on_startup( + instance_id="inst-epoch-retain", + owner_process_epoch="new-process", + ) + == 0 + ) + owner = await repository.get_session_by_id(claim.id) + assert owner is not None + assert owner.owner_instance_id is None + assert owner.owner_process_epoch == "test-process" finally: await session.close() @@ -1138,3 +2264,65 @@ def test_durable_lookup_allows_turn_state_takeover_requires_inactive_lease() -> assert allows(expired_draining) is True assert allows(released_draining) is True assert allows(closed) is True + assert _http_bridge_allow_durable_takeover(live_draining) is False + assert _http_bridge_allow_durable_takeover(expired_draining) is True + assert _http_bridge_allow_durable_takeover(released_draining) is True + assert _http_bridge_allow_durable_takeover(closed) is True + assert _http_bridge_claim_allows_takeover(live_draining, force=True) is False + assert _http_bridge_claim_allows_takeover(expired_draining, force=True) is True + assert _http_bridge_claim_allows_takeover(released_draining, force=True) is True + assert _http_bridge_claim_allows_takeover(closed, force=True) is True + live_active = _durable_lookup( + session_id="sess-5", + owner_instance_id="instance-b", + owner_epoch=1, + state=HttpBridgeSessionState.ACTIVE, + lease_seconds_from_now=60.0, + ) + assert _http_bridge_claim_allows_takeover(live_active, force=True) is True + assert _http_bridge_claim_allows_takeover(live_active, force=False) is False + + +@pytest.mark.asyncio +async def test_claim_does_not_retry_takeover_against_a_live_foreign_owner( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The repository drops takeover permission after losing a claim race, but + the service's own retry loop issues a fresh claim — which would restore the + permission and steal the winner's live lease. A live foreign owner must end + the retry so the 409 'retry to reach the correct replica' response stands + (issue #1695).""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + monkeypatch.setattr(proxy_service, "get_settings", _make_app_settings) + session = _make_bridge_session(key_value="sid-foreign-live") + session.account = cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE, plan_type="plus")) + + claims: list[bool] = [] + + async def claim_live_session(*, allow_takeover, **kwargs): + claims.append(allow_takeover) + # A live foreign owner: lease well in the future, ACTIVE. + return SimpleNamespace( + session_id="durable-foreign", + owner_instance_id="instance-other", + owner_epoch=7, + lease_expires_at=utcnow() + timedelta(seconds=600), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state=None, + latest_response_id=None, + canonical_kind=session.key.affinity_kind, + canonical_key=session.key.affinity_key, + account_id="acc-1", + api_key_scope="__anonymous__", + lease_is_active=lambda now: True, + ) + + service._durable_bridge = cast(Any, SimpleNamespace(claim_live_session=claim_live_session)) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._claim_durable_http_bridge_session(session, allow_takeover=True) + + assert exc_info.value.status_code == 409 + # Exactly one claim: the live foreign owner ends the retry instead of + # issuing a second, permission-restoring claim. + assert claims == [True] diff --git a/tests/unit/test_chat_request_mapping.py b/tests/unit/test_chat_request_mapping.py index 9fcc7b11f1..e5a4f589e7 100644 --- a/tests/unit/test_chat_request_mapping.py +++ b/tests/unit/test_chat_request_mapping.py @@ -10,6 +10,50 @@ from app.core.types import JsonValue +def test_chat_to_responses_omits_unset_tools() -> None: + req = ChatCompletionsRequest.model_validate( + { + "model": "gpt-5.2", + "messages": [{"role": "user", "content": "hi"}], + } + ) + + responses = req.to_responses_request() + + assert "tools" not in req.model_fields_set + assert "tools" not in responses.model_fields_set + assert "tools" not in responses.to_payload() + + +def test_chat_to_responses_preserves_explicit_empty_tools() -> None: + req = ChatCompletionsRequest.model_validate( + { + "model": "gpt-5.2", + "messages": [{"role": "user", "content": "hi"}], + "tools": [], + } + ) + + responses = req.to_responses_request() + + assert responses.to_payload()["tools"] == [] + + +def test_chat_responses_shaped_payload_omits_unset_tools() -> None: + req = ChatCompletionsRequest.model_validate( + { + "model": "gpt-5.2", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}], + } + ) + + responses = req.to_responses_request() + + assert "tools" not in req.model_fields_set + assert "tools" not in responses.model_fields_set + assert "tools" not in responses.to_payload() + + def test_chat_messages_to_responses_mapping(): payload = { "model": "gpt-5.2", @@ -315,6 +359,23 @@ def test_chat_reasoning_effort_maps_to_responses_reasoning(): assert reasoning_map.get("effort") == "high" +def test_chat_reasoning_effort_merges_with_reasoning_metadata(): + request = ChatCompletionsRequest.model_validate( + { + "model": "gpt-5.2", + "messages": [{"role": "user", "content": "hi"}], + "reasoning_effort": "max", + "reasoning": {"summary": "auto"}, + } + ) + + responses = request.to_responses_request() + + assert responses.reasoning is not None + assert responses.reasoning.effort == "max" + assert responses.reasoning.summary == "auto" + + def test_chat_enable_thinking_maps_to_default_reasoning_effort(): payload = { "model": "gpt-5.2", diff --git a/tests/unit/test_chat_response_mapping.py b/tests/unit/test_chat_response_mapping.py index 2935f0762d..0c32197214 100644 --- a/tests/unit/test_chat_response_mapping.py +++ b/tests/unit/test_chat_response_mapping.py @@ -74,6 +74,47 @@ def test_error_event_emits_done_chunk(): assert chunks[-1].strip() == "data: [DONE]" +@pytest.mark.parametrize( + "event_line", + [ + 'data: {"type":"response.failed","response":{"id":"r1","status":"failed"}}\n\n', + 'data: {"type":"response.failed","response":{"error":{}}}\n\n', + 'data: {"type":"error"}\n\n', + 'data: {"type":"error","error":{}}\n\n', + ], +) +@pytest.mark.asyncio +async def test_stream_chat_chunks_preserves_terminal_error_without_payload(event_line: str): + async def _stream(): + yield event_line + + chunks = [chunk async for chunk in stream_chat_chunks(_stream(), model="gpt-5.2")] + + error_payload = json.loads(chunks[-2][5:].strip()) + assert error_payload["error"] == { + "message": "Upstream error", + "type": "server_error", + "code": "upstream_error", + } + assert chunks[-1].strip() == "data: [DONE]" + + +@pytest.mark.asyncio +async def test_stream_chat_chunks_emits_error_and_done_when_upstream_ends_without_terminal_event(): + # #given + async def _stream(): + yield 'data: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + # #when + chunks = [chunk async for chunk in stream_chat_chunks(_stream(), model="gpt-5.2")] + + # #then + assert chunks[-1].strip() == "data: [DONE]" + error_chunk = json.loads(chunks[-2][5:].strip()) + assert error_chunk["error"]["code"] == "upstream_stream_truncated" + assert error_chunk["error"]["type"] == "server_error" + + @pytest.mark.asyncio async def test_collect_completion_parses_event_prefixed_sse_block(): lines = [ @@ -94,6 +135,22 @@ async def _stream(): assert result.error.code == "no_accounts" +@pytest.mark.asyncio +async def test_collect_chat_completion_returns_error_when_upstream_ends_without_terminal_event(): + # #given + async def _stream(): + yield 'data: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + # #when + result = await collect_chat_completion(_stream(), model="gpt-5.2") + + # #then + assert isinstance(result, OpenAIErrorEnvelope) + assert result.error is not None + assert result.error.code == "upstream_stream_truncated" + assert result.error.type == "server_error" + + def test_tool_call_delta_is_emitted(): lines = [ ( @@ -428,6 +485,61 @@ async def _stream(): assert result.error.code == "no_accounts" +@pytest.mark.asyncio +async def test_collect_completion_drains_after_first_failed_event(): + # #given + closed = {"value": False} + + async def _stream(): + try: + yield ( + 'data: {"type":"response.failed","response":{"error":' + '{"message":"limit","type":"rate_limit_error","code":"rate_limit_exceeded"}}}\n\n' + ) + yield 'data: {"type":"response.completed","response":{"id":"should-not-win"}}\n\n' + finally: + closed["value"] = True + + # #when + result = await collect_chat_completion(_stream(), model="gpt-5.2") + + # #then + assert isinstance(result, OpenAIErrorEnvelope) + assert result.error is not None + assert result.error.code == "rate_limit_exceeded" + assert closed["value"] is True + + +@pytest.mark.asyncio +async def test_collect_completion_drains_original_after_anext_split(): + # #given + from app.modules.proxy.api import _prepend_first + + closed = {"value": False} + + async def _stream(): + try: + yield ( + 'data: {"type":"response.failed","response":{"error":' + '{"message":"limit","type":"rate_limit_error","code":"rate_limit_exceeded"}}}\n\n' + ) + yield 'data: {"type":"response.completed","response":{"id":"tail"}}\n\n' + finally: + closed["value"] = True + + stream = _stream() + first = await stream.__anext__() + + # #when + result = await collect_chat_completion(_prepend_first(first, stream), model="gpt-5.2") + + # #then + assert isinstance(result, OpenAIErrorEnvelope) + assert result.error is not None + assert result.error.code == "rate_limit_exceeded" + assert closed["value"] is True + + @pytest.mark.asyncio async def test_collect_completion_includes_refusal_delta(): lines = [ diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 2415d82f4e..d70119a867 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -1,5 +1,6 @@ from __future__ import annotations +import builtins import json import logging import sqlite3 @@ -252,11 +253,14 @@ def run(self) -> None: cli._run_server("app.main:app", host="127.0.0.1", port=2455) + from app.core.http_protocol_httptools import UpgradeTolerantHttpToolsProtocol + assert captured["config_args"] == ("app.main:app",) assert captured["config_kwargs"] == { "host": "127.0.0.1", "port": 2455, "workers": 1, + "http": UpgradeTolerantHttpToolsProtocol, "timeout_graceful_shutdown": 17, } assert captured["drain_timeout_seconds"] == 17 @@ -264,6 +268,24 @@ def run(self) -> None: assert captured["ran"] is True +def test_load_http_protocol_class_falls_back_to_h11_without_httptools( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from app.core.http_protocol import UpgradeTolerantH11Protocol + + real_import = builtins.__import__ + + def fail_httptools_import(name: str, *args: Any, **kwargs: Any) -> object: + if name in {"httptools", "app.core.http_protocol_httptools"}: + raise ImportError(name) + return real_import(name, *args, **kwargs) + + monkeypatch.delitem(sys.modules, "app.core.http_protocol_httptools", raising=False) + monkeypatch.setattr(builtins, "__import__", fail_httptools_import) + + assert cli._load_http_protocol_class() is UpgradeTolerantH11Protocol + + def test_run_server_pins_one_worker_despite_ambient_web_concurrency( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_codex_review_wrapper.py b/tests/unit/test_codex_review_wrapper.py new file mode 100644 index 0000000000..4a84a554e0 --- /dev/null +++ b/tests/unit/test_codex_review_wrapper.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +_REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +_WRAPPER = _REPOSITORY_ROOT / ".agents/skills/codex-review-loop/scripts/codex-subagent.sh" + + +@pytest.mark.parametrize( + ("review_target", "expected_target"), + [ + pytest.param(("--base", "origin/main"), ["--base", "origin/main"], id="base"), + pytest.param(("--commit", "deadbeef"), ["--commit", "deadbeef"], id="commit"), + pytest.param(("--uncommitted",), ["--uncommitted"], id="uncommitted"), + ], +) +def test_codex_review_wrapper_forwards_supported_arguments_without_removed_flags( + tmp_path: Path, + review_target: tuple[str, ...], + expected_target: list[str], +) -> None: + mock_bin = tmp_path / "bin" + mock_bin.mkdir() + args_path = tmp_path / "codex-args" + mock_codex = mock_bin / "codex" + mock_codex.write_text( + "#!/usr/bin/env bash\nprintf '%s\\0' \"$@\" > \"$MOCK_CODEX_ARGS_PATH\"\nprintf 'codex\\nreview clean\\n'\n", + encoding="utf-8", + ) + mock_codex.chmod(0o755) + env = os.environ.copy() + env.update( + { + "CODEX_REVIEW_MODEL": "review-model", + "CODEX_REVIEW_REASONING": "high", + "MOCK_CODEX_ARGS_PATH": str(args_path), + "PATH": f"{mock_bin}{os.pathsep}{env['PATH']}", + } + ) + + result = subprocess.run( + [str(_WRAPPER), *review_target], + check=False, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + env=env, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout == "review clean\n" + forwarded_args = args_path.read_bytes().rstrip(b"\0").decode().split("\0") + assert forwarded_args == [ + "exec", + "review", + *expected_target, + "-m", + "review-model", + "-c", + 'model_reasoning_effort="high"', + ] + assert "--ephemeral" not in forwarded_args + assert "--full-auto" not in forwarded_args diff --git a/tests/unit/test_codex_upstream_paths.py b/tests/unit/test_codex_upstream_paths.py index 8375996c30..9c8d627799 100644 --- a/tests/unit/test_codex_upstream_paths.py +++ b/tests/unit/test_codex_upstream_paths.py @@ -2,7 +2,6 @@ import errno import socket -from types import SimpleNamespace from typing import Any, cast from unittest.mock import AsyncMock @@ -11,7 +10,7 @@ from aiohttp.client_reqrep import ConnectionKey import app.core.clients.proxy as proxy_module -from app.core.clients.codex import CodexClient, CodexTransportError, CodexWebSocketResult +from app.core.clients.codex import CodexClient, CodexRequestResult, CodexTransportError, CodexWebSocketResult from app.core.clients.files import create_file, finalize_file from app.core.clients.proxy import ( ProxyResponseError, @@ -23,12 +22,7 @@ thread_goal_request, transcribe_audio, ) -from app.core.clients.proxy_websocket import ( - UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - UpstreamWebSocketTransportError, - WebsocketsUpstreamWebSocket, - connect_responses_websocket, -) +from app.core.clients.proxy_websocket import UpstreamWebSocketTransportError, connect_responses_websocket from app.core.openai.requests import ResponsesCompactRequest, ResponsesRequest from app.core.upstream_proxy import ResolvedProxyEndpoint, ResolvedUpstreamRoute from tests.unit._proxy_test_helpers import runtime_basic_auth_url @@ -46,6 +40,19 @@ async def request(self, method: str, url: str, *, route: ResolvedUpstreamRoute, return self.response +class _RouteMetadataCodexClient(_CodexClient): + async def request_with_route_metadata( + self, + method: str, + url: str, + *, + route: ResolvedUpstreamRoute, + **kwargs: Any, + ) -> CodexRequestResult: + self.calls.append({"method": method, "url": url, "route": route, **kwargs}) + return CodexRequestResult(response=self.response, route=route, fallback_used=False) + + class _FailingRouteMetadataCodexClient: async def request_with_route_metadata( self, @@ -77,6 +84,123 @@ def json(self) -> dict[str, str]: return {"object": "response.compact", "id": "compact_1"} +class _CompactStreamContent: + async def iter_chunked(self, size: int): + del size + yield ( + b'data: {"type":"response.output_item.done","output_index":0,' + b'"item":{"id":"msg_compact_1","type":"message","role":"assistant",' + b'"status":"completed","content":[{"type":"output_text","text":"enc_compact_1"}]}}\n\n' + b'data: {"type":"response.completed","response":' + b'{"object":"response","id":"resp_compact_1","status":"completed","output":[]}}\n\n' + ) + + +class _CompactStreamWithoutOutputIndexContent: + async def iter_chunked(self, size: int): + del size + yield ( + b'data: {"type":"response.output_item.done",' + b'"item":{"id":"msg_compact_without_index","type":"message",' + b'"status":"completed","content":[{"type":"output_text",' + b'"text":"enc_compact_without_index"}]}}\n\n' + b'data: {"type":"response.completed","response":' + b'{"object":"response","id":"resp_compact_without_index",' + b'"status":"completed","output":[]}}\n\n' + ) + + +class _CompactStreamResponse: + status_code = 200 + headers: dict[str, str] = {} + content = _CompactStreamContent() + + +class _CompactStreamWithoutOutputIndexResponse: + status_code = 200 + headers: dict[str, str] = {} + content = _CompactStreamWithoutOutputIndexContent() + + +class _BufferedCompactStreamResponse: + status = 200 + status_code = 200 + headers = {"content-type": "text/event-stream"} + content = ( + b'data: {"type":"response.completed","response":{"object":"response","id":"resp_compact_buffered",' + b'"status":"completed","service_tier":"default","output":[' + b'{"id":"msg_history","type":"message","role":"assistant","status":"completed",' + b'"content":[{"type":"output_text","text":"historical plaintext"}]},' + b'{"id":"cmp_buffered","type":"compaction_summary","encrypted_content":"enc_buffered"}]}}\n\n' + ) + + +class _BufferedStrCompactStreamResponse: + status = 200 + status_code = 200 + headers = {"content-type": "text/event-stream"} + content = ( + 'data: {"type":"response.completed","response":{"object":"response","id":"resp_compact_str",' + '"status":"completed","output":[' + '{"id":"cmp_str","type":"compaction_summary","encrypted_content":"enc_str"}]}}\n\n' + ) + + +class _BufferedMessageOnlyCompactStreamResponse: + status = 200 + status_code = 200 + headers = {"content-type": "text/event-stream"} + content = ( + b'data: {"type":"response.completed","response":{"object":"response","id":"resp_compact_messages",' + b'"status":"completed","output":[' + b'{"id":"msg_history","type":"message","role":"assistant","status":"completed",' + b'"content":[{"type":"output_text","text":"historical plaintext"}]},' + b'{"id":"msg_summary","type":"message","role":"assistant","status":"completed",' + b'"content":[{"type":"output_text","text":"enc_summary"}]}]}}\n\n' + ) + + +class _CompactTerminalErrorStreamResponse: + status = 200 + status_code = 200 + headers = {"content-type": "text/event-stream"} + + def __init__(self, error_type: str, error_code: str) -> None: + self.content = ( + b'data: {"type":"error","error_type":"' + + error_type.encode("utf-8") + + b'","code":"' + + error_code.encode("utf-8") + + b'","message":"compact rejected","param":"previous_response_id"}\n\n' + ) + + +class _CompactTerminalFailedStreamResponse: + status = 200 + status_code = 200 + headers = {"content-type": "text/event-stream"} + content = ( + b'data: {"type":"response.failed","response":{"status_code":400,' + b'"error":{"code":"previous_response_not_found","message":"missing anchor",' + b'"type":"invalid_request_error","param":"previous_response_id"}}}\n\n' + ) + + +class _CompactTerminalFailedStreamWithoutStatusResponse: + status = 200 + status_code = 200 + headers = {"content-type": "text/event-stream"} + + def __init__(self, error_type: str, error_code: str) -> None: + self.content = ( + b'data: {"type":"response.failed","response":{"status":"failed","error":{"code":"' + + error_code.encode("utf-8") + + b'","message":"mapped from error detail","type":"' + + error_type.encode("utf-8") + + b'"}}}\n\n' + ) + + class _TranscribeResponse: status_code = 200 headers = {"content-type": "application/json"} @@ -307,7 +431,7 @@ async def test_codex_control_request_uses_codex_client_when_route_is_resolved(ro @pytest.mark.asyncio async def test_compact_responses_uses_codex_client_when_route_is_resolved(route: ResolvedUpstreamRoute) -> None: - client = _CodexClient(_CompactResponse()) + client = _CodexClient(_CompactStreamResponse()) trace = UpstreamProxyRouteTrace() payload = ResponsesCompactRequest(model="gpt-5.2", instructions="Summarize.", input="hello") @@ -322,14 +446,252 @@ async def test_compact_responses_uses_codex_client_when_route_is_resolved(route: route_trace=trace, ) - assert response.object == "response.compact" - assert response.id == "compact_1" - assert client.calls[0]["url"].endswith("/backend-api/codex/responses/compact") + assert response.object == "response.compaction" + assert response.id == "resp_compact_1" + assert response.model_extra is not None + assert response.model_extra["output"] == [ + {"type": "compaction", "status": "completed", "encrypted_content": "enc_compact_1"} + ] + assert client.calls[0]["url"].endswith("/backend-api/codex/responses") assert client.calls[0]["route"] is route assert client.calls[0]["json"]["model"] == "gpt-5.2" + assert client.calls[0]["json"]["store"] is False + assert client.calls[0]["json"]["stream"] is True + assert client.calls[0]["headers"]["Accept"] == "text/event-stream" assert trace.endpoint_id == "ep_1" +@pytest.mark.asyncio +async def test_compact_responses_recovers_terminal_item_without_output_index( + route: ResolvedUpstreamRoute, +) -> None: + client = _RouteMetadataCodexClient(_CompactStreamWithoutOutputIndexResponse()) + payload = ResponsesCompactRequest(model="gpt-5.2", instructions="Summarize.", input="hello") + + response = await compact_responses( + payload, + {"user-agent": "codex"}, + "access", + "chatgpt_account", + session=cast(Any, object()), + route=route, + codex_client=cast(Any, client), + ) + + assert response.model_extra is not None + assert response.model_extra["output"] == [ + { + "type": "compaction", + "status": "completed", + "encrypted_content": "enc_compact_without_index", + } + ] + + +@pytest.mark.asyncio +async def test_compact_responses_routed_buffered_sse_keeps_compact_protocol(route: ResolvedUpstreamRoute) -> None: + client = _RouteMetadataCodexClient(_BufferedCompactStreamResponse()) + payload = ResponsesCompactRequest(model="gpt-5.2", instructions="Summarize.", input="hello") + + response = await compact_responses( + payload, + {"user-agent": "codex"}, + "access", + "chatgpt_account", + session=cast(Any, object()), + route=route, + codex_client=cast(Any, client), + ) + + sent_input = client.calls[0]["json"]["input"] + assert sent_input[-1] == {"type": "compaction_trigger"} + assert sum(1 for item in sent_input if isinstance(item, dict) and item.get("type") == "compaction_trigger") == 1 + assert response.id == "resp_compact_buffered" + assert response.model_extra is not None + assert response.model_extra["service_tier"] == "default" + assert response.model_extra["output"] == [ + {"id": "cmp_buffered", "type": "compaction", "encrypted_content": "enc_buffered"} + ] + + +@pytest.mark.asyncio +async def test_compact_responses_routed_buffered_str_sse_body_keeps_compact_protocol( + route: ResolvedUpstreamRoute, +) -> None: + client = _RouteMetadataCodexClient(_BufferedStrCompactStreamResponse()) + payload = ResponsesCompactRequest(model="gpt-5.2", instructions="Summarize.", input="hello") + + response = await compact_responses( + payload, + {"user-agent": "codex"}, + "access", + "chatgpt_account", + session=cast(Any, object()), + route=route, + codex_client=cast(Any, client), + ) + + assert response.id == "resp_compact_str" + assert response.model_extra is not None + assert response.model_extra["output"] == [{"id": "cmp_str", "type": "compaction", "encrypted_content": "enc_str"}] + + +@pytest.mark.asyncio +async def test_compact_responses_message_fallback_selects_last_message( + route: ResolvedUpstreamRoute, +) -> None: + client = _RouteMetadataCodexClient(_BufferedMessageOnlyCompactStreamResponse()) + payload = ResponsesCompactRequest(model="gpt-5.2", instructions="Summarize.", input="hello") + + response = await compact_responses( + payload, + {"user-agent": "codex"}, + "access", + "chatgpt_account", + session=cast(Any, object()), + route=route, + codex_client=cast(Any, client), + ) + + assert response.id == "resp_compact_messages" + assert response.model_extra is not None + assert response.model_extra["output"] == [ + {"type": "compaction", "status": "completed", "encrypted_content": "enc_summary"} + ] + + +@pytest.mark.asyncio +async def test_compact_responses_routed_terminal_sse_error_keeps_openai_envelope( + route: ResolvedUpstreamRoute, +) -> None: + client = _RouteMetadataCodexClient(_CompactTerminalFailedStreamResponse()) + payload = ResponsesCompactRequest(model="gpt-5.2", instructions="Summarize.", input="hello") + + with pytest.raises(ProxyResponseError) as exc_info: + await compact_responses( + payload, + {"user-agent": "codex"}, + "access", + "chatgpt_account", + session=cast(Any, object()), + route=route, + codex_client=cast(Any, client), + ) + + error = exc_info.value.payload["error"] + assert error["code"] == "previous_response_not_found" + assert error["message"] == "missing anchor" + assert error["type"] == "invalid_request_error" + assert error["param"] == "previous_response_id" + assert exc_info.value.failure_phase == "upstream" + assert exc_info.value.upstream_status_code == 400 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("error_type", "error_code", "expected_status"), + [ + ("invalid_request_error", "invalid_request_error", 400), + ("authentication_error", "invalid_api_key", 401), + ("authentication_error", "invalid_authentication", 401), + ("authentication_error", "token_invalidated", 401), + ("rate_limit_error", "rate_limit_exceeded", 429), + ("server_error", "insufficient_quota", 429), + ], +) +async def test_compact_responses_terminal_sse_error_infers_status_from_error_detail( + route: ResolvedUpstreamRoute, + error_type: str, + error_code: str, + expected_status: int, +) -> None: + client = _RouteMetadataCodexClient(_CompactTerminalFailedStreamWithoutStatusResponse(error_type, error_code)) + payload = ResponsesCompactRequest(model="gpt-5.2", instructions="Summarize.", input="hello") + + with pytest.raises(ProxyResponseError) as exc_info: + await compact_responses( + payload, + {"user-agent": "codex"}, + "access", + "chatgpt_account", + session=cast(Any, object()), + route=route, + codex_client=cast(Any, client), + ) + + assert exc_info.value.status_code == expected_status + assert exc_info.value.upstream_status_code == expected_status + assert exc_info.value.payload["error"]["type"] == error_type + assert exc_info.value.payload["error"]["code"] == error_code + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("error_type", "error_code", "expected_status"), + [ + ("invalid_request_error", "invalid_request_error", 400), + ("rate_limit_error", "rate_limit_exceeded", 429), + ], +) +async def test_compact_responses_routed_top_level_sse_error_preserves_type( + route: ResolvedUpstreamRoute, + error_type: str, + error_code: str, + expected_status: int, +) -> None: + client = _RouteMetadataCodexClient(_CompactTerminalErrorStreamResponse(error_type, error_code)) + payload = ResponsesCompactRequest(model="gpt-5.2", instructions="Summarize.", input="hello") + + with pytest.raises(ProxyResponseError) as exc_info: + await compact_responses( + payload, + {"user-agent": "codex"}, + "access", + "chatgpt_account", + session=cast(Any, object()), + route=route, + codex_client=cast(Any, client), + ) + + assert exc_info.value.status_code == expected_status + error = exc_info.value.payload["error"] + assert error["type"] == error_type + assert error["code"] == error_code + assert error["message"] == "compact rejected" + assert error["param"] == "previous_response_id" + + +@pytest.mark.parametrize("error_type", [None, "", " ", 123]) +def test_compact_top_level_sse_error_type_uses_server_error_fallback( + error_type: object, +) -> None: + payload: dict[str, Any] = { + "type": "error", + "code": "upstream_error", + "message": "compact failed", + } + if error_type is not None: + payload["error_type"] = error_type + + detail = proxy_module._compact_sse_terminal_error_payload(payload, "error") + + assert detail["error"]["type"] == "server_error" + + +@pytest.mark.parametrize( + ("payload", "expected_status"), + [ + ({"type": "response.failed", "status_code": 200}, 502), + ({"type": "response.failed", "status_code": 599}, 599), + ], +) +def test_compact_sse_status_code_accepts_only_http_error_statuses( + payload: dict[str, Any], + expected_status: int, +) -> None: + assert proxy_module._compact_sse_terminal_status_code(payload) == expected_status + + @pytest.mark.asyncio async def test_compact_responses_uses_upstream_chatgpt_account_id_header(route: ResolvedUpstreamRoute) -> None: client = _CodexClient(_CompactResponse()) @@ -961,45 +1323,6 @@ async def test_responses_websocket_send_errors_do_not_expose_proxy_credentials( assert "proxy.test:8080" not in message -@pytest.mark.asyncio -async def test_routed_responses_websocket_proves_closed_before_send_without_invoking_transport( - route: ResolvedUpstreamRoute, -) -> None: - client = _WsCodexClient() - client.websocket.closed = True - - websocket = await connect_responses_websocket( - {"user-agent": "codex"}, - "access", - "chatgpt_account", - base_url="https://chatgpt.test/backend-api", - route=route, - codex_client=cast(Any, client), - ) - - with pytest.raises(UpstreamWebSocketTransportError) as exc_info: - await websocket.send_text('{"type":"response.create"}') - - assert exc_info.value.error_code == UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE - assert client.websocket.sent == [] - - -@pytest.mark.asyncio -async def test_direct_responses_websocket_proves_closed_before_send_without_invoking_transport() -> None: - transport_send = AsyncMock() - connection = SimpleNamespace( - state=SimpleNamespace(name="CLOSED"), - send=transport_send, - ) - websocket = WebsocketsUpstreamWebSocket(cast(Any, connection)) - - with pytest.raises(UpstreamWebSocketTransportError) as exc_info: - await websocket.send_text('{"type":"response.create"}') - - assert exc_info.value.error_code == UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE - transport_send.assert_not_awaited() - - @pytest.mark.asyncio async def test_responses_websocket_post_connect_network_failures_preserve_safe_code( route: ResolvedUpstreamRoute, diff --git a/tests/unit/test_dashboard_projection_history_cap.py b/tests/unit/test_dashboard_projection_history_cap.py new file mode 100644 index 0000000000..3719a04bd9 --- /dev/null +++ b/tests/unit/test_dashboard_projection_history_cap.py @@ -0,0 +1,91 @@ +"""The projections history fetch must request the per-account row cap. + +The cap is what keeps the PostgreSQL bulk read bounded on deployments where +live snapshot ingestion densifies ``usage_history``; losing the kwarg would +silently regress the read back to full-window row counts. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta +from typing import cast + +import pytest + +from app.db.models import UsageHistory +from app.modules.dashboard.repository import DashboardRepository +from app.modules.dashboard.service import ( + _PROJECTION_HISTORY_PER_ACCOUNT_ROW_CAP, + _load_projection_histories, +) + +pytestmark = pytest.mark.unit + + +class _RecordingRepo: + def __init__(self) -> None: + self.calls: list[dict] = [] + + async def bulk_usage_history_since( + self, + account_ids, + window, + since, + *, + cutoffs=None, + per_account_row_cap=None, + uncapped_recent_floor=None, + ): + self.calls.append( + { + "account_ids": list(account_ids), + "window": window, + "since": since, + "cutoffs": cutoffs, + "per_account_row_cap": per_account_row_cap, + "uncapped_recent_floor": uncapped_recent_floor, + } + ) + return {} + + +def _usage_entry(account_id: str, window: str, window_minutes: int, recorded_at: datetime) -> UsageHistory: + return UsageHistory( + id=1, + account_id=account_id, + used_percent=10.0, + window=window, + window_minutes=window_minutes, + recorded_at=recorded_at, + ) + + +@pytest.mark.asyncio +async def test_projection_history_fetch_passes_per_account_row_cap(): + now = datetime(2026, 8, 16, 12, 0, 0) + repo = _RecordingRepo() + primary_usage = { + "acc1": _usage_entry("acc1", "primary", 300, now - timedelta(minutes=1)), + } + secondary_usage = { + "acc1": _usage_entry("acc1", "secondary", 10080, now - timedelta(minutes=1)), + } + + await _load_projection_histories( + cast(DashboardRepository, repo), + primary_usage, + secondary_usage, + now, + smoothing_window_minutes=240, + ) + + assert len(repo.calls) == 2 + assert {call["window"] for call in repo.calls} == {"primary", "secondary"} + for call in repo.calls: + assert call["per_account_row_cap"] == _PROJECTION_HISTORY_PER_ACCOUNT_ROW_CAP + assert call["cutoffs"] is not None + # The weekly-pace smoothing mean weighs every in-window sample + # equally, so the fetch must exempt the configured smoothing window + # from the row cap; a write burst may otherwise out-write the cap and + # shift the smoothed schedule gap. + assert call["uncapped_recent_floor"] == now - timedelta(minutes=240) diff --git a/tests/unit/test_db_migrate.py b/tests/unit/test_db_migrate.py index 928d2b5cd0..e9c1cde9f8 100644 --- a/tests/unit/test_db_migrate.py +++ b/tests/unit/test_db_migrate.py @@ -2567,6 +2567,231 @@ def test_connection_request_kind_migration_is_additive_without_backfill(tmp_path engine.dispose() +def test_api_key_reasoning_policy_migration_round_trips_from_current_parent(tmp_path: Path) -> None: + from alembic.script import ScriptDirectory + + db_path = tmp_path / "api-key-reasoning-policy.db" + url = _db_url(db_path) + parent_revision = "20260816_000000_add_account_pending_deletion" + target_revision = "20260806_030000_add_api_key_allowed_reasoning_efforts" + + run_upgrade(url, parent_revision, bootstrap_legacy=False) + config = _build_alembic_config(url) + script_directory = ScriptDirectory.from_config(config) + assert script_directory.get_revision(target_revision).down_revision == parent_revision + # Assert reachability from the single head rather than "is the head": every + # later migration would otherwise have to edit this test. + heads = script_directory.get_heads() + assert len(heads) == 1 + assert target_revision in {revision.revision for revision in script_directory.iterate_revisions(heads[0], "base")} + + engine = create_engine(to_sync_database_url(url)) + try: + with engine.connect() as connection: + before = {column["name"] for column in inspect(connection).get_columns("api_keys")} + assert "allowed_reasoning_efforts" not in before + + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO api_keys " + "(id, name, key_hash, key_prefix, is_active, created_at) " + "VALUES (:id, :name, :key_hash, :key_prefix, :is_active, CURRENT_TIMESTAMP)" + ), + { + "id": "key_reasoning_policy_migration", + "name": "reasoning policy migration", + "key_hash": "hash_reasoning_policy_migration", + "key_prefix": "sk-migration", + "is_active": True, + }, + ) + + command.upgrade(config, target_revision) + with engine.connect() as connection: + inspector = inspect(connection) + columns = {column["name"] for column in inspector.get_columns("api_keys")} + assert "allowed_reasoning_efforts" in columns + assert "ck_api_keys_reasoning_policy_exclusive" in { + constraint["name"] + for constraint in inspector.get_check_constraints("api_keys") + if constraint.get("name") + } + assert ( + connection.execute( + text("SELECT allowed_reasoning_efforts FROM api_keys WHERE id = 'key_reasoning_policy_migration'") + ).scalar_one() + is None + ) + connection.execute( + text( + "UPDATE api_keys SET allowed_reasoning_efforts = :allowed " + "WHERE id = 'key_reasoning_policy_migration'" + ), + {"allowed": '["low"]'}, + ) + connection.commit() + assert ( + connection.execute( + text("SELECT key_hash FROM api_keys WHERE id = 'key_reasoning_policy_migration'") + ).scalar_one() + == "hash_reasoning_policy_migration" + ) + with pytest.raises(sa_exc.IntegrityError): + connection.execute( + text( + "UPDATE api_keys SET enforced_reasoning_effort = 'high' " + "WHERE id = 'key_reasoning_policy_migration'" + ) + ) + connection.rollback() + + command.downgrade(config, parent_revision) + with engine.connect() as connection: + columns = {column["name"] for column in inspect(connection).get_columns("api_keys")} + assert "allowed_reasoning_efforts" not in columns + assert ( + connection.execute( + text("SELECT key_hash FROM api_keys WHERE id = 'key_reasoning_policy_migration'") + ).scalar_one() + == "hash_reasoning_policy_migration" + ) + + command.upgrade(config, target_revision) + with engine.connect() as connection: + columns = {column["name"] for column in inspect(connection).get_columns("api_keys")} + assert "allowed_reasoning_efforts" in columns + assert ( + connection.execute( + text("SELECT allowed_reasoning_efforts FROM api_keys WHERE id = 'key_reasoning_policy_migration'") + ).scalar_one() + is None + ) + finally: + engine.dispose() + + +def test_http_bridge_operation_migrations_round_trip_existing_rows_and_rebuild_sqlite_defaults( + tmp_path: Path, +) -> None: + db_path = tmp_path / "http-bridge-operation-round-trip.db" + url = _db_url(db_path) + parent_revision = "20260804_000001_add_global_http_bridge_operation_fingerprint" + spool_revision = "20260805_000001_finalize_http_bridge_operation_spool" + + run_upgrade(url, parent_revision, bootstrap_legacy=False) + config = _build_alembic_config(url) + engine = create_engine(to_sync_database_url(url)) + try: + with engine.begin() as connection: + connection.execute( + text( + """ + INSERT INTO http_bridge_sessions ( + id, session_key_kind, session_key_value, session_key_hash, api_key_scope, + owner_epoch, state, last_seen_at, created_at, updated_at + ) + VALUES ( + 'migration-operation-session', 'session_header', 'migration-operation-key', + 'migration-operation-hash', '__anonymous__', 1, 'active', CURRENT_TIMESTAMP, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + """ + ) + ) + connection.execute( + text( + """ + INSERT INTO http_bridge_operations ( + operation_id, session_id, request_fingerprint, account_id, model, + parent_response_id, state, response_id + ) + VALUES ( + 'migration-operation', 'migration-operation-session', 'migration-fingerprint', + NULL, 'gpt-5.6', 'migration-parent', 'submitted', NULL + ) + """ + ) + ) + + command.upgrade(config, spool_revision) + with engine.connect() as connection: + inspector = inspect(connection) + operation_columns = {column["name"]: column for column in inspector.get_columns("http_bridge_operations")} + assert {"request_text", "event_bytes", "event_spool_complete"} <= operation_columns.keys() + row = connection.execute( + text( + """ + SELECT request_text, event_bytes, event_spool_complete + FROM http_bridge_operations + WHERE operation_id = 'migration-operation' + """ + ) + ).one() + assert row == (None, 0, False) + assert inspector.has_table("http_bridge_operation_events") + + command.downgrade(config, parent_revision) + with engine.connect() as connection: + inspector = inspect(connection) + assert inspector.has_table("http_bridge_operations") + assert not inspector.has_table("http_bridge_operation_events") + assert ( + connection.execute( + text( + "SELECT request_fingerprint FROM http_bridge_operations " + "WHERE operation_id = 'migration-operation'" + ) + ).scalar_one() + == "migration-fingerprint" + ) + + command.upgrade(config, "head") + with engine.connect() as connection: + inspector = inspect(connection) + operation_columns = {column["name"] for column in inspector.get_columns("http_bridge_operations")} + assert {"request_text", "event_bytes", "event_spool_complete"} <= operation_columns + assert ( + connection.execute( + text( + "SELECT event_spool_complete FROM http_bridge_operations " + "WHERE operation_id = 'migration-operation'" + ) + ).scalar_one() + == 0 + ) + assert inspector.has_table("http_bridge_operation_events") + finally: + engine.dispose() + + +def test_sticky_abandonment_scope_migration_is_additive_and_reversible(tmp_path: Path) -> None: + db_path = tmp_path / "sticky-abandonment-scope.db" + url = _db_url(db_path) + parent_revision = "20260813_000000_add_file_account_pins" + target_revision = "20260812_120000_add_sticky_abandonment_scope" + + run_upgrade(url, parent_revision, bootstrap_legacy=False) + config = _build_alembic_config(url) + engine = create_engine(to_sync_database_url(url)) + try: + with engine.connect() as connection: + columns = {column["name"] for column in inspect(connection).get_columns("sticky_sessions")} + assert "continuity_abandonment_scope" not in columns + + command.upgrade(config, target_revision) + with engine.connect() as connection: + columns = {column["name"]: column for column in inspect(connection).get_columns("sticky_sessions")} + assert columns["continuity_abandonment_scope"]["nullable"] is True + + command.downgrade(config, parent_revision) + with engine.connect() as connection: + columns = {column["name"] for column in inspect(connection).get_columns("sticky_sessions")} + assert "continuity_abandonment_scope" not in columns + finally: + engine.dispose() + + def test_check_schema_drift_detects_missing_dashboard_hot_path_indexes(tmp_path: Path) -> None: db_path = tmp_path / "missing-hot-path-indexes.db" url = _db_url(db_path) diff --git a/tests/unit/test_db_session.py b/tests/unit/test_db_session.py index a030c901a8..5af6d147e6 100644 --- a/tests/unit/test_db_session.py +++ b/tests/unit/test_db_session.py @@ -12,10 +12,10 @@ import pytest from sqlalchemy import event as sa_event -from sqlalchemy import text -from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError +from sqlalchemy import text as sa_text +from sqlalchemy.engine import make_url from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine -from sqlalchemy.pool import AsyncAdaptedQueuePool, NullPool +from sqlalchemy.pool import NullPool import app.db.session as session_module from app.db.models import Account, AccountStatus, Base @@ -31,7 +31,6 @@ class _FakeSettings: database_sqlite_pre_migrate_backup_enabled: bool = False database_sqlite_pre_migrate_backup_max_files: int = 5 database_sqlite_startup_check_mode: str = "quick" - database_postgres_schema: str | None = None database_migrations_fail_fast: bool = False @@ -331,27 +330,6 @@ def test_postgres_connect_args_pin_session_timezone_to_utc(monkeypatch) -> None: assert connect_args == {"server_settings": {"timezone": "UTC"}} -def test_postgres_connect_args_include_search_path_when_schema_is_configured(monkeypatch) -> None: - monkeypatch.delenv("CODEX_LB_TEST_DATABASE_URL", raising=False) - monkeypatch.setattr( - session_module, - "_settings", - _FakeSettings( - database_url="postgresql+asyncpg://u:p@h/db", - database_postgres_schema="codex_lb_prod", - ), - ) - - connect_args = session_module._postgres_async_connect_args("postgresql+asyncpg://u:p@h/db") - - assert connect_args == { - "server_settings": { - "timezone": "UTC", - "search_path": '"codex_lb_prod",public', - } - } - - def test_postgres_connect_args_pin_utc_and_keep_test_db_url_tuning(monkeypatch) -> None: monkeypatch.setenv("CODEX_LB_TEST_DATABASE_URL", "1") @@ -399,6 +377,54 @@ async def close(self) -> None: assert calls == ["rollback", "close"] +@pytest.mark.asyncio +async def test_close_session_outlives_caller_cancellation() -> None: + rollback_started = asyncio.Event() + rollback_release = asyncio.Event() + close_started = asyncio.Event() + close_release = asyncio.Event() + cleanup_done = asyncio.Event() + calls: list[str] = [] + + class FakeSession: + def in_transaction(self) -> bool: + return True + + async def rollback(self) -> None: + calls.append("rollback-start") + rollback_started.set() + await rollback_release.wait() + calls.append("rollback-end") + + async def close(self) -> None: + calls.append("close-start") + close_started.set() + await close_release.wait() + calls.append("close-end") + + async def run_cleanup() -> None: + try: + await session_module.close_session(cast(session_module.AsyncSession, FakeSession())) + finally: + cleanup_done.set() + + async with asyncio.TaskGroup() as group: + task = group.create_task(run_cleanup()) + await rollback_started.wait() + task.cancel() + await asyncio.sleep(0) + task.cancel() + await asyncio.sleep(0) + assert calls == ["rollback-start"] + assert not cleanup_done.is_set() + rollback_release.set() + await close_started.wait() + close_release.set() + + assert calls == ["rollback-start", "rollback-end", "close-start", "close-end"] + assert cleanup_done.is_set() + + @pytest.mark.asyncio async def test_detach_session_objects_keeps_loaded_fields_available_after_rollback() -> None: engine = create_async_engine("sqlite+aiosqlite:///:memory:") @@ -761,7 +787,7 @@ async def test_init_background_db_derives_postgres_pool_size_from_main_pool() -> if os.environ.get("CODEX_LB_TEST_DATABASE_URL"): assert isinstance(pool, NullPool) else: - assert cast(Any, pool).size() == 15 + assert cast(Any, pool).size() == 25 if session_module._background_engine is not None: await session_module._background_engine.dispose() @@ -793,135 +819,6 @@ async def test_get_background_session_falls_back_to_main_pool_when_not_initializ assert isinstance(session, session_module.AsyncSession) -@pytest.mark.asyncio -async def test_get_request_session_always_uses_main_pool(monkeypatch: pytest.MonkeyPatch) -> None: - main_session = object() - background_session = object() - closed: list[object] = [] - - monkeypatch.setattr(session_module, "SessionLocal", lambda: main_session) - monkeypatch.setattr(session_module, "_background_session_factory", lambda: background_session) - - async def _close_session(session: object) -> None: - closed.append(session) - - monkeypatch.setattr(session_module, "close_session", _close_session) - - async with session_module.get_request_session() as session: - assert session is main_session - assert session is not background_session - - assert closed == [main_session] - - -@pytest.mark.asyncio -async def test_get_request_session_returns_connection_after_cancellation(monkeypatch: pytest.MonkeyPatch) -> None: - events: list[str] = [] - - class FakeSession: - transaction_open = True - - def in_transaction(self) -> bool: - return self.transaction_open - - async def rollback(self) -> None: - events.append("rollback") - self.transaction_open = False - - async def close(self) -> None: - events.append("close") - - monkeypatch.setattr(session_module, "SessionLocal", FakeSession) - - with pytest.raises(asyncio.CancelledError): - async with session_module.get_request_session(): - raise asyncio.CancelledError - - assert events == ["rollback", "close"] - - -@pytest.mark.asyncio -async def test_get_request_session_returns_connection_after_pool_timeout(monkeypatch: pytest.MonkeyPatch) -> None: - events: list[str] = [] - - class FakeSession: - transaction_open = True - - def in_transaction(self) -> bool: - return self.transaction_open - - async def rollback(self) -> None: - events.append("rollback") - self.transaction_open = False - - async def close(self) -> None: - events.append("close") - - monkeypatch.setattr(session_module, "SessionLocal", FakeSession) - - with pytest.raises(SQLAlchemyTimeoutError, match="private topology"): - async with session_module.get_request_session(): - raise SQLAlchemyTimeoutError("private topology") - - assert events == ["rollback", "close"] - - -@pytest.mark.asyncio -async def test_request_pool_progresses_while_real_background_queue_pool_is_saturated( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - request_engine = create_async_engine( - f"sqlite+aiosqlite:///{tmp_path / 'request-pool.db'}", - pool_size=1, - max_overflow=0, - pool_timeout=0.1, - ) - background_engine = create_async_engine( - f"sqlite+aiosqlite:///{tmp_path / 'background-pool.db'}", - pool_size=2, - max_overflow=1, - pool_timeout=0.1, - ) - request_factory = async_sessionmaker(request_engine, expire_on_commit=False) - background_factory = async_sessionmaker(background_engine, expire_on_commit=False) - request_pool = cast(AsyncAdaptedQueuePool, request_engine.sync_engine.pool) - background_pool = cast(AsyncAdaptedQueuePool, background_engine.sync_engine.pool) - monkeypatch.setattr(session_module, "SessionLocal", request_factory) - monkeypatch.setattr(session_module, "_background_session_factory", background_factory) - - release = asyncio.Event() - all_background_checked_out = asyncio.Event() - started = 0 - - async def hold_background_checkout() -> None: - nonlocal started - async with session_module.get_background_session() as session: - await session.execute(text("SELECT 1")) - started += 1 - if started == 3: - all_background_checked_out.set() - await release.wait() - - holders = [asyncio.create_task(hold_background_checkout()) for _ in range(3)] - try: - await asyncio.wait_for(all_background_checked_out.wait(), timeout=1.0) - assert background_pool.checkedout() == 3 - - async with session_module.get_request_session() as request_session: - result = await asyncio.wait_for(request_session.execute(text("SELECT 1")), timeout=0.5) - assert result.scalar_one() == 1 - assert request_pool.checkedout() == 1 - finally: - release.set() - await asyncio.gather(*holders, return_exceptions=True) - await request_engine.dispose() - await background_engine.dispose() - - assert background_pool.checkedout() == 0 - assert request_pool.checkedout() == 0 - - @pytest.mark.asyncio async def test_safe_close_outlives_caller_cancellation() -> None: started = asyncio.Event() @@ -1030,3 +927,786 @@ async def execute(self, statement: object) -> None: await session_module.relax_commit_durability(cast(session_module.AsyncSession, _FakeSession())) assert executed == ["SET LOCAL synchronous_commit = off"] + + +@pytest.mark.asyncio +async def test_sqlite_long_write_watchdog_reports_the_holder(tmp_path, monkeypatch, caplog) -> None: + """Issue #1682: a write transaction outliving the busy timeout is the + holder that makes every other writer surface 'database is locked'. The + watchdog must attribute it — duration, first/last write statement, task — + when it finally ends, since the stall self-recovers.""" + monkeypatch.setattr(session_module, "_SQLITE_LONG_WRITE_TRANSACTION_WARN_SECONDS", 0.0) + engine = create_async_engine( + f"sqlite+aiosqlite:///{tmp_path / 'watchdog.db'}", + poolclass=NullPool, + connect_args={"timeout": 5.0}, + ) + session_module._configure_sqlite_engine(engine.sync_engine, enable_wal=True) + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + caplog.clear() + + factory = async_sessionmaker(engine, expire_on_commit=False) + with caplog.at_level(logging.WARNING, logger=session_module.__name__): + async with factory() as session: + session.add( + Account( + id="acc-watchdog", + chatgpt_account_id="workspace-w", + email="watchdog@example.com", + plan_type="plus", + access_token_encrypted=b"a", + refresh_token_encrypted=b"r", + id_token_encrypted=b"i", + last_refresh=datetime(2025, 1, 1), + status=AccountStatus.ACTIVE, + ) + ) + await session.commit() + + records = [ + record + for record in caplog.records + if record.levelno == logging.WARNING and "sqlite_long_write_transaction" in record.getMessage() + ] + assert records, "the watchdog must report a write transaction over the threshold" + message = records[0].getMessage() + assert "outcome=commit" in message + assert "INSERT INTO accounts" in message + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_sqlite_long_write_watchdog_stays_silent_below_threshold_and_for_reads(tmp_path, caplog) -> None: + """Fast writes and read-only transactions (which never take the writer + slot in WAL) must not produce reports.""" + engine = create_async_engine( + f"sqlite+aiosqlite:///{tmp_path / 'quiet.db'}", + poolclass=NullPool, + connect_args={"timeout": 5.0}, + ) + session_module._configure_sqlite_engine(engine.sync_engine, enable_wal=True) + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + caplog.clear() + + factory = async_sessionmaker(engine, expire_on_commit=False) + with caplog.at_level(logging.WARNING, logger=session_module.__name__): + async with factory() as session: + (await session.execute(sa_text("SELECT count(*) FROM accounts"))).scalar_one() + await session.commit() + async with factory() as session: + await session.execute(sa_text("DELETE FROM accounts")) + await session.commit() + + assert not [ + record + for record in caplog.records + if record.levelno >= logging.WARNING and "sqlite_long_write_transaction" in record.getMessage() + ] + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_sqlite_long_write_watchdog_does_not_blame_a_victim_waiting_for_the_lock( + tmp_path, monkeypatch, caplog +) -> None: + """A write statement can spend the whole busy timeout waiting for the slot + and fail with 'database is locked'. That transaction never held the slot, + so its rollback must not be reported as the holder — the clock starts only + after the first write statement succeeds.""" + monkeypatch.setattr(session_module, "_SQLITE_LONG_WRITE_TRANSACTION_WARN_SECONDS", 0.2) + db_path = tmp_path / "victim.db" + holder_engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", poolclass=NullPool, connect_args={"timeout": 5.0} + ) + # Victim gets a short busy timeout so the test stays fast; install the + # watchdog directly because the pragma configurer would override the + # driver timeout with the production 30s busy_timeout. + victim_engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", poolclass=NullPool, connect_args={"timeout": 0.4} + ) + session_module._install_sqlite_long_write_watchdog(victim_engine.sync_engine) + try: + async with holder_engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + caplog.clear() + + holder_factory = async_sessionmaker(holder_engine, expire_on_commit=False) + victim_factory = async_sessionmaker(victim_engine, expire_on_commit=False) + with caplog.at_level(logging.WARNING, logger=session_module.__name__): + async with holder_factory() as holder_session: + # Journal mode: this write holds the database lock until commit. + await holder_session.execute(sa_text("DELETE FROM accounts")) + async with victim_factory() as victim_session: + with pytest.raises(Exception, match="database is locked"): + await victim_session.execute(sa_text("DELETE FROM accounts")) + await victim_session.rollback() + await holder_session.commit() + + blamed = [ + record + for record in caplog.records + if record.levelno >= logging.WARNING and "sqlite_long_write_transaction" in record.getMessage() + ] + assert not blamed, "the victim's busy-timeout wait must not be reported as a held slot" + finally: + await victim_engine.dispose() + await holder_engine.dispose() + + +@pytest.mark.asyncio +async def test_sqlite_long_write_watchdog_includes_a_slow_transaction_end_in_the_hold( + tmp_path, monkeypatch, caplog +) -> None: + """ConnectionEvents.commit/rollback fire before the DBAPI call, and a + wedged rollback is exactly the holder this watchdog hunts. The report is + deferred to the first proof the transaction ended (next begin on the + connection, or pool checkin), so the wedge itself is inside the measured + hold.""" + monkeypatch.setattr(session_module, "_SQLITE_LONG_WRITE_TRANSACTION_WARN_SECONDS", 0.15) + engine = create_async_engine( + f"sqlite+aiosqlite:///{tmp_path / 'slow-end.db'}", + poolclass=NullPool, + connect_args={"timeout": 5.0}, + ) + session_module._install_sqlite_long_write_watchdog(engine.sync_engine) + + # A commit whose DBAPI call itself stalls: the event fires, then the + # "driver" spends longer than the threshold before the transaction is over. + real_commit_events = [] + + @sa_event.listens_for(engine.sync_engine, "commit") + def _stall_after_mark(conn) -> None: + # Runs after the watchdog's own commit listener marked the pending + # report; the sleep stands in for a wedged DBAPI commit/rollback. + real_commit_events.append(True) + import time as _time + + _time.sleep(0.2) + + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + caplog.clear() + + factory = async_sessionmaker(engine, expire_on_commit=False) + with caplog.at_level(logging.WARNING, logger=session_module.__name__): + async with factory() as session: + await session.execute(sa_text("DELETE FROM accounts")) + await session.commit() + + records = [ + record + for record in caplog.records + if record.levelno == logging.WARNING and "sqlite_long_write_transaction" in record.getMessage() + ] + assert real_commit_events, "the stalling commit listener must have run" + assert records, "a hold whose transaction end itself stalls must still be reported" + assert "outcome=commit" in records[0].getMessage() + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_sqlite_long_write_watchdog_tracks_begin_immediate_holders(tmp_path, monkeypatch, caplog) -> None: + """BEGIN IMMEDIATE acquires the writer slot with no DML at all (the + accounts merge lock does exactly this), so a holder that never runs a + write statement must still be attributed.""" + monkeypatch.setattr(session_module, "_SQLITE_LONG_WRITE_TRANSACTION_WARN_SECONDS", 0.0) + engine = create_async_engine( + f"sqlite+aiosqlite:///{tmp_path / 'immediate.db'}", + poolclass=NullPool, + connect_args={"timeout": 5.0}, + ) + session_module._install_sqlite_long_write_watchdog(engine.sync_engine) + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + caplog.clear() + + factory = async_sessionmaker(engine, expire_on_commit=False) + with caplog.at_level(logging.WARNING, logger=session_module.__name__): + async with factory() as session: + await session.execute(sa_text("BEGIN IMMEDIATE")) + (await session.execute(sa_text("SELECT count(*) FROM accounts"))).scalar_one() + await session.commit() + + records = [ + record + for record in caplog.records + if record.levelno == logging.WARNING and "sqlite_long_write_transaction" in record.getMessage() + ] + assert records, "a BEGIN IMMEDIATE holder with no DML must still be attributed" + assert "BEGIN IMMEDIATE" in records[0].getMessage() + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_sqlite_long_write_watchdog_reports_a_failed_commit_as_rollback(tmp_path, monkeypatch, caplog) -> None: + """A commit whose DBAPI call raises is followed by a rollback; the report + must not claim a durable commit that never happened.""" + monkeypatch.setattr(session_module, "_SQLITE_LONG_WRITE_TRANSACTION_WARN_SECONDS", 0.0) + engine = create_async_engine( + f"sqlite+aiosqlite:///{tmp_path / 'failed-commit.db'}", + poolclass=NullPool, + connect_args={"timeout": 5.0}, + ) + session_module._install_sqlite_long_write_watchdog(engine.sync_engine) + + fail_next_commit = {"armed": False} + + from sqlalchemy.dialects.sqlite.aiosqlite import AsyncAdapt_aiosqlite_connection + + real_commit = AsyncAdapt_aiosqlite_connection.commit + + def failing_commit(self) -> None: + if fail_next_commit["armed"]: + fail_next_commit["armed"] = False + raise RuntimeError("simulated DBAPI commit failure") + real_commit(self) + + monkeypatch.setattr(AsyncAdapt_aiosqlite_connection, "commit", failing_commit) + + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + caplog.clear() + + factory = async_sessionmaker(engine, expire_on_commit=False) + with caplog.at_level(logging.WARNING, logger=session_module.__name__): + async with factory() as session: + await session.execute(sa_text("DELETE FROM accounts")) + fail_next_commit["armed"] = True + with pytest.raises(Exception): + await session.commit() + await session.rollback() + + records = [ + record + for record in caplog.records + if record.levelno == logging.WARNING and "sqlite_long_write_transaction" in record.getMessage() + ] + assert records + assert "outcome=commit_failed_rollback" in records[0].getMessage() + assert "outcome=commit " not in records[0].getMessage() + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_shielded_bounded_returns_none_when_the_awaitable_finishes_in_time() -> None: + async def _fast() -> str: + return "done" + + assert await session_module._shielded_bounded(_fast(), 1.0) is None + + async def _boom() -> None: + raise RuntimeError("teardown failed") + + with pytest.raises(RuntimeError, match="teardown failed"): + await session_module._shielded_bounded(_boom(), 1.0) + + +@pytest.mark.asyncio +async def test_shielded_bounded_abandons_a_wedged_awaitable_at_the_deadline() -> None: + release = asyncio.Event() + + async def _wedged() -> None: + await release.wait() + + abandoned = await session_module._shielded_bounded(_wedged(), 0.05) + assert abandoned is not None + assert not abandoned.done(), "the wedged awaitable must be left running, not cancelled" + release.set() + await abandoned + + +@pytest.mark.asyncio +async def test_shielded_bounded_absorbs_caller_cancellation_like_the_unbounded_shield() -> None: + """Teardown runs in ``finally`` blocks: the bound, not the caller's + cancellation, must decide abandonment (matching ``_shielded`` + the + swallow in ``_safe_rollback``/``_safe_close``).""" + started = asyncio.Event() + release = asyncio.Event() + finished: list[bool] = [] + + async def _work() -> None: + started.set() + await release.wait() + finished.append(True) + + async def _caller() -> asyncio.Task[object] | None: + return await session_module._shielded_bounded(_work(), 5.0) + + caller = asyncio.ensure_future(_caller()) + await started.wait() + caller.cancel() + await asyncio.sleep(0.05) + assert not caller.done(), "cancellation must not abandon the shielded teardown" + release.set() + assert await caller is None + assert finished, "the shielded work must run to completion despite the cancellation" + + +@pytest.mark.asyncio +async def test_close_session_reclaims_a_wedged_sqlite_rollback_so_other_writers_recover( + tmp_path, monkeypatch, caplog +) -> None: + """Issue #1682 part 2: a wedged rollback used to be awaited forever while + the aiosqlite worker kept the single writer slot — a self-sustaining + 'database is locked' stall that starved leader election itself. The + teardown must be bounded, and the bound alone is not enough: the wedged + connection must be interrupted and invalidated so the writer slot is + actually released and the connection is never handed out again.""" + monkeypatch.setattr(session_module, "_SQLITE_TEARDOWN_TIMEOUT_SECONDS", 0.2) + db_path = tmp_path / "wedged-rollback.db" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + poolclass=NullPool, + connect_args={"timeout": 5.0}, + ) + session_module._configure_sqlite_engine(engine.sync_engine, enable_wal=True) + # An independent writer with a short busy timeout: if the reclaim fails to + # release the writer slot, its INSERT surfaces 'database is locked' fast. + other_writer = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + poolclass=NullPool, + connect_args={"timeout": 1.0}, + ) + release_wedge = asyncio.Event() + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + caplog.clear() + + factory = async_sessionmaker(engine, expire_on_commit=False) + session = factory() + # Take the writer slot with an uncommitted write. + await session.execute(sa_text("DELETE FROM accounts")) + + held = session_module._session_sync_connections(session) + assert held, "the open write transaction must expose its sync connection" + sync_connection = held[0] + driver = sync_connection.connection.driver_connection + assert driver is not None + + # Wedge this connection's rollback (a stuck aiosqlite worker queues + # the teardown behind itself exactly like this) and spy on interrupt. + original_rollback = driver.rollback + interrupted = asyncio.Event() + original_interrupt = driver.interrupt + + async def _wedged_rollback() -> None: + await release_wedge.wait() + await original_rollback() + + # Delegate without changing the installed driver's shape: the reclaim + # awaits ``interrupt()``'s result only when it is awaitable, so the + # spy hands back exactly what the real aiosqlite method returns and + # the production awaitable-handling is exercised against the installed + # contract (a coroutine in the pinned aiosqlite) instead of a stand-in. + def _spying_interrupt() -> object: + interrupted.set() + return original_interrupt() + + driver.rollback = _wedged_rollback + driver.interrupt = _spying_interrupt + + with caplog.at_level(logging.INFO, logger=session_module.__name__): + close_task = asyncio.ensure_future(session_module.close_session(session)) + done, _ = await asyncio.wait({close_task}, timeout=2.0) + # RED on the pre-fix teardown: the shielded rollback was awaited + # unboundedly, so close_session never returned. + assert done, "close_session must be bounded when the sqlite rollback wedges" + + assert interrupted.is_set(), "the wedged driver must be interrupted to unstick its worker" + assert sync_connection.invalidated, "the wedged connection must be invalidated, never reused" + assert session.info.get(session_module._SQLITE_TEARDOWN_WEDGED_INFO_KEY) is True + + reclaim_logs = [ + record + for record in caplog.records + if record.levelno == logging.WARNING and "sqlite_wedged_teardown" in record.getMessage() + ] + assert reclaim_logs, "the reclaim must be reported with the watchdog's identifiers" + message = reclaim_logs[0].getMessage() + assert "phase=rollback" in message + assert "DELETE FROM accounts" in message, "part 1 watchdog identifiers must attribute the holder" + + # The stall must not be self-sustaining: with the wedged rollback + # still pending, another writer takes the slot immediately. + async with other_writer.begin() as writer: + await writer.execute(sa_text("DELETE FROM accounts")) + + # A wedged session is fenced: further teardown returns immediately + # instead of driving the session concurrently with the abandoned + # greenlet. + await asyncio.wait_for(session_module.close_session(session), timeout=1.0) + + # Late completion: once the wedge resolves, the abandoned teardown + # finishes and the session is closed for bookkeeping. + release_wedge.set() + for _ in range(100): + if any("finished late" in record.getMessage() for record in caplog.records): + break + await asyncio.sleep(0.02) + assert any("finished late" in record.getMessage() for record in caplog.records), ( + "the abandoned teardown must be observed finishing late" + ) + # The deferred bookkeeping close is owned until completion (drained + # by close_db on shutdown), never fire-and-forget. + pending_cleanup = tuple(session_module._wedged_teardown_cleanup_tasks) + if pending_cleanup: + await asyncio.wait_for(asyncio.gather(*pending_cleanup, return_exceptions=True), timeout=2.0) + assert not session_module._wedged_teardown_cleanup_tasks, ( + "the deferred close must deregister itself once it completes" + ) + finally: + release_wedge.set() + await asyncio.sleep(0.05) + await engine.dispose() + await other_writer.dispose() + + +@pytest.mark.asyncio +async def test_close_session_keeps_the_unbounded_shield_for_non_sqlite_sessions(monkeypatch) -> None: + """PostgreSQL teardown semantics are untouched: a slow rollback/close far + beyond the SQLite bound is still awaited to completion, never reclaimed.""" + monkeypatch.setattr(session_module, "_SQLITE_TEARDOWN_TIMEOUT_SECONDS", 0.01) + + class _FakeDialect: + name = "postgresql" + + class _FakeBind: + dialect = _FakeDialect() + + class _FakeSession: + def __init__(self) -> None: + self.info: dict[str, object] = {} + self.rolled_back = False + self.closed = False + + def get_bind(self) -> _FakeBind: + return _FakeBind() + + def in_transaction(self) -> bool: + return not self.rolled_back + + async def rollback(self) -> None: + await asyncio.sleep(0.1) + self.rolled_back = True + + async def close(self) -> None: + await asyncio.sleep(0.1) + self.closed = True + + fake = _FakeSession() + await session_module.close_session(cast(session_module.AsyncSession, fake)) + + assert fake.rolled_back, "the slow PostgreSQL rollback must be awaited to completion" + assert fake.closed, "the slow PostgreSQL close must be awaited to completion" + assert session_module._SQLITE_TEARDOWN_WEDGED_INFO_KEY not in fake.info + + +@pytest.mark.asyncio +async def test_close_session_bounds_a_wedged_sqlite_close_without_a_transaction(monkeypatch, caplog) -> None: + """The close step can wedge on its own (connection release goes through + the same aiosqlite worker); it must be bounded and fenced too.""" + monkeypatch.setattr(session_module, "_SQLITE_TEARDOWN_TIMEOUT_SECONDS", 0.05) + release = asyncio.Event() + + class _FakeDialect: + name = "sqlite" + + class _FakeUrl: + database = "/tmp/wedged-close.db" + query: dict[str, str] = {} + + class _FakeBind: + dialect = _FakeDialect() + url = _FakeUrl() + + class _FakeSyncSession: + def get_transaction(self) -> None: + return None + + class _FakeSession: + def __init__(self) -> None: + self.info: dict[str, object] = {} + self.sync_session = _FakeSyncSession() + + def get_bind(self) -> _FakeBind: + return _FakeBind() + + def in_transaction(self) -> bool: + return False + + async def close(self) -> None: + await release.wait() + + fake = _FakeSession() + try: + with caplog.at_level(logging.WARNING, logger=session_module.__name__): + await asyncio.wait_for(session_module.close_session(cast(session_module.AsyncSession, fake)), timeout=2.0) + + assert fake.info.get(session_module._SQLITE_TEARDOWN_WEDGED_INFO_KEY) is True + messages = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING and "sqlite_wedged_teardown" in record.getMessage() + ] + assert messages + assert "phase=close" in messages[0] + finally: + release.set() + await asyncio.sleep(0.05) + + +@pytest.mark.asyncio +async def test_close_session_never_reclaims_the_shared_in_memory_sqlite_connection(monkeypatch) -> None: + """In-memory SQLite shares one StaticPool connection with the whole + process: invalidating it would destroy the entire database (the + database-backends spec preserves shared in-memory state), and a single + shared connection cannot starve other writers. The teardown must keep the + unbounded shield there.""" + monkeypatch.setattr(session_module, "_SQLITE_TEARDOWN_TIMEOUT_SECONDS", 0.01) + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + factory = async_sessionmaker(engine, expire_on_commit=False) + + session = factory() + await session.execute(sa_text("DELETE FROM accounts")) + assert session_module._session_teardown_bound_seconds(session) is None + + held = session_module._session_sync_connections(session) + assert held + driver = held[0].connection.driver_connection + assert driver is not None + original_rollback = driver.rollback + + async def _slow_rollback() -> None: + await asyncio.sleep(0.1) + await original_rollback() + + driver.rollback = _slow_rollback + await session_module.close_session(session) + driver.rollback = original_rollback + + assert not held[0].invalidated, "the shared in-memory connection must never be invalidated" + assert session_module._SQLITE_TEARDOWN_WEDGED_INFO_KEY not in session.info + + # The database survives: schema and connection are intact. + verify = factory() + (await verify.execute(sa_text("SELECT count(*) FROM accounts"))).scalar_one() + await session_module.close_session(verify) + finally: + await engine.dispose() + + +@pytest.mark.parametrize( + "url_text", + [ + "sqlite+aiosqlite:///:memory:", + "sqlite+aiosqlite://", + "sqlite+aiosqlite:///file:shared?mode=memory&cache=shared&uri=true", + "sqlite+aiosqlite:///file::memory:?cache=shared&uri=true", + ], +) +def test_session_teardown_bound_skips_every_in_memory_sqlite_url_form(url_text: str) -> None: + """Every in-memory SQLite URL form must keep the unbounded teardown: the + SQLite URI forms carry ``mode=memory`` in the parsed URL's query, not in + ``url.database``, and a shared in-memory database reclaimed by invalidation + would be destroyed for the whole process. URI forms count only with + ``uri=true`` — that is what makes the dialect pass the string as a URI.""" + + class _FakeDialect: + name = "sqlite" + + class _FakeBind: + dialect = _FakeDialect() + url = make_url(url_text) + + class _FakeSession: + def get_bind(self) -> _FakeBind: + return _FakeBind() + + fake = _FakeSession() + assert session_module._session_teardown_bound_seconds(cast(session_module.AsyncSession, fake)) is None + + +@pytest.mark.parametrize( + "url_text", + [ + "sqlite+aiosqlite:////data/store.db", + "sqlite+aiosqlite:///file:/data/store.db?uri=true", + # Without ``uri=true`` the dialect never enables SQLite URI mode: this + # connects to a file literally named ``file:shared`` and must keep the + # bounded teardown despite carrying ``mode=memory`` in the query. + "sqlite+aiosqlite:///file:shared?mode=memory&cache=shared", + ], +) +def test_session_teardown_bound_applies_to_file_backed_sqlite_url_forms(url_text: str) -> None: + """File-backed SQLite (plain path, ``file:`` URI without ``mode=memory``, + or a ``mode=memory`` query without ``uri=true``) is exactly the + wedge-prone single-writer case and must stay bounded.""" + + class _FakeDialect: + name = "sqlite" + + class _FakeBind: + dialect = _FakeDialect() + url = make_url(url_text) + + class _FakeSession: + def get_bind(self) -> _FakeBind: + return _FakeBind() + + fake = _FakeSession() + assert ( + session_module._session_teardown_bound_seconds(cast(session_module.AsyncSession, fake)) + == session_module._SQLITE_TEARDOWN_TIMEOUT_SECONDS + ) + + +@pytest.mark.asyncio +async def test_reclaim_interrupts_the_real_aiosqlite_driver_without_a_spy(tmp_path, caplog) -> None: + """The reclaim invokes the driver's real ``interrupt()`` and awaits the + result only when it is awaitable. Exercise the production path against the + installed aiosqlite with no stand-in, so a driver signature change + surfaces as a failure here instead of being swallowed by the reclaim's + broad except (the failure is logged, and asserted absent).""" + engine = create_async_engine( + f"sqlite+aiosqlite:///{tmp_path / 'interrupt-contract.db'}", + poolclass=NullPool, + ) + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + factory = async_sessionmaker(engine, expire_on_commit=False) + session = factory() + await session.execute(sa_text("DELETE FROM accounts")) + held = session_module._session_sync_connections(session) + assert held, "the open write transaction must expose its sync connection" + + async def _already_finished_teardown() -> None: + return None + + abandoned = asyncio.ensure_future(_already_finished_teardown()) + with caplog.at_level(logging.DEBUG, logger=session_module.__name__): + await session_module._reclaim_wedged_sqlite_session(session, abandoned, held, phase="rollback") + + assert not any( + "Interrupting a wedged SQLite connection failed" in record.getMessage() for record in caplog.records + ), "the installed aiosqlite interrupt() contract must be handled without error" + assert held[0].invalidated, "the reclaim must still invalidate the connection" + + # Drain the bookkeeping the reclaim registered so no task outlives + # the test (mirrors the close_db drain). + for _ in range(100): + pending = tuple(session_module._wedged_teardown_cleanup_tasks) + if not pending: + break + await asyncio.wait_for(asyncio.gather(*pending, return_exceptions=True), timeout=2.0) + await asyncio.sleep(0) + assert not session_module._wedged_teardown_cleanup_tasks + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_close_db_drains_a_pending_reclaimed_rollback_and_its_bookkeeping_close( + tmp_path, monkeypatch, caplog +) -> None: + """A rollback reclaimed as wedged can still be pending when close_db runs. + The abandoned task is registered in the teardown registry immediately, so + close_db must wait for it — and for the bookkeeping close it schedules only + after any one-time snapshot — instead of returning while the event loop + still has pending teardown tasks.""" + monkeypatch.setattr(session_module, "_SQLITE_TEARDOWN_TIMEOUT_SECONDS", 0.5) + db_path = tmp_path / "close-db-drain.db" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + poolclass=NullPool, + connect_args={"timeout": 5.0}, + ) + session_module._configure_sqlite_engine(engine.sync_engine, enable_wal=True) + release_wedge = asyncio.Event() + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + + factory = async_sessionmaker(engine, expire_on_commit=False) + session = factory() + await session.execute(sa_text("DELETE FROM accounts")) + + held = session_module._session_sync_connections(session) + assert held + driver = held[0].connection.driver_connection + assert driver is not None + original_rollback = driver.rollback + + async def _wedged_rollback() -> None: + await release_wedge.wait() + await original_rollback() + + driver.rollback = _wedged_rollback + + with caplog.at_level(logging.INFO, logger=session_module.__name__): + await asyncio.wait_for(session_module.close_session(session), timeout=5.0) + abandoned_pending = [task for task in session_module._wedged_teardown_cleanup_tasks if not task.done()] + # RED pre-fix: the reclaim only registered the deferred bookkeeping + # close (which does not exist yet), never the abandoned rollback. + assert abandoned_pending, "the reclaimed rollback must be registered while still pending" + + async def _release_soon() -> None: + await asyncio.sleep(0.05) + release_wedge.set() + + releaser = asyncio.ensure_future(_release_soon()) + await asyncio.wait_for(session_module.close_db(), timeout=5.0) + # RED pre-fix: close_db saw an empty registry and returned + # immediately, before the wedge was even released. + assert release_wedge.is_set(), "close_db must drain the pending reclaimed rollback" + assert all(task.done() for task in abandoned_pending), ( + "close_db must wait for the abandoned rollback itself" + ) + assert not session_module._wedged_teardown_cleanup_tasks, ( + "close_db must also drain the bookkeeping close scheduled after its first snapshot" + ) + await releaser + + assert any("finished late" in record.getMessage() for record in caplog.records) + finally: + release_wedge.set() + await asyncio.sleep(0.05) + await engine.dispose() + + +@pytest.mark.asyncio +async def test_close_db_bounds_the_wedged_teardown_drain(monkeypatch, caplog) -> None: + """A teardown that stays wedged despite the reclaim (the interrupt is + best-effort) must not wedge shutdown too: the registry drain is explicitly + bounded and abandons whatever remains after the deadline.""" + monkeypatch.setattr(session_module, "_SQLITE_TEARDOWN_TIMEOUT_SECONDS", 0.05) + never = asyncio.Event() + stuck: asyncio.Task[bool] = asyncio.ensure_future(never.wait()) + session_module._wedged_teardown_cleanup_tasks.add(stuck) + try: + with caplog.at_level(logging.WARNING, logger=session_module.__name__): + await asyncio.wait_for(session_module.close_db(), timeout=2.0) + assert any("still-pending wedged-teardown" in record.getMessage() for record in caplog.records), ( + "the bounded drain must report what it abandoned" + ) + assert stuck in session_module._wedged_teardown_cleanup_tasks + finally: + session_module._wedged_teardown_cleanup_tasks.discard(stuck) + never.set() + await stuck diff --git a/tests/unit/test_dependencies_db_pool_ownership.py b/tests/unit/test_dependencies_db_pool_ownership.py deleted file mode 100644 index 4469e07c72..0000000000 --- a/tests/unit/test_dependencies_db_pool_ownership.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import annotations - -from collections.abc import AsyncIterator -from contextlib import asynccontextmanager - -import pytest -from fastapi import FastAPI - -import app.dependencies as dependencies - -pytestmark = pytest.mark.unit - - -@pytest.mark.asyncio -async def test_proxy_repository_context_uses_foreground_request_session( - monkeypatch: pytest.MonkeyPatch, -) -> None: - request_session = object() - lifecycle: list[str] = [] - - @asynccontextmanager - async def request_scope() -> AsyncIterator[object]: - lifecycle.append("entered") - try: - yield request_session - finally: - lifecycle.append("exited") - - monkeypatch.setattr(dependencies, "get_request_session", request_scope) - - async with dependencies._proxy_repo_context() as repositories: - assert repositories.accounts.session is request_session - assert lifecycle == ["entered"] - - assert lifecycle == ["entered", "exited"] - - -def test_application_proxy_service_injects_per_operation_refresh_repository() -> None: - app = FastAPI() - - service = dependencies.get_proxy_service_for_app(app) - - assert service._repo_factory is dependencies._proxy_repo_context - assert service._refresh_repo_factory is dependencies._accounts_refresh_repo_context - assert dependencies.get_proxy_service_for_app(app) is service diff --git a/tests/unit/test_docker_compose_postgres.py b/tests/unit/test_docker_compose_postgres.py index 72b9591cc1..1accaa0150 100644 --- a/tests/unit/test_docker_compose_postgres.py +++ b/tests/unit/test_docker_compose_postgres.py @@ -11,6 +11,15 @@ def _compose() -> dict[str, Any]: return yaml.safe_load((repo_root / "docker-compose.yml").read_text(encoding="utf-8")) +def test_postgres_compose_service_sizes_dev_shm_for_parallel_query() -> None: + postgres = _compose()["services"]["postgres"] + + # Docker's default 64MB /dev/shm makes PostgreSQL parallel hash joins fail + # with "could not resize shared memory segment ... No space left on + # device" once they spill past the segment. Keep an explicit >= 1GB size. + assert postgres["shm_size"] == "1gb" + + def test_postgres18_compose_upgrade_helper_is_digest_pinned() -> None: services = _compose()["services"] postgres = services["postgres"] diff --git a/tests/unit/test_durable_bridge_sessions.py b/tests/unit/test_durable_bridge_sessions.py index 01ed65e707..d7c3ca1d8c 100644 --- a/tests/unit/test_durable_bridge_sessions.py +++ b/tests/unit/test_durable_bridge_sessions.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import contextlib import inspect from collections.abc import AsyncIterator, Callable from datetime import datetime, timedelta, timezone @@ -16,8 +17,6 @@ from app.core.utils.time import utcnow from app.db.models import ( Base, - HttpBridgeRecoveryAttemptRecord, - HttpBridgeRecoveryAttemptState, HttpBridgeSessionAlias, HttpBridgeSessionRecord, HttpBridgeSessionState, @@ -30,15 +29,13 @@ is_http_bridge_account_neutral_replay, make_http_bridge_account_neutral_replay_key, ) -from app.modules.proxy.durable_bridge_coordinator import DurableBridgeSessionCoordinator +from app.modules.proxy.durable_bridge_coordinator import DurableBridgeLookup, DurableBridgeSessionCoordinator from app.modules.proxy.durable_bridge_repository import ( - REQUIRED_DURABLE_BRIDGE_TABLES, DurableBridgeAliasRegistration, DurableBridgeRepository, - durable_bridge_api_key_scope, - missing_durable_bridge_tables, + durable_bridge_hash, + durable_bridge_operation_id, ) -from app.modules.proxy.response_transition_manifest import build_response_transition_manifest pytestmark = pytest.mark.unit @@ -145,516 +142,6 @@ async def test_durable_bridge_lookup_prefers_turn_state_then_previous_response_t assert by_session.canonical_key == "sid-123" -@pytest.mark.asyncio -async def test_recovery_required_marker_is_owner_anchor_bound_durable_and_terminal_cleared( - coordinator: DurableBridgeSessionCoordinator, - async_session_factory: Callable[[], AsyncSession], -) -> None: - claimed = await coordinator.claim_live_session( - session_key_kind="session_header", - session_key_value="sid-recovery-required", - api_key_id="key-recovery-required", - instance_id="instance-a", - owner_process_epoch="process-a", - lease_ttl_seconds=120.0, - account_id="acc-recovery-required", - model="gpt-5.6-sol", - service_tier=None, - latest_turn_state="turn-recovery-required", - latest_response_id=None, - allow_takeover=True, - ) - registered = await coordinator.register_previous_response_id( - session_id=claimed.session_id, - api_key_id="key-recovery-required", - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - response_id="resp-rejected-anchor", - lease_ttl_seconds=120.0, - input_item_count=4, - input_full_fingerprint="a" * 64, - pending_tool_calls={"call-pending": "custom_tool_call"}, - ) - assert registered == DurableBridgeAliasRegistration.REGISTERED - - concurrent_marks = await asyncio.gather( - *( - coordinator.mark_live_session_recovery_required( - session_id=claimed.session_id, - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - account_id="acc-recovery-required", - rejected_response_id="resp-rejected-anchor", - ) - for _ in range(2) - ) - ) - assert concurrent_marks == [True, True] - assert ( - await coordinator.mark_live_session_recovery_required( - session_id=claimed.session_id, - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - account_id="acc-other", - rejected_response_id="resp-rejected-anchor", - ) - is False - ) - assert ( - await coordinator.mark_live_session_recovery_required( - session_id=claimed.session_id, - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - account_id="acc-recovery-required", - rejected_response_id="resp-other-anchor", - ) - is False - ) - - wire_fingerprints = ("c" * 64, "d" * 64) - concurrent_attempt_claims = await asyncio.gather( - *( - coordinator.claim_live_session_recovery_attempt( - session_id=claimed.session_id, - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - account_id="acc-recovery-required", - rejected_response_id="resp-rejected-anchor", - attempt_fingerprint=fingerprint, - request_id=f"claim-{fingerprint[0]}", - ) - for fingerprint in wire_fingerprints - ) - ) - assert sorted(concurrent_attempt_claims) == [False, True] - claimed_fingerprint = wire_fingerprints[concurrent_attempt_claims.index(True)] - rejected_fingerprint = next(fingerprint for fingerprint in wire_fingerprints if fingerprint != claimed_fingerprint) - crashed_attempt = await coordinator.record_recovery_attempt( - session_id=claimed.session_id, - api_key_id="key-recovery-required", - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - request_fingerprint=claimed_fingerprint, - request_id="request-before-process-loss", - account_id="acc-recovery-required", - model="gpt-5.6-sol", - replay_safe=True, - ) - assert crashed_attempt is not None - assert crashed_attempt.state.value == "unknown" - - async with async_session_factory() as session: - await session.execute( - update(HttpBridgeSessionRecord) - .where(HttpBridgeSessionRecord.id == claimed.session_id) - .values( - lease_expires_at=utcnow() - timedelta(seconds=1), - owner_process_epoch="expired-process", - ) - ) - await session.commit() - - after_lease_expiry = await DurableBridgeSessionCoordinator(async_session_factory).lookup_request_targets( - session_key_kind="session_header", - session_key_value="sid-recovery-required", - api_key_id="key-recovery-required", - turn_state=None, - session_header="sid-recovery-required", - previous_response_id="resp-rejected-anchor", - ) - assert after_lease_expiry is not None - assert after_lease_expiry.recovery_is_required_for_latest_anchor() is True - assert after_lease_expiry.recovery_required_account_id == "acc-recovery-required" - assert after_lease_expiry.recovery_required_anchor_hash is not None - assert after_lease_expiry.recovery_required_anchor_hash != "resp-rejected-anchor" - assert len(after_lease_expiry.recovery_required_anchor_hash) == 64 - assert after_lease_expiry.recovery_required_attempt_fingerprint == claimed_fingerprint - assert after_lease_expiry.recovery_required_at is not None - replacement_coordinator = DurableBridgeSessionCoordinator(async_session_factory) - assert ( - await replacement_coordinator.claim_live_session_recovery_attempt( - session_id=claimed.session_id, - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - account_id="acc-recovery-required", - rejected_response_id="resp-rejected-anchor", - attempt_fingerprint=claimed_fingerprint, - request_id=f"claim-{claimed_fingerprint[0]}", - ) - is True - ) - assert ( - await replacement_coordinator.claim_live_session_recovery_attempt( - session_id=claimed.session_id, - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - account_id="acc-recovery-required", - rejected_response_id="resp-rejected-anchor", - attempt_fingerprint=rejected_fingerprint, - request_id=f"claim-{rejected_fingerprint[0]}", - ) - is False - ) - retained_unknown = await replacement_coordinator.lookup_recovery_attempt( - session_id=claimed.session_id, - request_fingerprint=claimed_fingerprint, - ) - assert retained_unknown is not None - assert retained_unknown.state.value == "unknown" - assert retained_unknown.request_id == "request-before-process-loss" - - replacement_registered = await coordinator.register_previous_response_id( - session_id=claimed.session_id, - api_key_id="key-recovery-required", - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - response_id="resp-replacement-anchor", - lease_ttl_seconds=120.0, - input_item_count=6, - input_full_fingerprint="b" * 64, - pending_tool_calls={}, - ) - assert replacement_registered == DurableBridgeAliasRegistration.REGISTERED - replacement = await coordinator.lookup_request_targets( - session_key_kind="session_header", - session_key_value="sid-recovery-required", - api_key_id="key-recovery-required", - turn_state=None, - session_header="sid-recovery-required", - previous_response_id="resp-replacement-anchor", - ) - assert replacement is not None - assert replacement.latest_response_id == "resp-replacement-anchor" - assert replacement.recovery_is_required_for_latest_anchor() is False - assert replacement.recovery_required_anchor_hash is None - assert replacement.recovery_required_account_id is None - assert replacement.recovery_required_attempt_fingerprint is None - assert replacement.recovery_required_at is None - - -@pytest.mark.asyncio -async def test_marker_terminal_alias_conflict_rolls_back_checkpoint_and_journal( - coordinator: DurableBridgeSessionCoordinator, - async_session_factory: Callable[[], AsyncSession], -) -> None: - api_key_id = "key-marker-terminal-atomic" - marker = await coordinator.claim_live_session( - session_key_kind="session_header", - session_key_value="sid-marker-terminal-atomic", - api_key_id=api_key_id, - instance_id="instance-a", - owner_process_epoch="process-a", - lease_ttl_seconds=120.0, - account_id="acc-marker-terminal-atomic", - model="gpt-5.6-sol", - service_tier=None, - latest_turn_state=None, - latest_response_id=None, - allow_takeover=True, - ) - assert ( - await coordinator.register_previous_response_id( - session_id=marker.session_id, - api_key_id=api_key_id, - instance_id="instance-a", - owner_epoch=marker.owner_epoch, - response_id="resp-marker-terminal-old", - lease_ttl_seconds=120.0, - input_item_count=4, - input_full_fingerprint="a" * 64, - pending_tool_calls={"call-a": "custom_tool_call"}, - ) - == DurableBridgeAliasRegistration.REGISTERED - ) - assert await coordinator.mark_live_session_recovery_required( - session_id=marker.session_id, - instance_id="instance-a", - owner_epoch=marker.owner_epoch, - account_id="acc-marker-terminal-atomic", - rejected_response_id="resp-marker-terminal-old", - ) - request_fingerprint = "b" * 64 - assert await coordinator.claim_live_session_recovery_attempt( - session_id=marker.session_id, - instance_id="instance-a", - owner_epoch=marker.owner_epoch, - account_id="acc-marker-terminal-atomic", - rejected_response_id="resp-marker-terminal-old", - attempt_fingerprint=request_fingerprint, - request_id="claim-marker-terminal-atomic", - ) - attempt = await coordinator.record_recovery_attempt( - session_id=marker.session_id, - api_key_id=api_key_id, - instance_id="instance-a", - owner_epoch=marker.owner_epoch, - request_fingerprint=request_fingerprint, - request_id="request-marker-terminal-atomic", - account_id="acc-marker-terminal-atomic", - model="gpt-5.6-sol", - replay_safe=True, - ) - assert attempt is not None - - replay_kind, replay_key = make_http_bridge_account_neutral_replay_key("protected-terminal-alias") - protected = await coordinator.claim_live_session( - session_key_kind=replay_kind, - session_key_value=replay_key, - api_key_id=api_key_id, - instance_id="instance-b", - owner_process_epoch="process-b", - lease_ttl_seconds=120.0, - account_id="acc-protected-terminal-alias", - model="gpt-5.6-sol", - service_tier=None, - latest_turn_state=None, - latest_response_id=None, - allow_takeover=True, - ) - assert ( - await coordinator.register_previous_response_id( - session_id=protected.session_id, - api_key_id=api_key_id, - instance_id="instance-b", - owner_epoch=protected.owner_epoch, - response_id="resp-marker-terminal-new", - lease_ttl_seconds=120.0, - ) - == DurableBridgeAliasRegistration.REGISTERED - ) - - async with async_session_factory() as session: - settled = await DurableBridgeRepository(session).settle_marker_recovery_completed( - session_id=marker.session_id, - api_key_scope=durable_bridge_api_key_scope(api_key_id), - instance_id="instance-a", - owner_epoch=marker.owner_epoch, - account_id="acc-marker-terminal-atomic", - request_fingerprint=request_fingerprint, - claim_request_id="claim-marker-terminal-atomic", - request_id="request-marker-terminal-atomic", - response_id="resp-marker-terminal-new", - input_item_count=5, - input_full_fingerprint="c" * 64, - pending_tool_calls={}, - response_transition_manifest=None, - lease_ttl_seconds=120.0, - ) - assert settled is False - - async with async_session_factory() as session: - retained_marker = await session.get(HttpBridgeSessionRecord, marker.session_id) - retained_attempt = await session.scalar( - select(HttpBridgeRecoveryAttemptRecord).where( - HttpBridgeRecoveryAttemptRecord.session_id == marker.session_id, - HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint, - ) - ) - protected_alias = await session.scalar( - select(HttpBridgeSessionAlias).where( - HttpBridgeSessionAlias.alias_kind == "previous_response_id", - HttpBridgeSessionAlias.alias_value == "resp-marker-terminal-new", - ) - ) - assert retained_marker is not None - assert retained_marker.latest_response_id == "resp-marker-terminal-old" - assert retained_marker.recovery_required_anchor_hash is not None - assert retained_marker.recovery_required_account_id == "acc-marker-terminal-atomic" - assert retained_marker.recovery_required_attempt_fingerprint == request_fingerprint - assert retained_attempt is not None - assert retained_attempt.state == HttpBridgeRecoveryAttemptState.UNKNOWN - assert retained_attempt.response_id is None - assert protected_alias is not None - assert protected_alias.session_id == protected.session_id - - -@pytest.mark.asyncio -async def test_recovery_required_marker_survives_all_retention_paths_until_terminal_clear( - coordinator: DurableBridgeSessionCoordinator, - async_session_factory: Callable[[], AsyncSession], -) -> None: - claimed = await coordinator.claim_live_session( - session_key_kind="session_header", - session_key_value="sid-recovery-retention", - api_key_id="key-recovery-retention", - instance_id="instance-a", - owner_process_epoch="process-a", - lease_ttl_seconds=120.0, - account_id="acc-recovery-retention", - model="gpt-5.6-sol", - service_tier=None, - latest_turn_state="turn-recovery-retention", - latest_response_id=None, - allow_takeover=True, - ) - registered = await coordinator.register_previous_response_id( - session_id=claimed.session_id, - api_key_id="key-recovery-retention", - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - response_id="resp-recovery-retention", - lease_ttl_seconds=120.0, - input_item_count=3, - input_full_fingerprint="a" * 64, - pending_tool_calls={"call-retained": "custom_tool_call"}, - ) - assert registered == DurableBridgeAliasRegistration.REGISTERED - assert await coordinator.mark_live_session_recovery_required( - session_id=claimed.session_id, - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - account_id="acc-recovery-retention", - rejected_response_id="resp-recovery-retention", - ) - - stale_time = utcnow() - timedelta(hours=12) - async with async_session_factory() as session: - await session.execute( - update(HttpBridgeSessionRecord) - .where(HttpBridgeSessionRecord.id == claimed.session_id) - .values( - owner_instance_id=None, - lease_expires_at=stale_time, - last_seen_at=stale_time, - state=HttpBridgeSessionState.ACTIVE, - ) - ) - await session.commit() - - # Process replacement/startup retention cannot erase the authority. - assert ( - await DurableBridgeSessionCoordinator(async_session_factory).purge_owned_sessions_on_startup( - instance_id="instance-b", - ownerless_cutoff=utcnow() - timedelta(hours=1), - ) - == 0 - ) - async with async_session_factory() as session: - await session.execute( - update(HttpBridgeSessionRecord) - .where(HttpBridgeSessionRecord.id == claimed.session_id) - .values(state=HttpBridgeSessionState.CLOSED, closed_at=stale_time) - ) - await session.commit() - assert await DurableBridgeRepository(session).purge_closed_before(utcnow() - timedelta(hours=1)) == 0 - await session.execute( - update(HttpBridgeSessionRecord) - .where(HttpBridgeSessionRecord.id == claimed.session_id) - .values(state=HttpBridgeSessionState.ACTIVE, closed_at=None) - ) - await session.commit() - assert await DurableBridgeRepository(session).purge_abandoned_before(utcnow() - timedelta(hours=1)) == 0 - - after_restart = await DurableBridgeSessionCoordinator(async_session_factory).lookup_request_targets( - session_key_kind="session_header", - session_key_value="sid-recovery-retention", - api_key_id="key-recovery-retention", - turn_state=None, - session_header="sid-recovery-retention", - previous_response_id="resp-recovery-retention", - ) - assert after_restart is not None - assert after_restart.recovery_is_required_for_latest_anchor() - assert after_restart.latest_pending_tool_calls == {"call-retained": "custom_tool_call"} - - replacement = await coordinator.claim_live_session( - session_key_kind="session_header", - session_key_value="sid-recovery-retention", - api_key_id="key-recovery-retention", - instance_id="instance-b", - owner_process_epoch="process-b", - lease_ttl_seconds=120.0, - account_id="acc-recovery-retention", - model="gpt-5.6-sol", - service_tier=None, - latest_turn_state=None, - latest_response_id=None, - allow_takeover=True, - ) - terminal = await coordinator.register_previous_response_id( - session_id=replacement.session_id, - api_key_id="key-recovery-retention", - instance_id="instance-b", - owner_epoch=replacement.owner_epoch, - response_id="resp-recovery-replacement", - lease_ttl_seconds=120.0, - input_item_count=5, - input_full_fingerprint="b" * 64, - pending_tool_calls={}, - ) - assert terminal == DurableBridgeAliasRegistration.REGISTERED - await coordinator.release_live_session( - session_id=replacement.session_id, - instance_id="instance-b", - owner_epoch=replacement.owner_epoch, - draining=False, - ) - async with async_session_factory() as session: - await session.execute( - update(HttpBridgeSessionRecord) - .where(HttpBridgeSessionRecord.id == replacement.session_id) - .values(last_seen_at=stale_time, closed_at=stale_time) - ) - await session.commit() - assert await DurableBridgeRepository(session).purge_closed_before(utcnow() - timedelta(hours=1)) == 1 - - -@pytest.mark.asyncio -async def test_missing_durable_bridge_tables_checks_current_postgres_schemas() -> None: - captured_sql: list[str] = [] - - class _PostgresSession: - def get_bind(self) -> SimpleNamespace: - return SimpleNamespace(dialect=SimpleNamespace(name="postgresql")) - - async def execute(self, statement: object, *args: object) -> SimpleNamespace: - del args - captured_sql.append(str(statement)) - return SimpleNamespace(fetchall=lambda: [("http_bridge_sessions",)]) - - missing = await missing_durable_bridge_tables(cast(AsyncSession, _PostgresSession())) - - assert "http_bridge_session_aliases" in missing - assert "http_bridge_rowless_recovery_authorities" in missing - assert any("current_schemas(false)" in sql for sql in captured_sql) - assert all("table_schema = 'public'" not in sql for sql in captured_sql) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("dialect", ("sqlite", "postgresql")) -async def test_missing_durable_bridge_tables_reports_required_rowless_columns( - dialect: str, -) -> None: - captured_sql: list[str] = [] - - class _SchemaSession: - def get_bind(self) -> SimpleNamespace: - return SimpleNamespace(dialect=SimpleNamespace(name=dialect)) - - async def execute(self, statement: object, *args: object) -> SimpleNamespace: - del args - rendered = str(statement) - captured_sql.append(rendered) - if "sqlite_master" in rendered or "information_schema.tables" in rendered: - rows = [(table,) for table in REQUIRED_DURABLE_BRIDGE_TABLES] - elif "PRAGMA table_info" in rendered: - rows = [(0, "id")] - else: - rows = [("id",)] - return SimpleNamespace(fetchall=lambda: rows) - - missing = await missing_durable_bridge_tables(cast(AsyncSession, _SchemaSession())) - - assert missing == ( - "http_bridge_rowless_recovery_authorities.authorization_mode", - "http_bridge_rowless_recovery_authorities.authorization_proof_sha256", - "http_bridge_rowless_recovery_authorities.origin_marker_session_id", - "http_bridge_sessions.latest_response_transition_manifest_json", - "http_bridge_sessions.recovery_required_attempt_request_id", - ) - assert len(captured_sql) == 3 - - @pytest.mark.asyncio async def test_reversible_recovery_turn_state_registration_restores_previous_owner( coordinator: DurableBridgeSessionCoordinator, @@ -2012,9 +1499,14 @@ async def test_durable_bridge_stale_owner_cannot_register_turn_state_after_epoch @pytest.mark.asyncio -async def test_durable_bridge_claim_renews_same_owner_epoch( +async def test_durable_bridge_same_owner_reclaim_advances_epoch_to_fence_the_predecessor( coordinator: DurableBridgeSessionCoordinator, ) -> None: + """Claims come only from a successor in-memory session (a reused session + renews instead of claiming), so a live same-owner row means the predecessor + local session is retiring concurrently. The claim must advance the epoch so + the predecessor's outstanding fenced release no-ops instead of racing the + successor into a closed, ownerless row (issue #1695).""" claimed = await coordinator.claim_live_session( session_key_kind="session_header", session_key_value="sid-123", @@ -2046,10 +1538,23 @@ async def test_durable_bridge_claim_renews_same_owner_epoch( ) assert renewed.session_id == claimed.session_id - assert renewed.owner_epoch == claimed.owner_epoch + assert renewed.owner_epoch == claimed.owner_epoch + 1 assert renewed.latest_turn_state == "http_turn_2" assert renewed.latest_response_id == "resp_2" + # The predecessor's release carries the old epoch: it must be fenced out, + # leaving the successor's claim live and owned. + released = await coordinator.release_live_session( + session_id=claimed.session_id, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + draining=False, + ) + assert released is not None + assert released.owner_instance_id == "instance-a" + assert released.state == HttpBridgeSessionState.ACTIVE + assert released.owner_epoch == renewed.owner_epoch + @pytest.mark.asyncio async def test_durable_bridge_account_change_advances_epoch_to_fence_stale_release( @@ -2614,92 +2119,18 @@ async def test_durable_bridge_pending_tool_calls_are_bound_to_response_id( @pytest.mark.asyncio -async def test_durable_bridge_response_transition_manifest_survives_lookup_and_clears_with_anchor( +async def test_durable_bridge_takeover_with_account_change_clears_stale_aliases( coordinator: DurableBridgeSessionCoordinator, ) -> None: claimed = await coordinator.claim_live_session( session_key_kind="session_header", - session_key_value="sid-transition-manifest", + session_key_value="sid-alias-reset", api_key_id=None, instance_id="instance-a", owner_process_epoch="test-process", lease_ttl_seconds=60.0, account_id="acc-1", - model="gpt-5.6-sol", - service_tier=None, - latest_turn_state=None, - latest_response_id=None, - allow_takeover=True, - ) - pending = {"call_transition": "custom_tool_call"} - manifest = build_response_transition_manifest( - { - "response": { - "id": "resp_transition", - "status": "completed", - "output": [ - { - "type": "custom_tool_call", - "id": "ctc_transition", - "call_id": "call_transition", - "name": "shell", - "input": "pwd", - "status": "completed", - } - ], - } - }, - pending_tool_calls=pending, - ) - assert manifest is not None - - await coordinator.register_previous_response_id( - session_id=claimed.session_id, - api_key_id=None, - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - response_id="resp_transition", - lease_ttl_seconds=60.0, - input_item_count=1, - input_full_fingerprint="c" * 64, - pending_tool_calls=pending, - response_transition_manifest=manifest, - ) - - lookup = await coordinator.lookup_request_targets( - session_key_kind="session_header", - session_key_value="sid-transition-manifest", - api_key_id=None, - turn_state=None, - session_header="sid-transition-manifest", - previous_response_id=None, - ) - assert lookup is not None - assert lookup.latest_response_transition_manifest == manifest - - cleared = await coordinator.clear_live_session_response_anchor( - session_id=claimed.session_id, - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - ) - assert cleared is not None - assert cleared.latest_response_id is None - assert cleared.latest_response_transition_manifest is None - - -@pytest.mark.asyncio -async def test_durable_bridge_takeover_with_account_change_clears_stale_aliases( - coordinator: DurableBridgeSessionCoordinator, -) -> None: - claimed = await coordinator.claim_live_session( - session_key_kind="session_header", - session_key_value="sid-alias-reset", - api_key_id=None, - instance_id="instance-a", - owner_process_epoch="test-process", - lease_ttl_seconds=60.0, - account_id="acc-1", - model="gpt-5.4", + model="gpt-5.4", service_tier=None, latest_turn_state="http_turn_old", latest_response_id="resp_old", @@ -2816,6 +2247,24 @@ async def test_durable_bridge_lookup_active_lease_survives_request_lookup( assert lookup.lease_is_active(now=utcnow()) is True +def test_durable_bridge_lookup_lease_accepts_offset_aware_timestamp() -> None: + lookup = DurableBridgeLookup( + session_id="session-aware-lease", + canonical_kind="session_header", + canonical_key="sid-aware-lease", + api_key_scope="anonymous", + account_id="acc-1", + owner_instance_id="instance-a", + owner_epoch=1, + lease_expires_at=datetime.now(timezone.utc) + timedelta(minutes=1), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state=None, + latest_response_id=None, + ) + + assert lookup.lease_is_active(now=utcnow()) is True + + @pytest.mark.asyncio async def test_durable_bridge_lookup_falls_back_to_latest_turn_state_when_alias_missing( coordinator: DurableBridgeSessionCoordinator, @@ -2968,6 +2417,138 @@ async def test_mark_instance_draining_keeps_current_owner_lease_active( assert lookup.lease_is_active(now=utcnow()) is True +@pytest.mark.asyncio +async def test_claim_refuses_live_draining_lease_without_allow_takeover( + coordinator: DurableBridgeSessionCoordinator, +) -> None: + claimed = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-draining-claim", + api_key_id=None, + instance_id="instance-a", + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state="http_turn_1", + latest_response_id="resp_1", + allow_takeover=True, + ) + updated = await coordinator.mark_instance_draining(instance_id="instance-a") + assert updated == 1 + + refused = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-draining-claim", + api_key_id=None, + instance_id="instance-b", + owner_process_epoch="test-process-b", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state="http_turn_2", + latest_response_id="resp_2", + allow_takeover=False, + ) + + assert refused.owner_instance_id == "instance-a" + assert refused.state == "draining" + assert refused.lease_expires_at == claimed.lease_expires_at + assert refused.lease_is_active(now=utcnow()) is True + + +@pytest.mark.asyncio +async def test_claim_refuses_live_draining_lease_even_with_allow_takeover( + coordinator: DurableBridgeSessionCoordinator, +) -> None: + claimed = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-draining-force", + api_key_id=None, + instance_id="instance-a", + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state="http_turn_1", + latest_response_id="resp_1", + allow_takeover=True, + ) + updated = await coordinator.mark_instance_draining(instance_id="instance-a") + assert updated == 1 + + refused = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-draining-force", + api_key_id=None, + instance_id="instance-b", + owner_process_epoch="test-process-b", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state="http_turn_2", + latest_response_id="resp_2", + allow_takeover=True, + ) + + assert refused.owner_instance_id == "instance-a" + assert refused.state == "draining" + assert refused.lease_expires_at == claimed.lease_expires_at + assert refused.lease_is_active(now=utcnow()) is True + + +@pytest.mark.asyncio +async def test_claim_takes_over_expired_draining_lease( + coordinator: DurableBridgeSessionCoordinator, + async_session_factory: Callable[[], AsyncSession], +) -> None: + claimed = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-draining-expired", + api_key_id=None, + instance_id="instance-a", + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state="http_turn_1", + latest_response_id="resp_1", + allow_takeover=True, + ) + updated = await coordinator.mark_instance_draining(instance_id="instance-a") + assert updated == 1 + + async with async_session_factory() as session: + record = await session.get(HttpBridgeSessionRecord, claimed.session_id) + assert record is not None + record.lease_expires_at = utcnow() - timedelta(seconds=1) + await session.commit() + + stolen = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-draining-expired", + api_key_id=None, + instance_id="instance-b", + owner_process_epoch="test-process-b", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state="http_turn_2", + latest_response_id="resp_2", + allow_takeover=False, + ) + + assert stolen.owner_instance_id == "instance-b" + assert stolen.state == "active" + assert stolen.lease_is_active(now=utcnow()) is True + + @pytest.mark.asyncio async def test_startup_purges_owned_bridge_rows( coordinator: DurableBridgeSessionCoordinator, @@ -3024,6 +2605,57 @@ async def test_startup_purges_owned_bridge_rows( assert sticky is not None +@pytest.mark.asyncio +async def test_startup_reclassifies_submitted_operation_for_recovery( + coordinator: DurableBridgeSessionCoordinator, + async_session_factory: Callable[[], AsyncSession], +) -> None: + claimed = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-submitted-recovery", + api_key_id=None, + instance_id="instance-submitted-recovery", + owner_process_epoch="old-process", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.6", + service_tier=None, + latest_turn_state="turn-state", + latest_response_id=None, + allow_takeover=True, + ) + fingerprint = durable_bridge_hash("submitted-recovery") + operation_id = durable_bridge_operation_id(claimed.session_id, fingerprint) + async with async_session_factory() as session: + repository = DurableBridgeRepository(session) + assert await repository.record_operation( + operation_id=operation_id, + session_id=claimed.session_id, + instance_id="instance-submitted-recovery", + owner_epoch=claimed.owner_epoch, + request_fingerprint=fingerprint, + account_id="acc-1", + model="gpt-5.6", + parent_response_id=None, + ) + await session.execute( + update(HttpBridgeSessionRecord) + .where(HttpBridgeSessionRecord.id == claimed.session_id) + .values(last_seen_at=utcnow() - timedelta(minutes=5)) + ) + await session.commit() + + deleted = await repository.purge_owned_sessions_on_startup( + instance_id="instance-submitted-recovery", + owner_process_epoch="new-process", + ) + + assert deleted == 0 + operation = await repository.get_operation(operation_id=operation_id) + assert operation is not None + assert operation.state == "unknown" + + @pytest.mark.asyncio async def test_startup_closes_same_instance_previous_process_epoch_rows( coordinator: DurableBridgeSessionCoordinator, @@ -3333,6 +2965,7 @@ async def test_startup_retention_normalizes_aware_postgres_timestamps() -> None: exhausted = SimpleNamespace(all=lambda: []) session = SimpleNamespace( execute=AsyncMock(side_effect=[selected, SimpleNamespace(), exhausted]), + scalars=AsyncMock(return_value=[]), commit=AsyncMock(), ) repository = DurableBridgeRepository(cast(AsyncSession, session)) @@ -3677,3 +3310,246 @@ def test_lease_is_active_accepts_timestamptz_aware_expiry(): # Existing naive-vs-naive behaviour is unchanged. assert _lookup_with_lease(naive_now + timedelta(minutes=5)).lease_is_active(now=naive_now) is True assert _lookup_with_lease(None).lease_is_active(now=naive_now) is False + + +@pytest.mark.asyncio +async def test_durable_bridge_claim_survives_a_release_committing_mid_claim( + coordinator: DurableBridgeSessionCoordinator, + async_session_factory: Callable[[], AsyncSession], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Deterministic reproduction of the #1695 CI flake. + + SQLite's with_for_update is a no-op, so the retiring predecessor's fenced + release can commit between the successor claim's SELECT and its write. + Before the fix, the claim mutated ORM attributes, SQLAlchemy omitted + fields whose values matched the stale read (owner unchanged on a single + instance), the release's owner=None/state=CLOSED survived the claim's + commit, and the refresh handed the claimant a closed, ownerless row — + surfaced to the client as 409 bridge_instance_mismatch. + """ + predecessor = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-race", + api_key_id=None, + instance_id="instance-a", + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state="http_turn_1", + latest_response_id="resp_1", + allow_takeover=True, + ) + + import app.modules.proxy.durable_bridge_repository as repository_module + + real_writer_section = repository_module.sqlite_writer_section + release_injected = False + + @contextlib.asynccontextmanager + async def writer_section_with_interleaved_release(): + nonlocal release_injected + if not release_injected: + release_injected = True + # The predecessor's fenced release lands exactly between the + # successor claim's SELECT and its write. + async with async_session_factory() as release_session: + await DurableBridgeRepository(release_session).release_session( + session_id=predecessor.session_id, + instance_id="instance-a", + owner_epoch=predecessor.owner_epoch, + draining=False, + ) + async with real_writer_section(): + yield + + monkeypatch.setattr(repository_module, "sqlite_writer_section", writer_section_with_interleaved_release) + + successor = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-race", + api_key_id=None, + instance_id="instance-a", + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=False, + ) + + assert release_injected is True + assert successor.session_id == predecessor.session_id + # The claim's write must be authoritative over the interleaved release. + assert successor.owner_instance_id == "instance-a" + assert successor.state == HttpBridgeSessionState.ACTIVE + assert successor.owner_epoch == predecessor.owner_epoch + 1 + + +@pytest.mark.asyncio +async def test_durable_bridge_concurrent_successor_claims_serialize_on_the_epoch_cas( + coordinator: DurableBridgeSessionCoordinator, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Two successor claims can both read epoch N (with_for_update is a no-op + on SQLite). Without the compare-and-set, both would write N+1 and both + believe they own the row with colliding fences. The loser must retry + against fresh state and land on a distinct, higher epoch.""" + predecessor = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-cas", + api_key_id=None, + instance_id="instance-a", + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=True, + ) + + import app.modules.proxy.durable_bridge_repository as repository_module + + real_writer_section = repository_module.sqlite_writer_section + competitor_epochs: list[int] = [] + injected = False + + @contextlib.asynccontextmanager + async def writer_section_with_competing_claim(): + nonlocal injected + if not injected: + injected = True + # A competing successor claim commits between this claim's SELECT + # and its CAS write. + competitor = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-cas", + api_key_id=None, + instance_id="instance-a", + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=False, + ) + competitor_epochs.append(competitor.owner_epoch) + async with real_writer_section(): + yield + + monkeypatch.setattr(repository_module, "sqlite_writer_section", writer_section_with_competing_claim) + + loser_turned_winner = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-cas", + api_key_id=None, + instance_id="instance-a", + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=False, + ) + + assert competitor_epochs == [predecessor.owner_epoch + 1] + # The raced claim lost the CAS, retried against fresh state, and landed on + # its own distinct epoch above the competitor's. + assert loser_turned_winner.owner_instance_id == "instance-a" + assert loser_turned_winner.owner_epoch == competitor_epochs[0] + 1 + + +@pytest.mark.asyncio +async def test_durable_bridge_cas_loser_does_not_steal_a_foreign_winners_live_lease( + coordinator: DurableBridgeSessionCoordinator, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Two replicas recovering the same released row both enter with takeover + permission decided against that released state. Once one wins the CAS, the + loser re-reads a live foreign ACTIVE lease — reusing the stale permission + would steal it. The loser must fail closed and report the real owner, which + the bridge surfaces as the cross-replica retry response.""" + released = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-foreign-cas", + api_key_id=None, + instance_id="instance-old", + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=True, + ) + # The previous owner released: both recovering replicas legitimately see a + # takeover-eligible row. + await coordinator.release_live_session( + session_id=released.session_id, + instance_id="instance-old", + owner_epoch=released.owner_epoch, + draining=False, + ) + + import app.modules.proxy.durable_bridge_repository as repository_module + + real_writer_section = repository_module.sqlite_writer_section + injected = False + winner_epoch: list[int] = [] + + @contextlib.asynccontextmanager + async def writer_section_with_foreign_winner(): + nonlocal injected + if not injected: + injected = True + winner = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-foreign-cas", + api_key_id=None, + instance_id="instance-b", + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=True, + ) + winner_epoch.append(winner.owner_epoch) + async with real_writer_section(): + yield + + monkeypatch.setattr(repository_module, "sqlite_writer_section", writer_section_with_foreign_winner) + + loser = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-foreign-cas", + api_key_id=None, + instance_id="instance-a", + owner_process_epoch="test-process", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=True, + ) + + assert winner_epoch, "the foreign winner must have claimed first" + # The loser reports the winner as owner instead of stealing the live lease. + assert loser.owner_instance_id == "instance-b" + assert loser.owner_epoch == winner_epoch[0] + assert loser.state == HttpBridgeSessionState.ACTIVE diff --git a/tests/unit/test_file_pin_repository.py b/tests/unit/test_file_pin_repository.py new file mode 100644 index 0000000000..cc79dc13a6 --- /dev/null +++ b/tests/unit/test_file_pin_repository.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import pytest +from sqlalchemy import String + +from app.db.models import FileAccountPin +from app.modules.proxy.file_pin_repository import ( + build_file_account_pin_claim, + build_file_account_pin_cleanup, + build_file_account_pin_live_lookup, + build_file_account_pin_refresh, +) + +pytestmark = pytest.mark.unit + + +def test_file_account_pin_keeps_upstream_file_id_opaque() -> None: + file_id_type = FileAccountPin.__table__.c.file_id.type + assert isinstance(file_id_type, String) + assert file_id_type.length is None + + +def test_postgresql_file_pin_statements_use_current_database_clock() -> None: + claim_sql = str(build_file_account_pin_claim(dialect_name="postgresql")) + conflict_clause = claim_sql.split("DO UPDATE SET", 1)[1] + + assert claim_sql.count("clock_timestamp() + make_interval(secs => :ttl)") == 2 + assert "excluded.expires_at" not in conflict_clause + assert "expires_at = clock_timestamp() + make_interval(secs => :ttl)" in conflict_clause + assert "file_account_pins.expires_at <= clock_timestamp()" in conflict_clause + cleanup_sql = str(build_file_account_pin_cleanup(dialect_name="postgresql")) + assert "expires_at <= statement_timestamp()" in cleanup_sql + assert "clock_timestamp()" not in cleanup_sql + refresh_sql = str(build_file_account_pin_refresh(dialect_name="postgresql")) + assert "expires_at = clock_timestamp() + make_interval(secs => :ttl)" in refresh_sql + assert "file_id = :file_id" in refresh_sql + assert "account_id = :account_id" in refresh_sql + assert "RETURNING account_id" in refresh_sql + assert "expires_at > clock_timestamp()" in str(build_file_account_pin_live_lookup(dialect_name="postgresql")) + assert "expires_at > clock_timestamp()" in str( + build_file_account_pin_live_lookup(dialect_name="postgresql", many=True) + ) + + +def test_sqlite_file_pin_statements_use_padded_statement_clock() -> None: + claim_sql = str(build_file_account_pin_claim(dialect_name="sqlite")) + sqlite_now = "(strftime('%Y-%m-%d %H:%M:%f', 'now') || '000')" + sqlite_now_plus_ttl = "(strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || :ttl || ' seconds') || '000')" + conflict_clause = claim_sql.split("DO UPDATE SET", 1)[1] + + assert claim_sql.count(sqlite_now_plus_ttl) == 2 + assert "excluded.expires_at" not in conflict_clause + assert f"expires_at = {sqlite_now_plus_ttl}" in conflict_clause + assert f"file_account_pins.expires_at <= {sqlite_now}" in conflict_clause + assert sqlite_now in str(build_file_account_pin_cleanup(dialect_name="sqlite")) + refresh_sql = str(build_file_account_pin_refresh(dialect_name="sqlite")) + assert sqlite_now_plus_ttl in refresh_sql + assert "file_id = :file_id" in refresh_sql + assert "account_id = :account_id" in refresh_sql + assert "RETURNING account_id" in refresh_sql + assert sqlite_now in str(build_file_account_pin_live_lookup(dialect_name="sqlite")) + assert sqlite_now in str(build_file_account_pin_live_lookup(dialect_name="sqlite", many=True)) + assert "CURRENT_TIMESTAMP" not in claim_sql + + +@pytest.mark.parametrize( + "builder", + [ + build_file_account_pin_claim, + build_file_account_pin_cleanup, + build_file_account_pin_refresh, + build_file_account_pin_live_lookup, + ], +) +def test_file_pin_statement_builders_reject_unknown_dialect(builder) -> None: + with pytest.raises(RuntimeError, match="Unsupported database dialect"): + builder(dialect_name="mysql") diff --git a/tests/unit/test_generate_codex_client_evidence.py b/tests/unit/test_generate_codex_client_evidence.py deleted file mode 100644 index 47ee141f24..0000000000 --- a/tests/unit/test_generate_codex_client_evidence.py +++ /dev/null @@ -1,256 +0,0 @@ -from __future__ import annotations - -import importlib.util -import json -import stat -import sys -from pathlib import Path -from types import ModuleType -from typing import Any - -import pytest - - -def _load_generator() -> ModuleType: - script = Path(__file__).parents[2] / "scripts" / "generate_codex_client_evidence.py" - spec = importlib.util.spec_from_file_location("generate_codex_client_evidence", script) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -generator = _load_generator() -TASK_ID = "01a00000-0000-7000-8000-000000000001" -FIXTURE = Path(__file__).parents[1] / "fixtures" / "codex_client_evidence" / "sanitized_terminal.jsonl" - - -def _record(record_type: str, payload_type: str, **payload: object) -> dict[str, Any]: - return {"type": record_type, "payload": {"type": payload_type, **payload}} - - -def _valid_records(*, terminal_error: object = None) -> list[dict[str, Any]]: - records = [json.loads(line) for line in FIXTURE.read_text().splitlines()] - records[-2]["payload"]["error"] = terminal_error - return records - - -def _write_jsonl(path: Path, records: list[dict[str, Any]], *, newline: bool = True) -> bytes: - data = b"\n".join(json.dumps(record, separators=(",", ":")).encode() for record in records) - if newline: - data += b"\n" - path.write_bytes(data) - return data - - -def _snapshot(path: Path): - return generator._read_once(path) - - -def test_generates_content_free_client_domain_only(tmp_path: Path) -> None: - jsonl = tmp_path / "session.jsonl" - raw = _write_jsonl(jsonl, _valid_records()) - - evidence = generator.generate_evidence(_snapshot(jsonl), TASK_ID) - encoded = json.dumps(evidence, sort_keys=True, separators=(",", ":")) - - assert evidence["schema"] == "qk_codex_client_evidence_v2" - assert evidence["content_free"] is True - assert evidence["server_challenge_fields_included"] is False - assert evidence["remote_session_jsonl_size_bytes"] == len(raw) - assert evidence["remote_session_jsonl_last_offset"] == len(raw) - assert evidence["full_checkpoint_tool_ledger_event_count"] == 2 - assert evidence["unresolved_count"] == 0 - assert evidence["terminal"] == { - "last_task_started_line": 3, - "last_task_complete_line": 8, - "post_terminal_item_completed_count": 1, - "error_terminal": False, - "error": {"present": False, "class": "none", "code_digest": None, "message_digest": None}, - "latest_turn_assistant_output_count": 1, - "latest_turn_tool_call_count": 1, - "latest_turn_tool_output_count": 1, - } - for secret in ( - "SANITIZED_USER_TEXT", - "SANITIZED_ASSISTANT_TEXT", - "RAW_CALL_ID_MUST_NOT_ESCAPE", - "RAW_ARGUMENT_MUST_NOT_ESCAPE", - "RAW_OUTPUT_MUST_NOT_ESCAPE", - ): - assert secret not in encoded - for server_field in ( - "captured_input_item_count", - "captured_input_fingerprint", - "non_input_contract_fingerprint", - "retained_request_direct_call_ledger_digest", - "captured_projected_payload_fingerprint", - "captured_actual_wire_fingerprint", - "captured_request_binding_provenance", - ): - assert server_field not in evidence - - -def test_cli_create_new_mode_600_and_refuses_overwrite(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - jsonl = tmp_path / "session.jsonl" - output = tmp_path / "receipt.json" - _write_jsonl(jsonl, _valid_records()) - - arguments = [ - "--jsonl", - str(jsonl), - "--task-id", - TASK_ID, - "--output", - str(output), - "--stability-delay-ms", - "0", - ] - assert generator.main(arguments) == 0 - assert stat.S_IMODE(output.stat().st_mode) == 0o600 - stdout = capsys.readouterr().out - assert f"evidence_path={output}" in stdout - assert "evidence_sha256=" in stdout - - assert generator.main(arguments) == 2 - assert "ERROR output_exists" in capsys.readouterr().err - - -@pytest.mark.parametrize( - ("mutate", "expected"), - [ - (lambda records: records[:-1], None), - (lambda records: [*records[:-2], records[-1]], "latest_turn_not_terminal"), - ( - lambda records: [ - record for record in records if record.get("payload", {}).get("type") != "custom_tool_call_output" - ], - "tool_ledger_unresolved", - ), - ( - lambda records: [ - *records[:-2], - _record( - "response_item", - "custom_tool_call_output", - call_id="ORPHAN_CALL_ID", - output="SANITIZED", - ), - *records[-2:], - ], - "tool_ledger_unresolved", - ), - ( - lambda records: [*records[:6], records[5], *records[6:]], - "tool_ledger_unresolved", - ), - ( - lambda records: [*records[:7], records[6], *records[7:]], - "tool_ledger_unresolved", - ), - ( - lambda records: [ - *records[:6], - _record( - "response_item", - "function_call_output", - call_id="RAW_CALL_ID_MUST_NOT_ESCAPE", - output="SANITIZED", - ), - *records[7:], - ], - "tool_ledger_unresolved", - ), - ( - lambda records: [*records, _record("response_item", "message", role="assistant", content="LATE")], - "unsupported_post_terminal_event", - ), - ], - ids=( - "no_post_terminal_event", - "missing_task_complete", - "pending_call", - "orphan_output", - "duplicate_call_id", - "duplicate_output", - "call_output_type_mismatch", - "unsupported_event_after_terminal", - ), -) -def test_fail_closed_structural_cases( - tmp_path: Path, - mutate, - expected: str | None, -) -> None: - records = mutate(_valid_records()) - jsonl = tmp_path / "session.jsonl" - _write_jsonl(jsonl, records) - if expected is None: - # Removing only the permitted post-terminal event remains a valid terminal prefix. - assert generator.generate_evidence(_snapshot(jsonl), TASK_ID)["unresolved_count"] == 0 - else: - with pytest.raises(generator.EvidenceError, match=expected): - generator.generate_evidence(_snapshot(jsonl), TASK_ID) - - -def test_refuses_incomplete_newline_and_wrong_task(tmp_path: Path) -> None: - jsonl = tmp_path / "session.jsonl" - _write_jsonl(jsonl, _valid_records(), newline=False) - with pytest.raises(generator.EvidenceError, match="jsonl_missing_complete_newline"): - generator.generate_evidence(_snapshot(jsonl), TASK_ID) - - _write_jsonl(jsonl, _valid_records()) - with pytest.raises(generator.EvidenceError, match="session_meta_task_mismatch"): - generator.generate_evidence(_snapshot(jsonl), "01a00000-0000-7000-8000-000000000099") - - -def test_refuses_subagent_source_and_mixed_transport_identity(tmp_path: Path) -> None: - jsonl = tmp_path / "session.jsonl" - records = _valid_records() - records[0]["payload"]["source"] = {"subagent": "replacement"} - _write_jsonl(jsonl, records) - with pytest.raises(generator.EvidenceError, match="non_root_session_source_unsupported"): - generator.generate_evidence(_snapshot(jsonl), TASK_ID) - - records = _valid_records() - records[1]["payload"]["thread_id"] = "different-thread" - _write_jsonl(jsonl, records) - with pytest.raises(generator.EvidenceError, match="root_transport_identity_not_derivable"): - generator.generate_evidence(_snapshot(jsonl), TASK_ID) - - -def test_terminal_error_is_hashed_without_copying_text(tmp_path: Path) -> None: - jsonl = tmp_path / "session.jsonl" - _write_jsonl( - jsonl, - _valid_records(terminal_error={"code": "PRIVATE_CODE", "message": "PRIVATE_MESSAGE"}), - ) - evidence = generator.generate_evidence(_snapshot(jsonl), TASK_ID) - encoded = json.dumps(evidence) - error = evidence["terminal"]["error"] # type: ignore[index] - assert error["present"] is True - assert evidence["terminal"]["error_terminal"] is True # type: ignore[index] - assert error["class"] == "object" - assert error["code_digest"] - assert error["message_digest"] - assert "PRIVATE_CODE" not in encoded - assert "PRIVATE_MESSAGE" not in encoded - - -def test_stable_snapshot_refuses_drift(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - jsonl = tmp_path / "session.jsonl" - _write_jsonl(jsonl, _valid_records()) - first = _snapshot(jsonl) - second = generator.Snapshot( - first.data + b"x", - first.device, - first.inode, - first.size + 1, - first.mtime_ns + 1, - "0" * 64, - ) - snapshots = iter((first, second)) - monkeypatch.setattr(generator, "_read_once", lambda _path: next(snapshots)) - with pytest.raises(generator.EvidenceError, match="jsonl_not_stable_across_reads"): - generator.stable_snapshot(jsonl, 0) diff --git a/tests/unit/test_graceful_shutdown.py b/tests/unit/test_graceful_shutdown.py index b7046b2751..a99839e7f5 100644 --- a/tests/unit/test_graceful_shutdown.py +++ b/tests/unit/test_graceful_shutdown.py @@ -8,7 +8,12 @@ import pytest from app.core.shutdown import wait_for_tasks_to_drain -from app.main import InFlightMiddleware, _drain_detached_control_plane_tasks, _release_leader_lease_within +from app.main import ( + InFlightMiddleware, + _drain_detached_control_plane_tasks, + _drain_proxy_persistence_tasks, + _release_leader_lease_within, +) app_main = import_module("app.main") shutdown_state = import_module("app.core.shutdown") @@ -124,6 +129,30 @@ async def drain_fleet(_: float) -> bool: assert "Failed to drain audit log tasks during shutdown" in caplog.text +@pytest.mark.asyncio +async def test_lifespan_recovery_settlement_pre_drain_uses_remaining_deadline() -> None: + calls: list[dict[str, object]] = [] + + class _ProxyService: + async def drain_persistence_tasks(self, **kwargs: object) -> bool: + calls.append(kwargs) + return True + + assert await _drain_proxy_persistence_tasks( + _ProxyService(), + 3.25, + task_name_prefixes=("http-bridge-recovery-settlement-",), + failure_message="unused", + ) + + assert calls == [ + { + "timeout_seconds": 3.25, + "task_name_prefixes": ("http-bridge-recovery-settlement-",), + } + ] + + @pytest.mark.asyncio async def test_control_plane_drain_requires_stable_clean_pass( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_helm_monitoring_artifacts.py b/tests/unit/test_helm_monitoring_artifacts.py index 5d6afc1fb6..c6e52b209a 100644 --- a/tests/unit/test_helm_monitoring_artifacts.py +++ b/tests/unit/test_helm_monitoring_artifacts.py @@ -12,6 +12,48 @@ _REPO_ROOT = Path(__file__).resolve().parents[2] _CHART_DIR = _REPO_ROOT / "deploy" / "helm" / "codex-lb" +_POSTGRES_DATASOURCE_TYPE = "grafana-postgresql-datasource" + + +def _ttft_dashboard() -> dict: + return json.loads((_CHART_DIR / "dashboards" / "ttft-breakdown.json").read_text()) + + +def test_ttft_dashboard_declares_visible_single_select_postgres_datasource() -> None: + dashboard = _ttft_dashboard() + (datasource,) = dashboard["templating"]["list"] + + assert { + "name": datasource["name"], + "label": datasource["label"], + "type": datasource["type"], + "query": datasource["query"], + "hide": datasource["hide"], + "multi": datasource["multi"], + "includeAll": datasource["includeAll"], + } == { + "name": "DS_SQL", + "label": "PostgreSQL", + "type": "datasource", + "query": _POSTGRES_DATASOURCE_TYPE, + "hide": 0, + "multi": False, + "includeAll": False, + } + + +def test_ttft_dashboard_panels_bind_selected_postgres_datasource_uid() -> None: + dashboard = _ttft_dashboard() + + assert len(dashboard["panels"]) == 4 + assert all( + panel["datasource"] + == { + "type": _POSTGRES_DATASOURCE_TYPE, + "uid": "${DS_SQL}", + } + for panel in dashboard["panels"] + ) def test_high_error_rate_alert_aggregates_request_series_before_division() -> None: diff --git a/tests/unit/test_helm_replica_artifacts.py b/tests/unit/test_helm_replica_artifacts.py index 8b46a505ac..d11197ea53 100644 --- a/tests/unit/test_helm_replica_artifacts.py +++ b/tests/unit/test_helm_replica_artifacts.py @@ -164,6 +164,33 @@ def test_grafana_dashboard_titles_can_be_overridden() -> None: assert dashboard_config["data"]["ttft-breakdown.json"] == raw_dashboard_values["ttft-breakdown.json"] +def test_rendered_ttft_dashboard_keeps_runtime_postgres_datasource_binding() -> None: + rendered = _helm_template( + "--set", + "metrics.grafanaDashboard.enabled=true", + "--show-only", + "templates/grafana-dashboard.yaml", + ) + (dashboard_config,) = _helm_documents(rendered) + dashboard = json.loads(dashboard_config["data"]["ttft-breakdown.json"]) + (datasource,) = dashboard["templating"]["list"] + + assert datasource["name"] == "DS_SQL" + assert datasource["type"] == "datasource" + assert datasource["query"] == "grafana-postgresql-datasource" + assert datasource["hide"] == 0 + assert datasource["multi"] is False + assert datasource["includeAll"] is False + assert all( + panel["datasource"] + == { + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}", + } + for panel in dashboard["panels"] + ) + + def _prod_overlay_args(*args: str) -> tuple[str, ...]: return ( "-f", diff --git a/tests/unit/test_hot_path_caches.py b/tests/unit/test_hot_path_caches.py index dc1d115b77..ce7fbfeb9b 100644 --- a/tests/unit/test_hot_path_caches.py +++ b/tests/unit/test_hot_path_caches.py @@ -11,13 +11,11 @@ import pytest from fastapi import FastAPI from httpx import ASGITransport, AsyncClient -from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError import app.core.auth.dependencies as auth_dependencies import app.core.middleware.api_firewall as api_firewall_module from app.core.auth.api_key_cache import get_api_key_cache from app.core.crypto import TokenEncryptor -from app.core.handlers import add_exception_handlers from app.core.middleware.api_firewall import add_api_firewall_middleware from app.core.middleware.firewall_cache import get_firewall_ip_cache, reset_firewall_ip_cache_for_testing from app.db.models import Account, AccountStatus, UsageHistory @@ -71,7 +69,7 @@ async def _fake_session() -> AsyncIterator[object]: yield object() monkeypatch.setattr(auth_dependencies, "get_settings_cache", lambda: _SettingsCache()) - monkeypatch.setattr(auth_dependencies, "get_request_session", _fake_session) + monkeypatch.setattr(auth_dependencies, "get_background_session", _fake_session) monkeypatch.setattr(auth_dependencies, "ApiKeysRepository", lambda _session: object()) monkeypatch.setattr(auth_dependencies, "ApiKeysService", _Service) @@ -110,7 +108,7 @@ async def _fake_session() -> AsyncIterator[object]: "get_settings", lambda: SimpleNamespace(firewall_trusted_proxy_cidrs=[], firewall_trust_proxy_headers=False), ) - monkeypatch.setattr(api_firewall_module, "get_request_session", _fake_session) + monkeypatch.setattr(api_firewall_module, "get_background_session", _fake_session) monkeypatch.setattr(api_firewall_module, "FirewallRepository", lambda _session: object()) monkeypatch.setattr(api_firewall_module, "FirewallService", _Service) @@ -130,81 +128,6 @@ async def _v1_test() -> dict[str, str]: assert calls == 1 -@pytest.mark.asyncio -async def test_firewall_pool_timeout_returns_sanitized_retryable_503(monkeypatch: pytest.MonkeyPatch) -> None: - @asynccontextmanager - async def _timed_out_session() -> AsyncIterator[object]: - raise SQLAlchemyTimeoutError("private QueuePool topology") - yield object() - - monkeypatch.setattr( - api_firewall_module, - "get_settings", - lambda: SimpleNamespace(firewall_trusted_proxy_cidrs=[], firewall_trust_proxy_headers=False), - ) - monkeypatch.setattr(api_firewall_module, "get_request_session", _timed_out_session) - - app = FastAPI() - add_api_firewall_middleware(app) - - @app.get("/v1/test") - async def _v1_test() -> dict[str, str]: - return {"ok": "true"} - - transport = ASGITransport(app=app) - async with AsyncClient(transport=transport, base_url="http://testserver") as client: - response = await client.get("/v1/test") - - assert response.status_code == 503 - assert response.headers["Retry-After"] == "1" - assert response.json() == { - "error": { - "code": "database_pool_unavailable", - "message": "Database capacity is temporarily unavailable; retry shortly.", - "type": "server_error", - } - } - assert "QueuePool" not in response.text - - -@pytest.mark.asyncio -async def test_api_key_pool_timeout_uses_global_sanitized_retryable_503(monkeypatch: pytest.MonkeyPatch) -> None: - class _SettingsCache: - async def get(self) -> SimpleNamespace: - return SimpleNamespace(api_key_auth_enabled=True) - - @asynccontextmanager - async def _timed_out_session() -> AsyncIterator[object]: - raise SQLAlchemyTimeoutError("private QueuePool topology") - yield object() - - monkeypatch.setattr(auth_dependencies, "get_settings_cache", lambda: _SettingsCache()) - monkeypatch.setattr(auth_dependencies, "get_request_session", _timed_out_session) - - app = FastAPI() - add_exception_handlers(app) - - @app.get("/v1/test") - async def _v1_test() -> dict[str, str]: - await auth_dependencies.validate_proxy_api_key_authorization("Bearer sk-clb-private") - return {"ok": "true"} - - transport = ASGITransport(app=app) - async with AsyncClient(transport=transport, base_url="http://testserver") as client: - response = await client.get("/v1/test") - - assert response.status_code == 503 - assert response.headers["Retry-After"] == "1" - assert response.json() == { - "error": { - "code": "database_pool_unavailable", - "message": "Database capacity is temporarily unavailable; retry shortly.", - "type": "server_error", - } - } - assert "QueuePool" not in response.text - - @pytest.mark.asyncio async def test_account_selection_cache_reuses_inputs_and_invalidates_on_refresh() -> None: encryptor = TokenEncryptor() diff --git a/tests/unit/test_http_bridge_event_batcher.py b/tests/unit/test_http_bridge_event_batcher.py new file mode 100644 index 0000000000..bb6462b936 --- /dev/null +++ b/tests/unit/test_http_bridge_event_batcher.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from app.modules.proxy.http_bridge_event_batcher import HttpBridgeOperationEventBatcher + + +class _FakeDurableBridge: + def __init__(self, *, append_result: bool = True, update_result: bool = True) -> None: + self.append_result = append_result + self.update_result = update_result + self.batches: list[list[str]] = [] + self.finalized: list[str] = [] + self.updated: list[dict[str, object]] = [] + + async def append_operation_events(self, *, events, max_bytes: int) -> bool: + del max_bytes + self.batches.append([event.event_text for event in events]) + return self.append_result + + async def finalize_operation_event_spool(self, **kwargs) -> bool: + self.finalized.append(kwargs["operation_id"]) + return True + + async def update_operation(self, **kwargs) -> bool: + self.updated.append(kwargs) + return self.update_result + + async def settle_terminal_append_failure(self, **kwargs) -> bool: + kwargs["event_spool_complete"] = False + return await self.update_operation(**kwargs) + + +class _TerminalAppendFailingDurableBridge(_FakeDurableBridge): + def __init__(self, *, append_result: bool = True, update_result: bool = True) -> None: + super().__init__(append_result=append_result, update_result=update_result) + self.update_called = asyncio.Event() + + async def append_terminal_operation_event(self, **kwargs) -> bool: + del kwargs + raise RuntimeError("injected terminal append failure") + + async def update_operation(self, **kwargs) -> bool: + result = await super().update_operation(**kwargs) + self.update_called.set() + return result + + +async def _enqueue( + batcher: HttpBridgeOperationEventBatcher, + text: str, + *, + terminal: bool = False, +) -> None: + await batcher.enqueue( + operation_id="op-1", + session_id="session-1", + instance_id="instance-1", + owner_epoch=1, + event_text=text, + terminal=terminal, + ) + + +@pytest.mark.asyncio +async def test_batches_without_blocking_and_finalizes_terminal_event() -> None: + durable = _FakeDurableBridge() + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + batch_size=8, + flush_interval_seconds=0.01, + max_pending_events=32, + ) + try: + await _enqueue(batcher, "one") + await _enqueue(batcher, "two") + await _enqueue(batcher, "three", terminal=True) + assert durable.batches == [["one", "two", "three"]] + assert durable.finalized == ["op-1"] + finally: + await batcher.close() + + +@pytest.mark.asyncio +async def test_background_flushes_nonterminal_events_as_one_batch() -> None: + durable = _FakeDurableBridge() + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + batch_size=8, + flush_interval_seconds=0.01, + max_pending_events=32, + ) + try: + await _enqueue(batcher, "one") + await _enqueue(batcher, "two") + for _ in range(20): + if durable.batches: + break + await asyncio.sleep(0.01) + assert durable.batches == [["one", "two"]] + assert durable.finalized == [] + finally: + await batcher.close() + + +@pytest.mark.asyncio +async def test_dropped_batch_is_never_marked_replayable() -> None: + durable = _FakeDurableBridge(append_result=False) + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + batch_size=8, + flush_interval_seconds=0.01, + max_pending_events=32, + ) + try: + await _enqueue(batcher, "one") + for _ in range(20): + if durable.batches: + break + await asyncio.sleep(0.01) + result = await batcher.append_terminal_event( + operation_id="op-1", + session_id="session-1", + instance_id="instance-1", + owner_epoch=1, + event_text="terminal", + max_bytes=1024, + state="failed", + ) + assert result.persisted is False + assert result.settlement_required is False + assert durable.finalized == [] + assert durable.updated[0]["state"] == "failed" + assert batcher._contexts == {} + assert batcher._dropped_operations == set() + finally: + await batcher.close() + + +@pytest.mark.asyncio +async def test_terminal_append_failure_settles_operation() -> None: + durable = _TerminalAppendFailingDurableBridge() + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + flush_interval_seconds=60.0, + ) + + result = await batcher.append_terminal_event( + operation_id="op-1", + session_id="session-1", + instance_id="instance-1", + owner_epoch=7, + event_text="terminal", + max_bytes=1024, + state="failed", + response_id="resp-1", + ) + + assert result.persisted is False + assert result.settlement_required is True + await batcher.settle_terminal_event( + operation_id="op-1", + session_id="session-1", + instance_id="instance-1", + owner_epoch=7, + state="failed", + expected_response_id="resp-upstream-1", + response_id="resp-1", + ) + await asyncio.wait_for(durable.update_called.wait(), timeout=1.0) + assert durable.updated == [ + { + "operation_id": "op-1", + "session_id": "session-1", + "instance_id": "instance-1", + "owner_epoch": 7, + "state": "failed", + "expected_response_id": "resp-upstream-1", + "expected_recovery_dispatch_count": 0, + "alternate_expected_response_id": None, + "response_id": "resp-1", + "event_spool_complete": False, + } + ] + + +@pytest.mark.asyncio +async def test_terminal_append_failure_reports_fenced_settlement( + caplog: pytest.LogCaptureFixture, +) -> None: + durable = _TerminalAppendFailingDurableBridge(update_result=False) + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + flush_interval_seconds=60.0, + ) + + result = await batcher.append_terminal_event( + operation_id="op-1", + session_id="session-1", + instance_id="stale-instance", + owner_epoch=6, + event_text="terminal", + max_bytes=1024, + state="failed", + ) + + assert result.persisted is False + assert result.settlement_required is True + await batcher.settle_terminal_event( + operation_id="op-1", + session_id="session-1", + instance_id="stale-instance", + owner_epoch=6, + state="failed", + expected_response_id=None, + ) + await asyncio.wait_for(durable.update_called.wait(), timeout=1.0) + assert durable.updated[0]["owner_epoch"] == 6 + assert "fallback settlement was fenced operation_id=op-1" in caplog.text + + +@pytest.mark.asyncio +async def test_discard_operation_releases_partial_nonterminal_context() -> None: + durable = _FakeDurableBridge() + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + batch_size=8, + flush_interval_seconds=60.0, + max_pending_events=32, + ) + try: + await _enqueue(batcher, "partial") + await batcher.discard_operation(operation_id="op-1") + assert batcher._pending == {} + assert batcher._contexts == {} + assert batcher._pending_count == 0 + assert batcher._pending_bytes == 0 + assert durable.batches == [] + assert durable.finalized == [] + finally: + await batcher.close() + + +@pytest.mark.asyncio +async def test_close_cancels_background_flusher() -> None: + durable = _FakeDurableBridge() + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + batch_size=8, + flush_interval_seconds=60.0, + max_pending_events=32, + ) + await _enqueue(batcher, "one") + task = batcher._task + assert task is not None + + await batcher.close() + + assert batcher._task is None + assert task.done() diff --git a/tests/unit/test_http_bridge_forwarding.py b/tests/unit/test_http_bridge_forwarding.py index 983761c7ec..fb286368f1 100644 --- a/tests/unit/test_http_bridge_forwarding.py +++ b/tests/unit/test_http_bridge_forwarding.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio from collections.abc import AsyncIterator, Iterator from pathlib import Path from types import SimpleNamespace @@ -7,6 +8,7 @@ import aiohttp import pytest +from aiohttp.client_reqrep import ConnectionKey from app.core.config.settings import get_settings from app.core.openai.requests import ResponsesRequest @@ -1242,6 +1244,231 @@ def post(self, url: str, **kwargs: object) -> FakeResponse: assert aiohttp.hdrs.CONTENT_TYPE not in skip_auto_headers +def _connector_error() -> aiohttp.ClientConnectorError: + connection_key = ConnectionKey( + host="instance-b", + port=2455, + is_ssl=False, + ssl=False, + proxy=None, + proxy_auth=None, + proxy_headers_hash=None, + ) + return aiohttp.ClientConnectorError(connection_key, ConnectionRefusedError("connection refused")) + + +@pytest.mark.asyncio +async def test_owner_forward_connector_failure_does_not_mark_dispatched( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeResponse: + async def __aenter__(self) -> "FakeResponse": + raise _connector_error() + + async def __aexit__(self, exc_type, exc, tb) -> None: + return None + + class FakeSession: + def __init__(self, *, timeout: aiohttp.ClientTimeout, trust_env: bool) -> None: + del timeout, trust_env + + async def __aenter__(self) -> "FakeSession": + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + return None + + def post(self, url: str, **kwargs: object) -> FakeResponse: + del url, kwargs + return FakeResponse() + + monkeypatch.setattr("app.modules.proxy.http_bridge_forwarding.aiohttp.ClientSession", FakeSession) + monkeypatch.setattr("app.modules.proxy.http_bridge_forwarding.time.monotonic", lambda: 10.0) + dispatched = {"called": False} + + async def collect() -> None: + client = HTTPBridgeOwnerClient() + async for _event in client.stream_responses( + owner_endpoint="http://instance-b:2455", + payload=_payload(), + headers={"Authorization": "Bearer proxy-key"}, + context=HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ), + request_started_at=10.0, + on_request_dispatched=lambda: dispatched.__setitem__("called", True), + ): + return + + with pytest.raises(aiohttp.ClientConnectorError): + await collect() + assert dispatched["called"] is False + + +@pytest.mark.asyncio +async def test_owner_forward_midflight_transport_failure_marks_dispatched( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeResponse: + async def __aenter__(self) -> "FakeResponse": + raise aiohttp.ClientError("connection reset after request") + + async def __aexit__(self, exc_type, exc, tb) -> None: + return None + + class FakeSession: + def __init__(self, *, timeout: aiohttp.ClientTimeout, trust_env: bool) -> None: + del timeout, trust_env + + async def __aenter__(self) -> "FakeSession": + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + return None + + def post(self, url: str, **kwargs: object) -> FakeResponse: + del url, kwargs + return FakeResponse() + + monkeypatch.setattr("app.modules.proxy.http_bridge_forwarding.aiohttp.ClientSession", FakeSession) + monkeypatch.setattr("app.modules.proxy.http_bridge_forwarding.time.monotonic", lambda: 10.0) + dispatched = {"called": False} + + async def collect() -> None: + client = HTTPBridgeOwnerClient() + async for _event in client.stream_responses( + owner_endpoint="http://instance-b:2455", + payload=_payload(), + headers={"Authorization": "Bearer proxy-key"}, + context=HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ), + request_started_at=10.0, + on_request_dispatched=lambda: dispatched.__setitem__("called", True), + ): + return + + with pytest.raises(aiohttp.ClientError): + await collect() + assert dispatched["called"] is True + + +@pytest.mark.asyncio +async def test_owner_forward_non_200_body_read_failure_keeps_rejected( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeResponse: + status = 502 + + async def __aenter__(self) -> "FakeResponse": + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + return None + + async def text(self) -> str: + raise aiohttp.ClientPayloadError("truncated owner error body") + + class FakeSession: + def __init__(self, *, timeout: aiohttp.ClientTimeout, trust_env: bool) -> None: + del timeout, trust_env + + async def __aenter__(self) -> "FakeSession": + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + return None + + def post(self, url: str, **kwargs: object) -> FakeResponse: + del url, kwargs + return FakeResponse() + + monkeypatch.setattr("app.modules.proxy.http_bridge_forwarding.aiohttp.ClientSession", FakeSession) + monkeypatch.setattr("app.modules.proxy.http_bridge_forwarding.time.monotonic", lambda: 10.0) + dispatched = {"called": False} + rejected = {"called": False} + + async def collect() -> None: + client = HTTPBridgeOwnerClient() + async for _event in client.stream_responses( + owner_endpoint="http://instance-b:2455", + payload=_payload(), + headers={"Authorization": "Bearer proxy-key"}, + context=HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ), + request_started_at=10.0, + on_request_dispatched=lambda: dispatched.__setitem__("called", True), + on_response_rejected=lambda: rejected.__setitem__("called", True), + ): + return + + with pytest.raises(aiohttp.ClientPayloadError): + await collect() + assert rejected["called"] is True + assert dispatched["called"] is False + + +@pytest.mark.asyncio +async def test_owner_forward_cancel_during_aenter_marks_dispatched( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeResponse: + async def __aenter__(self) -> "FakeResponse": + raise asyncio.CancelledError() + + async def __aexit__(self, exc_type, exc, tb) -> None: + return None + + class FakeSession: + def __init__(self, *, timeout: aiohttp.ClientTimeout, trust_env: bool) -> None: + del timeout, trust_env + + async def __aenter__(self) -> "FakeSession": + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + return None + + def post(self, url: str, **kwargs: object) -> FakeResponse: + del url, kwargs + return FakeResponse() + + monkeypatch.setattr("app.modules.proxy.http_bridge_forwarding.aiohttp.ClientSession", FakeSession) + monkeypatch.setattr("app.modules.proxy.http_bridge_forwarding.time.monotonic", lambda: 10.0) + dispatched = {"called": False} + + async def collect() -> None: + client = HTTPBridgeOwnerClient() + async for _event in client.stream_responses( + owner_endpoint="http://instance-b:2455", + payload=_payload(), + headers={"Authorization": "Bearer proxy-key"}, + context=HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ), + request_started_at=10.0, + on_request_dispatched=lambda: dispatched.__setitem__("called", True), + ): + return + + with pytest.raises(asyncio.CancelledError): + await collect() + assert dispatched["called"] is True + + def test_build_owner_forward_headers_strips_hop_by_hop_headers() -> None: payload = _payload() context = HTTPBridgeForwardContext( diff --git a/tests/unit/test_limit_warmup.py b/tests/unit/test_limit_warmup.py index bc439b7990..0f45c0afa4 100644 --- a/tests/unit/test_limit_warmup.py +++ b/tests/unit/test_limit_warmup.py @@ -43,13 +43,19 @@ def _usage( window: str = "primary", recorded_at: datetime | None = None, ) -> UsageHistory: + window_minutes = {"primary": 300, "secondary": 10_080, "monthly": 43_200}[window] + if recorded_at is None: + recorded_at = datetime.fromtimestamp( + reset_at - window_minutes * 60, + tz=timezone.utc, + ).replace(tzinfo=None) return UsageHistory( account_id=account_id, used_percent=used_percent, reset_at=reset_at, window=window, - window_minutes=300 if window == "primary" else 10_080, - recorded_at=recorded_at or utcnow(), + window_minutes=window_minutes, + recorded_at=recorded_at, ) @@ -387,8 +393,16 @@ async def send(self, account: Account, *, model: str, prompt: str) -> LimitWarmu class _WarmupAccountsRepo: - def __init__(self, session: object | None = None) -> None: + def __init__(self, session: object | None = None, *, account: Account | None = None) -> None: self.session = session or object() + self.account = account + self.fresh_reads = 0 + + async def get_by_id_fresh(self, account_id: str) -> Account | None: + self.fresh_reads += 1 + if self.account is None or self.account.id != account_id: + return None + return self.account class _WarmupAccountsRepoContext: @@ -438,6 +452,70 @@ async def stream(*args: object, **kwargs: object): assert calls["stream_kwargs"]["route_trace"].endpoint_id is None +@pytest.mark.asyncio +async def test_streaming_limit_warmup_sender_rejects_ineligible_account_before_auth( + monkeypatch: pytest.MonkeyPatch, +) -> None: + selected = _account(status=AccountStatus.RATE_LIMITED) + current = _account(status=AccountStatus.RATE_LIMITED) + repo = _WarmupAccountsRepo(account=current) + sender = StreamingLimitWarmupSender(cast(Any, repo)) + + async def ensure_fresh(_account: Account) -> Account: + raise AssertionError("OAuth refresh must not run for an ineligible account") + + monkeypatch.setattr(sender._auth_manager, "ensure_fresh", ensure_fresh) + + result = await sender.send(selected, model="gpt-5.2", prompt="Say OK.") + + assert result.error_code == "account_not_active" + assert repo.fresh_reads == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("selected_status", "current_status", "current_enabled"), + [ + (AccountStatus.ACTIVE, AccountStatus.RATE_LIMITED, True), + (AccountStatus.ACTIVE, AccountStatus.PAUSED, True), + (AccountStatus.ACTIVE, AccountStatus.ACTIVE, False), + ], +) +async def test_streaming_limit_warmup_sender_rechecks_account_after_auth( + monkeypatch: pytest.MonkeyPatch, + selected_status: AccountStatus, + current_status: AccountStatus, + current_enabled: bool, +) -> None: + selected = _account(status=selected_status) + repo = _WarmupAccountsRepo(account=selected) + sender = StreamingLimitWarmupSender(cast(Any, repo)) + stream_calls = 0 + + async def ensure_fresh(target: Account) -> Account: + repo.account = _account(status=current_status, enabled=current_enabled) + return target + + async def resolve_route(_account: Account) -> None: + return None + + async def stream(*_args: object, **_kwargs: object): + nonlocal stream_calls + stream_calls += 1 + yield 'data: {"type":"response.completed","response":{"id":"resp_1"}}\n\n' + + monkeypatch.setattr(sender._auth_manager, "ensure_fresh", ensure_fresh) + monkeypatch.setattr(sender, "_resolve_upstream_route", resolve_route) + monkeypatch.setattr(sender._encryptor, "decrypt", lambda value: "access") + monkeypatch.setattr(limit_warmup_service, "stream_responses", stream) + + result = await sender.send(selected, model="gpt-5.2", prompt="Say OK.") + + assert result.error_code == "account_not_active" + assert repo.fresh_reads == 2 + assert stream_calls == 0 + + @pytest.mark.asyncio async def test_streaming_limit_warmup_sender_resolves_route_with_owned_repo_factory( monkeypatch: pytest.MonkeyPatch, @@ -584,7 +662,7 @@ async def test_reset_confirmed_candidate_sends_one_warmup() -> None: @pytest.mark.asyncio -async def test_exhausted_threshold_accepts_pre_reset_99_percent_usage() -> None: +async def test_reset_warms_after_pre_reset_99_percent_usage() -> None: repo = FakeWarmupRepo() logs = FakeRequestLogsRepo() sender = FakeSender() @@ -606,7 +684,7 @@ async def test_exhausted_threshold_accepts_pre_reset_99_percent_usage() -> None: @pytest.mark.asyncio -async def test_exhausted_threshold_skips_usage_below_threshold() -> None: +async def test_reset_warms_regardless_of_pre_reset_usage() -> None: repo = FakeWarmupRepo() sender = FakeSender() service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) @@ -615,12 +693,152 @@ async def test_exhausted_threshold_skips_usage_below_threshold() -> None: await service.run_after_usage_refresh( accounts=[account], settings=_settings(limit_warmup_exhausted_threshold_percent=99.0), - before_primary={account.id: _usage(account.id, used_percent=98.9, reset_at=1000)}, + before_primary={account.id: _usage(account.id, used_percent=15.0, reset_at=1000)}, before_secondary={}, after_primary={account.id: _usage(account.id, used_percent=0, reset_at=2000)}, after_secondary={}, ) + assert sender.calls == [(account.id, "gpt-5.1-codex-mini")] + assert [(row.window, row.reset_at, row.status) for row in repo.rows] == [("primary", 2000, "succeeded")] + + +@pytest.mark.asyncio +async def test_early_reset_reanchor_uses_sampling_interval() -> None: + repo = FakeWarmupRepo() + sender = FakeSender() + service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) + account = _account() + before_recorded_at = datetime.fromtimestamp(900, tz=timezone.utc).replace(tzinfo=None) + observed_at = datetime.fromtimestamp(1201, tz=timezone.utc).replace(tzinfo=None) + + await service.run_after_usage_refresh( + accounts=[account], + settings=_settings(), + before_primary={ + account.id: _usage( + account.id, + used_percent=23.0, + reset_at=10_000, + recorded_at=before_recorded_at, + ) + }, + before_secondary={}, + after_primary={ + account.id: _usage( + account.id, + used_percent=0, + reset_at=19_000, + recorded_at=observed_at, + ) + }, + after_secondary={}, + ) + + assert sender.calls == [(account.id, "gpt-5.1-codex-mini")] + assert [(row.window, row.reset_at, row.status) for row in repo.rows] == [("primary", 19_000, "succeeded")] + + +@pytest.mark.asyncio +async def test_full_window_reset_warms_when_usage_was_already_zero() -> None: + repo = FakeWarmupRepo() + sender = FakeSender() + service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) + account = _account() + + await service.run_after_usage_refresh( + accounts=[account], + settings=_settings(), + before_primary={account.id: _usage(account.id, used_percent=0, reset_at=1000)}, + before_secondary={}, + after_primary={account.id: _usage(account.id, used_percent=0, reset_at=19_000)}, + after_secondary={}, + ) + + assert sender.calls == [(account.id, "gpt-5.1-codex-mini")] + assert [(row.window, row.reset_at, row.status) for row in repo.rows] == [("primary", 19_000, "succeeded")] + + +@pytest.mark.asyncio +async def test_sliding_reset_at_without_quota_recovery_does_not_warm() -> None: + repo = FakeWarmupRepo() + sender = FakeSender() + service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) + account = _account() + + await service.run_after_usage_refresh( + accounts=[account], + settings=_settings(), + before_primary={account.id: _usage(account.id, used_percent=0, reset_at=1000)}, + before_secondary={}, + after_primary={account.id: _usage(account.id, used_percent=0, reset_at=1120)}, + after_secondary={}, + ) + + assert sender.calls == [] + assert repo.rows == [] + + +@pytest.mark.asyncio +async def test_future_full_window_slide_without_quota_recovery_does_not_warm() -> None: + repo = FakeWarmupRepo() + sender = FakeSender() + service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) + account = _account() + observed_at = datetime.fromtimestamp(0, tz=timezone.utc).replace(tzinfo=None) + + await service.run_after_usage_refresh( + accounts=[account], + settings=_settings(), + before_primary={account.id: _usage(account.id, used_percent=0, reset_at=1000)}, + before_secondary={}, + after_primary={ + account.id: _usage( + account.id, + used_percent=0, + reset_at=19_000, + recorded_at=observed_at, + ) + }, + after_secondary={}, + ) + + assert sender.calls == [] + assert repo.rows == [] + + +@pytest.mark.asyncio +async def test_stale_past_boundary_update_without_quota_recovery_does_not_warm() -> None: + repo = FakeWarmupRepo() + sender = FakeSender() + service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) + account = _account() + before_recorded_at = datetime.fromtimestamp(2000, tz=timezone.utc).replace(tzinfo=None) + observed_at = datetime.fromtimestamp(2100, tz=timezone.utc).replace(tzinfo=None) + + await service.run_after_usage_refresh( + accounts=[account], + settings=_settings(), + before_primary={ + account.id: _usage( + account.id, + used_percent=0, + reset_at=1000, + recorded_at=before_recorded_at, + ) + }, + before_secondary={}, + after_primary={ + account.id: _usage( + account.id, + used_percent=0, + reset_at=2200, + recorded_at=observed_at, + ) + }, + after_secondary={}, + ) + assert sender.calls == [] assert repo.rows == [] @@ -631,7 +849,13 @@ async def test_warmup_request_log_persists_route_metadata() -> None: logs = FakeRequestLogsRepo() class RouteMetadataSender: - async def send(self, account: Account, *, model: str, prompt: str) -> LimitWarmupSendResult: + async def send( + self, + account: Account, + *, + model: str, + prompt: str, + ) -> LimitWarmupSendResult: del account, model, prompt return LimitWarmupSendResult( request_id="warmup-route", @@ -886,6 +1110,135 @@ async def test_monthly_free_quota_reset_warms_and_records_monthly_window() -> No assert [(row.window, row.reset_at, row.status) for row in repo.rows] == [("monthly", 2000, "succeeded")] +@pytest.mark.asyncio +async def test_confirmed_paid_to_free_transition_warms_fresh_monthly_window() -> None: + repo = FakeWarmupRepo() + sender = FakeSender() + service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) + account = _account() + account.plan_type = "free" + refresh_started_at = datetime(2026, 8, 18, 18, 8, tzinfo=timezone.utc).replace(tzinfo=None) + monthly_reset_at = int(refresh_started_at.replace(tzinfo=timezone.utc).timestamp()) + 43_200 * 60 + + await service.run_after_usage_refresh( + accounts=[account], + settings=_settings(limit_warmup_windows="secondary"), + before_primary={}, + before_secondary={account.id: _usage(account.id, used_percent=37, reset_at=10_000, window="secondary")}, + after_primary={}, + after_secondary={ + account.id: _usage( + account.id, + used_percent=0, + reset_at=monthly_reset_at, + window="monthly", + recorded_at=refresh_started_at, + ) + }, + previous_plan_types={account.id: "plus"}, + refresh_started_at=refresh_started_at, + ) + + assert sender.calls == [(account.id, "gpt-5.1-codex-mini")] + assert [(row.window, row.reset_at, row.status) for row in repo.rows] == [("monthly", monthly_reset_at, "succeeded")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("previous_plan_type", "current_plan_type", "sample_age_seconds", "used_percent", "minimum_available"), + [ + ("free", "free", 0, 0.0, 100.0), + ("plus", "plus", 0, 0.0, 100.0), + ("plus", "free", -1, 0.0, 100.0), + ("plus", "free", 0, 2.0, 99.0), + ("plus", "free", 0, 100.0, 100.0), + ], + ids=[ + "already-free", + "unconfirmed", + "stale-monthly", + "below-availability-gate", + "exhausted-monthly-at-default-gate", + ], +) +async def test_paid_to_free_transition_candidate_rejects_unsafe_evidence( + previous_plan_type: str, + current_plan_type: str, + sample_age_seconds: int, + used_percent: float, + minimum_available: float, +) -> None: + repo = FakeWarmupRepo() + sender = FakeSender() + service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) + account = _account() + account.plan_type = current_plan_type + refresh_started_at = datetime(2026, 8, 18, 18, 8, tzinfo=timezone.utc).replace(tzinfo=None) + recorded_at = refresh_started_at + timedelta(seconds=sample_age_seconds) + + await service.run_after_usage_refresh( + accounts=[account], + settings=_settings( + limit_warmup_windows="secondary", + limit_warmup_min_available_percent=minimum_available, + ), + before_primary={}, + before_secondary={}, + after_primary={}, + after_secondary={ + account.id: _usage( + account.id, + used_percent=used_percent, + reset_at=2_000_000_000, + window="monthly", + recorded_at=recorded_at, + ) + }, + previous_plan_types={account.id: previous_plan_type}, + refresh_started_at=refresh_started_at, + ) + + assert sender.calls == [] + assert repo.rows == [] + + +@pytest.mark.asyncio +async def test_paid_to_free_transition_warmup_is_deduplicated_by_monthly_reset() -> None: + repo = FakeWarmupRepo() + sender = FakeSender() + service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) + account = _account() + account.plan_type = "free" + refresh_started_at = datetime(2026, 8, 18, 18, 8, tzinfo=timezone.utc).replace(tzinfo=None) + after_secondary = { + account.id: _usage( + account.id, + used_percent=0, + reset_at=2_000_000_000, + window="monthly", + recorded_at=refresh_started_at, + ) + } + + async def run_once() -> None: + await service.run_after_usage_refresh( + accounts=[account], + settings=_settings(limit_warmup_windows="secondary"), + before_primary={}, + before_secondary={}, + after_primary={}, + after_secondary=after_secondary, + previous_plan_types={account.id: "pro"}, + refresh_started_at=refresh_started_at, + ) + + await run_once() + await run_once() + + assert sender.calls == [(account.id, "gpt-5.1-codex-mini")] + assert [(row.window, row.reset_at) for row in repo.rows] == [("monthly", 2_000_000_000)] + + @pytest.mark.asyncio async def test_long_window_warmup_ignores_cross_window_transition() -> None: repo = FakeWarmupRepo() @@ -1426,6 +1779,28 @@ async def test_regular_warmup_ignores_reset_at_jitter(monkeypatch) -> None: assert repo.rows == [] +@pytest.mark.asyncio +async def test_regular_warmup_dedupes_same_reset_across_after_reset_at_jitter() -> None: + repo = FakeWarmupRepo() + sender = FakeSender() + service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) + account = _account("acc_1") + settings = _settings(limit_warmup_cooldown_seconds=0) + + for reset_at in (18_000, 18_001): + await service.run_after_usage_refresh( + accounts=[account], + settings=settings, + before_primary={account.id: _usage(account.id, used_percent=98.0, reset_at=1000)}, + before_secondary={}, + after_primary={account.id: _usage(account.id, used_percent=0.0, reset_at=reset_at)}, + after_secondary={}, + ) + + assert sender.calls == [(account.id, "gpt-5.1-codex-mini")] + assert [(row.window, row.reset_at) for row in repo.rows] == [("primary", 18_000)] + + @pytest.mark.asyncio async def test_regular_warmup_boundary_59_seconds_rejected(monkeypatch) -> None: """A reset_at jump of exactly 59 seconds must NOT trigger a warm-up.""" @@ -1592,7 +1967,7 @@ async def test_staggered_idle_warmup_rejects_stale_entry_from_prior_cycle(monkey @pytest.mark.asyncio -async def test_recent_attempt_cooldown_blocks_new_reset() -> None: +async def test_recent_attempt_cooldown_does_not_block_distinct_reset() -> None: repo = FakeWarmupRepo() account = _account() repo.rows.append( @@ -1606,6 +1981,7 @@ async def test_recent_attempt_cooldown_blocks_new_reset() -> None: attempted_at=utcnow() - timedelta(minutes=10), ) ) + repo.next_id = 2 sender = FakeSender() service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) @@ -1618,5 +1994,5 @@ async def test_recent_attempt_cooldown_blocks_new_reset() -> None: after_secondary={}, ) - assert sender.calls == [] - assert len(repo.rows) == 1 + assert sender.calls == [(account.id, "gpt-5.1-codex-mini")] + assert [(row.reset_at, row.status) for row in repo.rows] == [(1000, "failed"), (3000, "succeeded")] diff --git a/tests/unit/test_live_ingest_leak_fence.py b/tests/unit/test_live_ingest_leak_fence.py new file mode 100644 index 0000000000..63d3b585c9 --- /dev/null +++ b/tests/unit/test_live_ingest_leak_fence.py @@ -0,0 +1,174 @@ +"""Regression tests for issue #1755: cross-test live-usage-ingestor leakage. + +The unit suite runs on a session-scoped asyncio loop, so a background task +leaked by one test survives into every later test. Tests that enter the real +app lifespan start the module-global live-usage ingestor; when the lifespan is +cancelled before its shutdown path reaches ``stop_live_usage_ingestor()`` +(e.g. a ``wait_for``-bounded assertion times out mid-drain), the +``live-usage-ingestor`` consumer task outlives the test and later poisons the +otel lifespan-drain test and test_proxy_utils' startup-probe loop-exception +assertions. + +The first two tests are ORDER-DEPENDENT by design (pytest runs them in +definition order): the first reproduces the leak by starting the ingestor +singleton exactly like the app lifespan does and deliberately never stopping +it; the second asserts the autouse ``_stop_leaked_live_usage_ingestor`` fence +in tests/conftest.py reclaimed the consumer at the previous test's boundary. +Without the fence the second test fails with a pending +``live-usage-ingestor`` task — the coupling observed in #1755. +""" + +from __future__ import annotations + +import asyncio +import time + +import pytest + +from app.core.usage import live_hub +from app.modules.usage import live_ingest + + +def _pending_ingestor_tasks() -> list[asyncio.Task[object]]: + return [task for task in asyncio.all_tasks() if not task.done() and task.get_name() == "live-usage-ingestor"] + + +async def test_abandoned_ingestor_simulates_lifespan_cancelled_before_stop() -> None: + # This is exactly what app.main's lifespan startup does; a lifespan + # cancelled mid-shutdown-drain never reaches stop_live_usage_ingestor(), + # so nothing in this test stops the singleton either. The autouse fence + # in tests/conftest.py must reclaim it at this test's boundary. + ingestor = live_ingest.start_live_usage_ingestor() + + assert ingestor is not None + assert live_ingest._ingestor is ingestor + assert live_hub._publisher is not None + assert len(_pending_ingestor_tasks()) == 1 + + +async def test_fence_reclaims_leaked_consumer_at_test_boundary() -> None: + assert _pending_ingestor_tasks() == [] + assert live_ingest._ingestor is None + assert live_hub._publisher is None + + +async def test_reap_settles_and_reports_already_failed_consumer(monkeypatch: pytest.MonkeyPatch) -> None: + # A leaked consumer can already be dead with an exception by the time the + # fence runs (#1755 observed RuntimeError('cannot reuse already awaited + # coroutine')). The fence must retrieve that exception — so it neither + # crashes mid-cleanup nor resurfaces later as an unobserved-task loop + # exception in an unrelated test — and report it exactly once even though + # both the done callback and the fence sweep observe the dead task. + from tests import conftest as suite_conftest + + async def _boom(self: live_ingest.LiveUsageIngestor) -> None: + raise RuntimeError("cannot reuse already awaited coroutine") + + monkeypatch.setattr(live_ingest.LiveUsageIngestor, "_run", _boom) + ingestor = live_ingest.LiveUsageIngestor(queue_size=1, write_min_interval_seconds=0.0) + ingestor.start() + live_ingest._ingestor = ingestor + live_hub.register_live_usage_publisher(ingestor.publish) + await asyncio.sleep(0) + assert ingestor._consumer is not None and ingestor._consumer.done() + + await suite_conftest._reap_leaked_live_usage_ingestor() + failures = suite_conftest._drain_live_ingest_task_failures() + + assert failures == ["'live-usage-ingestor' died with RuntimeError('cannot reuse already awaited coroutine')"] + assert live_ingest._ingestor is None + assert live_hub._publisher is None + assert _pending_ingestor_tasks() == [] + # Settlement is exactly-once: a second pass reports nothing. + assert suite_conftest._drain_live_ingest_task_failures() == [] + + +async def test_reap_sweeps_orphaned_tasks_not_tracked_by_singleton() -> None: + # A stop that is itself cancelled between clearing the module global and + # awaiting the ingestor's tasks leaves pending tasks no singleton tracks; + # the reap's name-based sweep must still cancel and await both the + # consumer and the trailing cache-invalidation sleeper. + from tests import conftest as suite_conftest + + async def _pending_forever() -> None: + await asyncio.Event().wait() + + consumer = asyncio.create_task(_pending_forever(), name="live-usage-ingestor") + trailing = asyncio.create_task(_pending_forever(), name="live-usage-trailing-invalidation") + await asyncio.sleep(0) + assert live_ingest._ingestor is None + + await suite_conftest._reap_leaked_live_usage_ingestor() + + assert consumer.cancelled() + assert trailing.cancelled() + assert suite_conftest._drain_live_ingest_task_failures() == [] + assert suite_conftest._pending_live_ingest_tasks(asyncio.get_running_loop()) == [] + + +async def test_dead_detached_owned_task_failure_is_recorded_and_drained_loop_free( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # An ingestor-owned task can die with an exception after the singleton and + # its task fields are already cleared. asyncio.all_tasks() only returns + # unfinished tasks, so the pending sweep cannot see it — the done callback + # installed at task creation must have already retrieved the exception + # into the strong failure handoff, which the fence drains without running + # the event loop. + from tests import conftest as suite_conftest + + async def _boom(self: live_ingest.LiveUsageIngestor) -> None: + raise RuntimeError("late detached failure") + + monkeypatch.setattr(live_ingest.LiveUsageIngestor, "_run", _boom) + ingestor = live_ingest.LiveUsageIngestor(queue_size=1, write_min_interval_seconds=0.0) + ingestor.start() + await asyncio.sleep(0) + await asyncio.sleep(0) # let the done callback run + ingestor._consumer = None # fully detach: no owner, no pending task + del ingestor + assert live_ingest._ingestor is None + assert live_hub._publisher is None + assert suite_conftest._pending_live_ingest_tasks(asyncio.get_running_loop()) == [] + + failures = suite_conftest._drain_live_ingest_task_failures() + + assert failures == ["'live-usage-ingestor' died with RuntimeError('late detached failure')"] + assert suite_conftest._drain_live_ingest_task_failures() == [] + + +async def test_drain_settles_dead_task_whose_done_callback_has_not_run() -> None: + # A task that finishes in the loop's final iteration can still have its + # done callback queued when the sync fence runs; the drain's sweep over + # the weak ownership registry must settle it directly, and the callback + # running later must not report it a second time. + from tests import conftest as suite_conftest + + async def _boom() -> None: + raise RuntimeError("callback still queued") + + task = asyncio.create_task(_boom(), name="live-usage-trailing-invalidation") + live_ingest._owned_tasks.add(task) # enrolled, but callback never attached + await asyncio.sleep(0) + assert task.done() + + failures = suite_conftest._drain_live_ingest_task_failures() + assert failures == ["'live-usage-trailing-invalidation' died with RuntimeError('callback still queued')"] + + # The (simulated late) callback observes an already-settled task. + live_ingest._record_owned_task_result(task) + assert suite_conftest._drain_live_ingest_task_failures() == [] + + +async def test_ingestor_enrolls_both_task_types_in_the_ownership_registry() -> None: + # The registry only protects tests if production task creation actually + # enrolls both task types. + ingestor = live_ingest.LiveUsageIngestor(queue_size=1, write_min_interval_seconds=0.0) + ingestor.start() + ingestor._last_cache_invalidation = time.monotonic() + await ingestor._invalidate_caches_throttled() + + assert ingestor._consumer in live_ingest._owned_tasks + assert ingestor._trailing_invalidation in live_ingest._owned_tasks + + await ingestor.stop() diff --git a/tests/unit/test_live_snapshot_owner_relock.py b/tests/unit/test_live_snapshot_owner_relock.py new file mode 100644 index 0000000000..11d59f28ab --- /dev/null +++ b/tests/unit/test_live_snapshot_owner_relock.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, call + +import pytest + +from app.db.account_identity_lock import lock_postgresql_account_identities +from app.modules.usage import repository as usage_repository_module +from app.modules.usage.repository import ( + LiveSnapshotOwnerIdentityRelockError, + UsageRepository, + UsageWindowWrite, +) + +pytestmark = pytest.mark.unit + + +def _identity_result(account_id: str | None, chatgpt_account_id: str | None = None) -> MagicMock: + result = MagicMock() + if account_id is None: + result.one_or_none.return_value = None + else: + result.one_or_none.return_value = MagicMock( + id=account_id, + chatgpt_account_id=chatgpt_account_id, + ) + return result + + +def _postgresql_session(results: list[MagicMock]) -> MagicMock: + session = MagicMock() + session.get_bind.return_value.dialect.name = "postgresql" + session.execute = AsyncMock(side_effect=results) + session.add_all = MagicMock() + session.commit = AsyncMock() + session.rollback = AsyncMock() + return session + + +async def _settle(session: MagicMock) -> str | None: + return await UsageRepository(session).settle_live_account_snapshot( + account_id="acc-selected", + chatgpt_account_id="workspace-x", + windows=[UsageWindowWrite(window="primary", used_percent=25.0)], + should_skip=lambda _account_id: False, + ) + + +@pytest.mark.asyncio +async def test_postgresql_live_snapshot_same_identity_does_not_relock( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = _postgresql_session( + [ + _identity_result("acc-selected", "workspace-x"), + _identity_result("acc-selected", "workspace-x"), + ] + ) + identity_lock = AsyncMock() + monkeypatch.setattr(usage_repository_module, "lock_postgresql_account_identities", identity_lock) + monkeypatch.setattr(usage_repository_module, "relax_commit_durability", AsyncMock()) + + resolved = await _settle(session) + + assert resolved == "acc-selected" + identity_lock.assert_awaited_once_with(session, ("workspace-x",)) + session.rollback.assert_not_awaited() + session.commit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_postgresql_live_snapshot_relocks_once_for_current_owner_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = _postgresql_session( + [ + _identity_result("acc-selected", "workspace-y"), + _identity_result("acc-selected", "workspace-y"), + _identity_result("acc-selected", "workspace-y"), + ] + ) + identity_lock = AsyncMock() + monkeypatch.setattr(usage_repository_module, "lock_postgresql_account_identities", identity_lock) + monkeypatch.setattr(usage_repository_module, "relax_commit_durability", AsyncMock()) + + resolved = await _settle(session) + + assert resolved == "acc-selected" + assert identity_lock.await_args_list == [ + call(session, ("workspace-x",)), + call(session, ("workspace-x", "workspace-y")), + ] + session.rollback.assert_awaited_once() + session.add_all.assert_called_once() + session.commit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_postgresql_live_snapshot_second_owner_identity_change_is_terminal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = _postgresql_session( + [ + _identity_result("acc-selected", "workspace-y"), + _identity_result("acc-selected", "workspace-z"), + ] + ) + identity_lock = AsyncMock() + monkeypatch.setattr(usage_repository_module, "lock_postgresql_account_identities", identity_lock) + + with pytest.raises(LiveSnapshotOwnerIdentityRelockError): + await _settle(session) + + assert identity_lock.await_args_list == [ + call(session, ("workspace-x",)), + call(session, ("workspace-x", "workspace-y")), + ] + assert session.rollback.await_count == 2 + session.add_all.assert_not_called() + session.commit.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_postgresql_identity_lock_does_not_fabricate_none_key() -> None: + session = MagicMock() + session.get_bind.return_value.dialect.name = "postgresql" + session.execute = AsyncMock() + + lock_keys = await lock_postgresql_account_identities(session, (None,)) + + assert lock_keys == () + session.execute.assert_not_awaited() diff --git a/tests/unit/test_live_usage_ingest.py b/tests/unit/test_live_usage_ingest.py index c1108e56f3..c37335d451 100644 --- a/tests/unit/test_live_usage_ingest.py +++ b/tests/unit/test_live_usage_ingest.py @@ -281,7 +281,6 @@ async def fake_lease(session: Any = None): assert (account_id, chatgpt_account_id) == (None, "workspace-live") assert snapshot.primary is not None assert snapshot.primary.used_percent == pytest.approx(55.0) - # When the caller knows the selected internal account, attribution - # prefers it so multi-seat workspaces are not dropped as ambiguous. + # Local attribution stays preferred while retaining its recovery identity. _, account_id_internal, chatgpt_internal = captured[1] - assert (account_id_internal, chatgpt_internal) == ("acc-internal", None) + assert (account_id_internal, chatgpt_internal) == ("acc-internal", "workspace-live") diff --git a/tests/unit/test_load_balancer_concurrency.py b/tests/unit/test_load_balancer_concurrency.py index d07334c14b..09de146a6f 100644 --- a/tests/unit/test_load_balancer_concurrency.py +++ b/tests/unit/test_load_balancer_concurrency.py @@ -5,7 +5,7 @@ import time from collections.abc import AsyncIterator, Collection from contextlib import asynccontextmanager -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Any, Literal, cast from unittest.mock import AsyncMock @@ -29,7 +29,11 @@ from app.core.crypto import TokenEncryptor from app.db.models import Account, AccountStatus, StickySessionKind, UsageHistory from app.modules.api_keys.repository import ApiKeysRepository -from app.modules.proxy.affinity import _codex_session_selection_key +from app.modules.proxy.affinity import ( + _AffinityPolicy, + _codex_backend_identity, + _codex_session_selection_key, +) from app.modules.proxy.cap_partitioning import CapPartition from app.modules.proxy.load_balancer import LoadBalancer, RuntimeState, effective_account_concurrency_caps from app.modules.proxy.repo_bundle import ProxyRepositories @@ -222,32 +226,90 @@ def __init__(self) -> None: # abandoned so run_sticky_selection_path can bypass the # ambiguous-owner check for them. self.abandoned_keys: set[str] = set() + self.scoped_abandoned_account_ids_by_key: dict[str, str] = {} + # Refresh-skip deadlines reported alongside the owner lookup, keyed by + # sticky key (see StickyOwnerLookup.refresh_skip_deadline). + self.refresh_skip_deadlines_by_key: dict[str, datetime] = {} self.deleted: list[tuple[str, StickySessionKind | None]] = [] self.upserts: list[tuple[str, str, StickySessionKind | None]] = [] + self.insert_if_absent_calls: list[tuple[str, str, StickySessionKind]] = [] + self.seeded_upserts: list[tuple[str, str, StickySessionKind, str, StickySessionKind]] = [] async def get_account_id(self, *args: Any, **kwargs: Any) -> str | None: lookup = await self.get_account_id_and_abandonment(*args, **kwargs) return lookup.account_id + async def release_read_snapshot(self) -> None: + # The shared owner-lookup session releases its read snapshot between + # ownership sources; the stub has no transaction to end. + return None + async def get_account_id_and_abandonment(self, *args: Any, **kwargs: Any) -> StickyOwnerLookup: key = cast(str, args[0]) + scoped_abandoned_account_id = self.scoped_abandoned_account_ids_by_key.get(key) + if scoped_abandoned_account_id is not None: + return StickyOwnerLookup( + account_id=None, + continuity_abandoned=True, + abandoned_account_id=scoped_abandoned_account_id, + ) if key in self.abandoned_keys: return StickyOwnerLookup(account_id=None, continuity_abandoned=True) if self.account_ids_by_key is not None: - return StickyOwnerLookup(account_id=self.account_ids_by_key.get(key), continuity_abandoned=False) + return StickyOwnerLookup( + account_id=self.account_ids_by_key.get(key), + continuity_abandoned=False, + refresh_skip_deadline=self.refresh_skip_deadlines_by_key.get(key), + ) del kwargs - return StickyOwnerLookup(account_id=self.account_id, continuity_abandoned=False) + return StickyOwnerLookup( + account_id=self.account_id, + continuity_abandoned=False, + refresh_skip_deadline=self.refresh_skip_deadlines_by_key.get(key), + ) async def upsert(self, *args: Any, **kwargs: Any) -> Any: sticky_key = cast(str, args[0]) account_id = cast(str, args[1]) self.account_id = account_id + if self.account_ids_by_key is not None: + self.account_ids_by_key[sticky_key] = account_id self.upserts.append((sticky_key, account_id, kwargs.get("kind"))) return None + async def insert_if_absent( + self, + key: str, + account_id: str, + kind: StickySessionKind, + ) -> str: + self.insert_if_absent_calls.append((key, account_id, kind)) + if self.account_ids_by_key is None: + self.account_ids_by_key = {} + return self.account_ids_by_key.setdefault(key, account_id) + + async def upsert_with_seed_if_absent( + self, + key: str, + account_id: str, + *, + kind: StickySessionKind, + seed_key: str, + seed_kind: StickySessionKind, + ) -> None: + self.seeded_upserts.append((key, account_id, kind, seed_key, seed_kind)) + if self.account_ids_by_key is None: + self.account_ids_by_key = {} + self.account_ids_by_key.setdefault(seed_key, account_id) + self.account_ids_by_key[key] = account_id + self.account_id = account_id + self.upserts.append((key, account_id, kind)) + async def delete(self, *args: Any, **kwargs: Any) -> bool: sticky_key = cast(str, args[0]) self.deleted.append((sticky_key, kwargs.get("kind"))) + if self.account_ids_by_key is not None: + self.account_ids_by_key.pop(sticky_key, None) self.account_id = None return True @@ -259,13 +321,20 @@ async def restore_if_current( expected_account_id: str | None, restore_account_id: str | None, ) -> bool: - if self.account_id != expected_account_id: + current_account_id = ( + self.account_ids_by_key.get(key) if self.account_ids_by_key is not None else self.account_id + ) + if current_account_id != expected_account_id: return False if restore_account_id is None: self.deleted.append((key, kind)) + if self.account_ids_by_key is not None: + self.account_ids_by_key.pop(key, None) self.account_id = None return True self.upserts.append((key, restore_account_id, kind)) + if self.account_ids_by_key is not None: + self.account_ids_by_key[key] = restore_account_id self.account_id = restore_account_id return True @@ -310,6 +379,56 @@ async def upsert(self, *args: Any, **kwargs: Any) -> Any: raise RuntimeError("sticky persistence unavailable") +class _RetiringStaleOwnerStickySessionsRepository(_StubStickySessionsRepository): + def __init__(self, *, raw_key: str, owner_account_id: str) -> None: + super().__init__() + self.account_ids_by_key = {raw_key: owner_account_id} + self.tombstones: list[tuple[str, str]] = [] + + async def abandon_legacy_session_header_owner_if_unavailable( + self, + key: str, + *, + kind: StickySessionKind, + expected_account_id: str, + ) -> bool: + assert kind == StickySessionKind.CODEX_SESSION + assert self.account_ids_by_key is not None + if self.account_ids_by_key.get(key) != expected_account_id: + return False + # The account objects supplied to selection intentionally remain stale + # and ACTIVE after this authoritative repository decision. + self.scoped_abandoned_account_ids_by_key[key] = expected_account_id + self.tombstones.append((key, expected_account_id)) + return True + + async def upsert(self, *args: Any, **kwargs: Any) -> None: + sticky_key = cast(str, args[0]) + account_id = cast(str, args[1]) + assert self.account_ids_by_key is not None + self.account_ids_by_key[sticky_key] = account_id + self.upserts.append((sticky_key, account_id, kwargs.get("kind"))) + + +class _LosingRetirementRaceStickySessionsRepository(_RetiringStaleOwnerStickySessionsRepository): + async def abandon_legacy_session_header_owner_if_unavailable( + self, + key: str, + *, + kind: StickySessionKind, + expected_account_id: str, + ) -> bool: + assert kind == StickySessionKind.CODEX_SESSION + assert self.account_ids_by_key is not None + assert self.account_ids_by_key.get(key) == expected_account_id + # Another selector wins the source-scoped retirement CAS. The losing + # selector's authoritative reread must carry this retained owner into + # its stale-snapshot exclusion set. + self.scoped_abandoned_account_ids_by_key[key] = expected_account_id + self.tombstones.append((key, expected_account_id)) + return False + + @asynccontextmanager async def _repo_factory( accounts_repo: _StubAccountsRepository, @@ -2111,6 +2230,74 @@ async def test_sticky_probe_reservation_restores_affinity_after_repeated_commit_ await balancer.release_account_lease(selected.lease) +@pytest.mark.asyncio +async def test_provisional_recovery_probe_does_not_publish_process_seed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + now_epoch = int(datetime.now(tz=timezone.utc).timestamp()) + healthy = _make_account("acc-probe-seed-healthy") + probing = _make_account("acc-probe-seed-probing") + accounts_repo = _StubAccountsRepository([healthy, probing]) + usage_repo = _StubUsageRepository( + primary={ + healthy.id: _usage_row_with_percent( + 211, + healthy.id, + used_percent=30.0, + reset_at=now_epoch + 300, + ), + probing.id: _usage_row_with_percent( + 212, + probing.id, + used_percent=10.0, + reset_at=now_epoch + 300, + ), + }, + secondary={}, + ) + sticky_repo = _StubStickySessionsRepository() + sticky_repo.account_ids_by_key = {} + balancer = LoadBalancer(lambda: _repo_factory(accounts_repo, usage_repo, sticky_repo)) + balancer._runtime[probing.id] = RuntimeState( + health_tier=HEALTH_TIER_PROBING, + last_selected_at=0.0, + version=37, + health_version=13, + ) + monkeypatch.setattr(balancer, "_commit_due_probe_reservation_locked", lambda *args, **kwargs: False) + + selected = await balancer.select_account( + sticky_key="thread-after-probe-loss", + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + legacy_sticky_key="process-after-probe-loss", + sticky_seed_key="process-seed-after-probe-loss", + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + lease_kind="stream", + ) + + assert selected.account is not None + assert selected.account.id == healthy.id + assert selected.lease is not None + assert sticky_repo.account_ids_by_key == { + "process-seed-after-probe-loss": healthy.id, + "thread-after-probe-loss": healthy.id, + } + assert sticky_repo.seeded_upserts == [ + ( + "thread-after-probe-loss", + healthy.id, + StickySessionKind.PROMPT_CACHE, + "process-seed-after-probe-loss", + StickySessionKind.CODEX_SESSION, + ) + ] + + await balancer.release_account_lease(selected.lease) + + @pytest.mark.asyncio async def test_sticky_probe_reservation_restore_does_not_clobber_newer_owner( monkeypatch: pytest.MonkeyPatch, @@ -2764,6 +2951,341 @@ async def test_legacy_raw_session_mapping_wins_when_namespaced_row_also_exists() await balancer.release_account_lease(lease) +@pytest.mark.asyncio +async def test_new_codex_thread_is_seeded_from_process_preference_without_rewriting_process_row() -> None: + balancer, owner, alternate, sticky_repo = _make_cap_spillover_balancer("thread-process-seed") + assert alternate is not None + process_session = "process-seed" + process_key = _codex_session_selection_key(process_session) + thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "thread-new"} + ).thread_selection_key + assert thread_key is not None + sticky_repo.account_ids_by_key = {process_key: owner.id} + + selected = await balancer.select_account( + sticky_key=thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + legacy_sticky_key=process_session, + sticky_seed_key=process_key, + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + ) + + assert selected.account is not None + assert selected.account.id == owner.id + assert sticky_repo.account_ids_by_key == { + process_key: owner.id, + thread_key: owner.id, + } + assert sticky_repo.deleted == [] + assert sticky_repo.upserts == [(thread_key, owner.id, StickySessionKind.PROMPT_CACHE)] + + +@pytest.mark.asyncio +async def test_first_codex_thread_initializes_process_preference_once_for_later_siblings() -> None: + balancer, owner, alternate, sticky_repo = _make_cap_spillover_balancer("thread-first-process") + assert alternate is not None + process_session = "process-first-thread" + process_key = _codex_session_selection_key(process_session) + first_thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "thread-first"} + ).thread_selection_key + sibling_thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "thread-sibling"} + ).thread_selection_key + assert first_thread_key is not None + assert sibling_thread_key is not None + sticky_repo.account_ids_by_key = {} + + first = await balancer.select_account( + sticky_key=first_thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + legacy_sticky_key=process_session, + sticky_seed_key=process_key, + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + ) + assert first.account is not None + first_account_id = first.account.id + + sibling = await balancer.select_account( + sticky_key=sibling_thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + legacy_sticky_key=process_session, + sticky_seed_key=process_key, + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + ) + + assert sibling.account is not None + assert sibling.account.id == first_account_id + assert sticky_repo.account_ids_by_key[process_key] == first_account_id + assert sticky_repo.insert_if_absent_calls == [] + assert sticky_repo.seeded_upserts == [ + ( + first_thread_key, + first_account_id, + StickySessionKind.PROMPT_CACHE, + process_key, + StickySessionKind.CODEX_SESSION, + ) + ] + assert sticky_repo.upserts == [ + (first_thread_key, first_account_id, StickySessionKind.PROMPT_CACHE), + (sibling_thread_key, first_account_id, StickySessionKind.PROMPT_CACHE), + ] + + +@pytest.mark.asyncio +async def test_fresh_same_owner_retention_skips_refresh_write_when_seed_exists() -> None: + """A hot same-owner retention whose lookup observed the row inside the + refresh-skip window issues no sticky write at all.""" + balancer, owner, alternate, sticky_repo = _make_cap_spillover_balancer("thread-skip-refresh") + assert alternate is not None + process_session = "process-skip-refresh" + process_key = _codex_session_selection_key(process_session) + thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "thread-skip"} + ).thread_selection_key + assert thread_key is not None + sticky_repo.account_ids_by_key = {process_key: owner.id, thread_key: owner.id} + sticky_repo.refresh_skip_deadlines_by_key[thread_key] = datetime.now(tz=timezone.utc).replace( + tzinfo=None + ) + timedelta(seconds=10) + + selected = await balancer.select_account( + sticky_key=thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + legacy_sticky_key=process_session, + sticky_seed_key=process_key, + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + ) + + assert selected.account is not None + assert selected.account.id == owner.id + assert sticky_repo.upserts == [] + assert sticky_repo.seeded_upserts == [] + assert sticky_repo.deleted == [] + + +@pytest.mark.asyncio +async def test_fresh_thread_row_with_missing_seed_still_writes_and_initializes_seed() -> None: + """Seed initialization piggybacks on the thread retention write; a fresh + thread row must not suppress it while the process seed is absent.""" + balancer, owner, alternate, sticky_repo = _make_cap_spillover_balancer("thread-skip-seedless") + assert alternate is not None + process_session = "process-skip-seedless" + process_key = _codex_session_selection_key(process_session) + thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "thread-seedless"} + ).thread_selection_key + assert thread_key is not None + sticky_repo.account_ids_by_key = {thread_key: owner.id} + sticky_repo.refresh_skip_deadlines_by_key[thread_key] = datetime.now(tz=timezone.utc).replace( + tzinfo=None + ) + timedelta(seconds=10) + + selected = await balancer.select_account( + sticky_key=thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + legacy_sticky_key=process_session, + sticky_seed_key=process_key, + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + ) + + assert selected.account is not None + assert selected.account.id == owner.id + assert sticky_repo.account_ids_by_key[process_key] == owner.id + assert sticky_repo.seeded_upserts == [ + ( + thread_key, + owner.id, + StickySessionKind.PROMPT_CACHE, + process_key, + StickySessionKind.CODEX_SESSION, + ) + ] + + +@pytest.mark.asyncio +async def test_expired_refresh_skip_deadline_still_writes_through() -> None: + """A deadline that lapsed between lookup and persist must not suppress the + refresh: the skip window is revalidated at write time.""" + balancer, owner, alternate, sticky_repo = _make_cap_spillover_balancer("thread-skip-expired") + assert alternate is not None + process_session = "process-skip-expired" + process_key = _codex_session_selection_key(process_session) + thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "thread-expired"} + ).thread_selection_key + assert thread_key is not None + sticky_repo.account_ids_by_key = {process_key: owner.id, thread_key: owner.id} + sticky_repo.refresh_skip_deadlines_by_key[thread_key] = datetime.now(tz=timezone.utc).replace( + tzinfo=None + ) - timedelta(seconds=1) + + selected = await balancer.select_account( + sticky_key=thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + legacy_sticky_key=process_session, + sticky_seed_key=process_key, + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + ) + + assert selected.account is not None + assert selected.account.id == owner.id + assert sticky_repo.upserts == [(thread_key, owner.id, StickySessionKind.PROMPT_CACHE)] + + +@pytest.mark.asyncio +async def test_required_file_owner_does_not_rewrite_existing_thread_row() -> None: + balancer, thread_owner, file_owner, sticky_repo = _make_cap_spillover_balancer("file-pin-thread") + assert file_owner is not None + process_session = "file-pin-process" + thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "file-pin-thread"} + ).thread_selection_key + assert thread_key is not None + sticky_repo.account_ids_by_key = {thread_key: thread_owner.id} + preferred = _AffinityPolicy.preferred_owner_sticky_inputs( + thread_key, + StickySessionKind.PROMPT_CACHE, + False, + 300, + "thread_header", + process_session, + ) + + selected = await balancer.select_account( + sticky_key=preferred[0], + sticky_kind=preferred[1], + reallocate_sticky=preferred[2], + sticky_max_age_seconds=preferred[3], + sticky_source=preferred[4], + legacy_sticky_key=preferred[5], + required_account_id=file_owner.id, + required_account_is_ownership_constraint=True, + routing_strategy="usage_weighted", + lease_kind="stream", + ) + + assert selected.account is not None + assert selected.account.id == file_owner.id + assert sticky_repo.account_ids_by_key == {thread_key: thread_owner.id} + assert sticky_repo.upserts == [] + await balancer.release_account_lease(selected.lease) + + +@pytest.mark.asyncio +async def test_required_file_owner_seeds_process_preference_for_later_sibling() -> None: + balancer, thread_owner, file_owner, sticky_repo = _make_cap_spillover_balancer("file-pin-seed") + assert file_owner is not None + process_session = "file-pin-seed-process" + process_key = _codex_session_selection_key(process_session) + first_thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "file-pin-first"} + ).thread_selection_key + sibling_thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "file-pin-sibling"} + ).thread_selection_key + assert first_thread_key is not None + assert sibling_thread_key is not None + sticky_repo.account_ids_by_key = {} + preferred = _AffinityPolicy.preferred_owner_sticky_inputs( + first_thread_key, + StickySessionKind.PROMPT_CACHE, + False, + 300, + "thread_header", + process_session, + ) + + first = await balancer.select_account( + sticky_key=preferred[0], + sticky_kind=preferred[1], + reallocate_sticky=preferred[2], + sticky_max_age_seconds=preferred[3], + sticky_source=preferred[4], + legacy_sticky_key=preferred[5], + sticky_seed_key=process_key, + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + required_account_id=file_owner.id, + required_account_is_ownership_constraint=True, + routing_strategy="usage_weighted", + lease_kind="stream", + ) + assert first.account is not None + assert first.account.id == file_owner.id + assert first_thread_key not in (sticky_repo.account_ids_by_key or {}) + assert sticky_repo.account_ids_by_key == {process_key: file_owner.id} + + sibling = await balancer.select_account( + sticky_key=sibling_thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + legacy_sticky_key=process_session, + sticky_seed_key=process_key, + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + lease_kind="stream", + ) + assert sibling.account is not None + assert sibling.account.id == file_owner.id + assert sticky_repo.account_ids_by_key[process_key] == file_owner.id + await balancer.release_account_lease(first.lease) + await balancer.release_account_lease(sibling.lease) + + +@pytest.mark.asyncio +async def test_legacy_raw_process_owner_wins_over_thread_locality() -> None: + balancer, owner, alternate, sticky_repo = _make_cap_spillover_balancer("thread-legacy-owner") + assert alternate is not None + process_session = "legacy-process-owner" + thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "thread-existing"} + ).thread_selection_key + assert thread_key is not None + sticky_repo.account_ids_by_key = { + process_session: owner.id, + thread_key: alternate.id, + } + + selected = await balancer.select_account( + sticky_key=thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + legacy_sticky_key=process_session, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + ) + + assert selected.account is not None + assert selected.account.id == owner.id + assert sticky_repo.account_ids_by_key == { + process_session: owner.id, + thread_key: alternate.id, + } + assert sticky_repo.deleted == [] + assert sticky_repo.upserts == [] + + @pytest.mark.asyncio async def test_legacy_raw_owner_conflict_blocks_resolved_preferred_owner() -> None: balancer, owner, alternate, sticky_repo = _make_cap_spillover_balancer("legacy-preferred-conflict") @@ -2786,6 +3308,208 @@ async def test_legacy_raw_owner_conflict_blocks_resolved_preferred_owner() -> No assert sticky_repo.upserts == [] +@pytest.mark.asyncio +async def test_goal_restart_does_not_repin_retired_owner_from_stale_selection_snapshot() -> None: + now_epoch = int(datetime.now(tz=timezone.utc).timestamp()) + stale_owner = _make_account("goal-restart-stale-snapshot-owner") + replacement = _make_account("goal-restart-stale-snapshot-replacement") + raw_session = "goal-restart-stale-snapshot" + selection_key = _codex_session_selection_key(raw_session) + sticky_repo = _RetiringStaleOwnerStickySessionsRepository( + raw_key=raw_session, + owner_account_id=stale_owner.id, + ) + # Keep both account objects ACTIVE to model inputs loaded before the + # repository's guarded retirement observes the owner's unavailable row. + balancer = LoadBalancer( + lambda: _repo_factory( + _StubAccountsRepository([stale_owner, replacement]), + _StubUsageRepository( + { + stale_owner.id: _usage_row(301, stale_owner.id, window="primary", reset_at=now_epoch + 300), + replacement.id: _usage_row(302, replacement.id, window="primary", reset_at=now_epoch + 300), + }, + {}, + ), + sticky_repo, + ) + ) + + selected = await balancer.select_account( + sticky_key=selection_key, + sticky_kind=StickySessionKind.CODEX_SESSION, + sticky_source="session_header", + legacy_sticky_key=raw_session, + abandon_unavailable_legacy_owner=True, + routing_strategy="single_account", + lease_kind="stream", + ) + + assert selected.account is not None + assert selected.account.id == replacement.id + assert sticky_repo.tombstones == [(raw_session, stale_owner.id)] + assert sticky_repo.account_ids_by_key == { + raw_session: stale_owner.id, + selection_key: replacement.id, + } + assert all(account_id != stale_owner.id for _, account_id, _ in sticky_repo.upserts) + await balancer.release_account_lease(selected.lease) + + +@pytest.mark.asyncio +async def test_goal_restart_with_thread_header_retires_unavailable_legacy_owner() -> None: + now_epoch = int(datetime.now(tz=timezone.utc).timestamp()) + stale_owner = _make_account("goal-restart-thread-header-owner") + replacement = _make_account("goal-restart-thread-header-replacement") + raw_session = "goal-restart-thread-header-session" + thread_key = _codex_backend_identity( + {"session-id": raw_session, "thread-id": "goal-restart-thread"} + ).thread_selection_key + assert thread_key is not None + sticky_repo = _RetiringStaleOwnerStickySessionsRepository( + raw_key=raw_session, + owner_account_id=stale_owner.id, + ) + balancer = LoadBalancer( + lambda: _repo_factory( + _StubAccountsRepository([stale_owner, replacement]), + _StubUsageRepository( + { + stale_owner.id: _usage_row(311, stale_owner.id, window="primary", reset_at=now_epoch + 300), + replacement.id: _usage_row(312, replacement.id, window="primary", reset_at=now_epoch + 300), + }, + {}, + ), + sticky_repo, + ) + ) + + selected = await balancer.select_account( + sticky_key=thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + sticky_max_age_seconds=300, + legacy_sticky_key=raw_session, + abandon_unavailable_legacy_owner=True, + routing_strategy="single_account", + lease_kind="stream", + ) + + assert selected.account is not None + assert selected.account.id == replacement.id + assert sticky_repo.tombstones == [(raw_session, stale_owner.id)] + assert sticky_repo.account_ids_by_key == { + raw_session: stale_owner.id, + thread_key: replacement.id, + } + assert all(account_id != stale_owner.id for _, account_id, _ in sticky_repo.upserts) + await balancer.release_account_lease(selected.lease) + + +@pytest.mark.asyncio +async def test_goal_restart_cas_loser_does_not_repin_concurrently_retired_owner() -> None: + now_epoch = int(datetime.now(tz=timezone.utc).timestamp()) + stale_owner = _make_account("goal-restart-cas-loser-owner") + replacement = _make_account("goal-restart-cas-loser-replacement") + raw_session = "goal-restart-cas-loser" + selection_key = _codex_session_selection_key(raw_session) + sticky_repo = _LosingRetirementRaceStickySessionsRepository( + raw_key=raw_session, + owner_account_id=stale_owner.id, + ) + balancer = LoadBalancer( + lambda: _repo_factory( + _StubAccountsRepository([stale_owner, replacement]), + _StubUsageRepository( + { + stale_owner.id: _usage_row(305, stale_owner.id, window="primary", reset_at=now_epoch + 300), + replacement.id: _usage_row(306, replacement.id, window="primary", reset_at=now_epoch + 300), + }, + {}, + ), + sticky_repo, + ) + ) + + selected = await balancer.select_account( + sticky_key=selection_key, + sticky_kind=StickySessionKind.CODEX_SESSION, + sticky_source="session_header", + legacy_sticky_key=raw_session, + abandon_unavailable_legacy_owner=True, + routing_strategy="single_account", + lease_kind="stream", + ) + + assert selected.account is not None + assert selected.account.id == replacement.id + assert sticky_repo.tombstones == [(raw_session, stale_owner.id)] + assert sticky_repo.account_ids_by_key == { + raw_session: stale_owner.id, + selection_key: replacement.id, + } + assert all(account_id != stale_owner.id for _, account_id, _ in sticky_repo.upserts) + await balancer.release_account_lease(selected.lease) + + +@pytest.mark.asyncio +async def test_goal_restart_mutation_authority_precedes_model_eligibility( + monkeypatch: pytest.MonkeyPatch, +) -> None: + now_epoch = int(datetime.now(tz=timezone.utc).timestamp()) + owner = _make_account("goal-restart-model-ineligible-owner") + replacement = _make_account("goal-restart-model-eligible-replacement") + raw_session = "goal-restart-model-authority" + selection_key = _codex_session_selection_key(raw_session) + sticky_repo = _RetiringStaleOwnerStickySessionsRepository( + raw_key=raw_session, + owner_account_id=owner.id, + ) + balancer = LoadBalancer( + lambda: _repo_factory( + _StubAccountsRepository([owner, replacement]), + _StubUsageRepository( + { + owner.id: _usage_row(303, owner.id, window="primary", reset_at=now_epoch + 300), + replacement.id: _usage_row(304, replacement.id, window="primary", reset_at=now_epoch + 300), + }, + {}, + ), + sticky_repo, + ) + ) + + monkeypatch.setattr(load_balancer_module, "_mapped_model_has_registry_entry", lambda _model: True) + monkeypatch.setattr( + load_balancer_module, + "_filter_accounts_for_model", + lambda accounts, _model, **_kwargs: [account for account in accounts if account.id == replacement.id], + ) + monkeypatch.setattr( + load_balancer_module, + "_filter_accounts_for_model_with_catalog_evidence", + lambda accounts, _model, **_kwargs: load_balancer_module._ModelAccountFilterResult( + accounts=[account for account in accounts if account.id == replacement.id], + general_model_account_ids=frozenset({replacement.id}), + ), + ) + + selected = await balancer.select_account( + sticky_key=selection_key, + sticky_kind=StickySessionKind.CODEX_SESSION, + sticky_source="session_header", + legacy_sticky_key=raw_session, + abandon_unavailable_legacy_owner=True, + model="gpt-model-authority", + lease_kind="stream", + ) + + assert selected.account is not None + assert selected.account.id == replacement.id + assert sticky_repo.tombstones == [(raw_session, owner.id)] + await balancer.release_account_lease(selected.lease) + + @pytest.mark.asyncio async def test_bare_session_mapping_does_not_prove_ambiguous_conversation_owner() -> None: balancer, owner, _, sticky_repo = _make_cap_spillover_balancer("conversation-ambiguous") @@ -2805,6 +3529,31 @@ async def test_bare_session_mapping_does_not_prove_ambiguous_conversation_owner( assert selected.error_code == "conversation_owner_unavailable" +@pytest.mark.asyncio +async def test_scoped_restart_marker_does_not_prove_ambiguous_conversation_owner() -> None: + balancer, retired_owner, replacement, sticky_repo = _make_cap_spillover_balancer("conversation-scoped-restart") + assert replacement is not None + raw_session = "conversation-scoped-restart-session" + selection_key = _codex_session_selection_key(raw_session) + sticky_repo.account_ids_by_key = { + raw_session: retired_owner.id, + selection_key: replacement.id, + } + sticky_repo.scoped_abandoned_account_ids_by_key[raw_session] = retired_owner.id + + selected = await balancer.select_account( + sticky_key=selection_key, + sticky_kind=StickySessionKind.CODEX_SESSION, + sticky_source="session_header", + legacy_sticky_key=raw_session, + require_unambiguous_account=True, + lease_kind="response_create", + ) + + assert selected.account is None + assert selected.error_code == "conversation_owner_unavailable" + + @pytest.mark.asyncio async def test_tombstoned_hard_owner_lets_conversation_continuity_reselect() -> None: """A purge-tombstoned mapping (see purge_stale_hard_codex_session_mappings) @@ -3816,3 +4565,201 @@ async def test_api_key_fair_share_concurrent_sticky_selections_cannot_overshoot_ # The commit re-check kept heavy at exactly its share across both paths. heavy_total = sum((runtime.stream_key_inflight or {}).get("heavy", 0) for runtime in balancer._runtime.values()) assert heavy_total == 2 + + +@pytest.mark.asyncio +async def test_fresh_same_owner_retention_skips_refresh_write_on_probe_admission() -> None: + """The recovery-probe admission path honors the refresh-skip deadline the + same way the non-probe persist site does: a fresh same-owner retention of + a due-probing pinned owner issues no sticky write.""" + now_epoch = int(datetime.now(tz=timezone.utc).timestamp()) + healthy = _make_account("acc-probe-skip-healthy") + probing = _make_account("acc-probe-skip-probing") + key = "probe-skip-session" + + def _build(sticky_repo: _StubStickySessionsRepository) -> LoadBalancer: + accounts_repo = _StubAccountsRepository([healthy, probing]) + usage_repo = _StubUsageRepository( + primary={ + healthy.id: _usage_row_with_percent( + 150, + healthy.id, + used_percent=30.0, + reset_at=now_epoch + 300, + ), + probing.id: _usage_row_with_percent( + 151, + probing.id, + used_percent=10.0, + reset_at=now_epoch + 300, + ), + }, + secondary={}, + ) + balancer = LoadBalancer(lambda: _repo_factory(accounts_repo, usage_repo, sticky_repo)) + balancer._runtime[probing.id] = RuntimeState( + health_tier=HEALTH_TIER_PROBING, + last_selected_at=0.0, + version=17, + ) + return balancer + + # Control: without a freshness observation the probe admission persists + # the retention write, proving this scenario exercises the probe branch. + control_repo = _StubStickySessionsRepository() + control_repo.account_ids_by_key = {key: probing.id} + control_balancer = _build(control_repo) + control = await control_balancer.select_account( + sticky_key=key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + lease_kind="stream", + ) + assert control.account is not None + assert control.account.id == probing.id + assert control_repo.upserts == [(key, probing.id, StickySessionKind.PROMPT_CACHE)] + await control_balancer.release_account_lease(control.lease) + + skip_repo = _StubStickySessionsRepository() + skip_repo.account_ids_by_key = {key: probing.id} + skip_repo.refresh_skip_deadlines_by_key[key] = datetime.now(tz=timezone.utc).replace(tzinfo=None) + timedelta( + seconds=10 + ) + skip_balancer = _build(skip_repo) + selected = await skip_balancer.select_account( + sticky_key=key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + lease_kind="stream", + ) + assert selected.account is not None + assert selected.account.id == probing.id + assert skip_repo.upserts == [] + assert skip_repo.deleted == [] + # The probe reservation itself still committed: runtime advanced. + probing_runtime = skip_balancer._runtime[probing.id] + assert probing_runtime.version > 17 + assert probing_runtime.last_selected_at is not None + assert probing_runtime.last_selected_at > 0.0 + await skip_balancer.release_account_lease(selected.lease) + + +@pytest.mark.asyncio +async def test_fresh_thread_only_retention_without_seed_key_skips_refresh_write() -> None: + """Thread-only affinity (no process seed key at all) has nothing to + initialize, so a fresh same-owner retention skips its refresh write.""" + balancer, owner, alternate, sticky_repo = _make_cap_spillover_balancer("thread-skip-no-seedkey") + assert alternate is not None + thread_key = _codex_backend_identity({"thread-id": "thread-only-skip"}).thread_selection_key + assert thread_key is not None + sticky_repo.account_ids_by_key = {thread_key: owner.id} + sticky_repo.refresh_skip_deadlines_by_key[thread_key] = datetime.now(tz=timezone.utc).replace( + tzinfo=None + ) + timedelta(seconds=10) + + selected = await balancer.select_account( + sticky_key=thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + ) + + assert selected.account is not None + assert selected.account.id == owner.id + assert sticky_repo.upserts == [] + assert sticky_repo.seeded_upserts == [] + assert sticky_repo.deleted == [] + + +class _LookupCountingStickyRepo(_StubStickySessionsRepository): + """Records owner-lookup and snapshot-release events, each stamped with the + repository context that issued it, so tests can pin lookup count/order and + the one-session/fresh-transaction-per-source contract.""" + + def __init__(self) -> None: + super().__init__() + # Set by the test's repo factory each time a repo bundle opens. + self.current_context_id: int | None = None + self.owner_lookup_events: list[tuple[str, int | None, str | None]] = [] + + async def get_account_id_and_abandonment(self, *args: Any, **kwargs: Any) -> StickyOwnerLookup: + self.owner_lookup_events.append(("lookup", self.current_context_id, cast(str, args[0]))) + return await super().get_account_id_and_abandonment(*args, **kwargs) + + async def release_read_snapshot(self) -> None: + self.owner_lookup_events.append(("release_snapshot", self.current_context_id, None)) + await super().release_read_snapshot() + + +@pytest.mark.asyncio +async def test_shared_owner_lookup_session_reads_each_owner_key_exactly_once() -> None: + """Regression for the shared owner-lookup session. + + The legacy/seed/first-sticky owner reads moved into one repo bundle in + ``select_account``; the sticky selection loop consumes the hoisted first + read exactly once instead of re-reading. Each owner key must be looked up + exactly once, in the legacy -> seed -> sticky order, all three reads must + share one repository context (one session), each later ownership source + must first release the shared read snapshot so it starts a fresh + transaction, and the resolved hard owner must still win selection. + """ + now_epoch = int(datetime.now(tz=timezone.utc).timestamp()) + owner = _make_account("acc-shared-owner-lookup") + other = _make_account("acc-shared-owner-other") + accounts_repo = _StubAccountsRepository([owner, other]) + usage_repo = _StubUsageRepository( + primary={ + owner.id: _usage_row(70, owner.id, window="primary", reset_at=now_epoch + 300), + other.id: _usage_row(71, other.id, window="primary", reset_at=now_epoch + 300), + }, + secondary={}, + ) + sticky_repo = _LookupCountingStickyRepo() + sticky_repo.account_ids_by_key = {"shared-lookup-sticky": owner.id} + opened_context_count = 0 + + @asynccontextmanager + async def context_stamping_repo_factory() -> AsyncIterator[ProxyRepositories]: + # Stamp every bundle open with a distinct identifier so the events + # recorded by the sticky repo prove which context issued each read; + # the old per-lookup-session flow would record three distinct ids. + nonlocal opened_context_count + opened_context_count += 1 + sticky_repo.current_context_id = opened_context_count + async with _repo_factory(accounts_repo, usage_repo, sticky_repo) as repos: + yield repos + + balancer = LoadBalancer(context_stamping_repo_factory) + + selected = await balancer.select_account( + sticky_key="shared-lookup-sticky", + sticky_kind=StickySessionKind.CODEX_SESSION, + sticky_source="turn_state", + legacy_sticky_key="shared-lookup-legacy", + sticky_seed_key="shared-lookup-seed", + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + routing_strategy="usage_weighted", + ) + + assert selected.account is not None + assert selected.account.id == owner.id + # get_account_id (seed) delegates to get_account_id_and_abandonment in the + # stub, so this also proves the seed lookup ran exactly once. The + # release_snapshot events pin the fix semantics: one shared session, but a + # fresh read transaction before each later ownership source so a + # concurrently committed owner stays visible on SQLite/WAL. + assert [(event, key) for event, _, key in sticky_repo.owner_lookup_events] == [ + ("lookup", "shared-lookup-legacy"), + ("release_snapshot", None), + ("lookup", "shared-lookup-seed"), + ("release_snapshot", None), + ("lookup", "shared-lookup-sticky"), + ] + lookup_context_ids = {context_id for _, context_id, _ in sticky_repo.owner_lookup_events} + # One repository context served every ownership source; the old + # session-per-lookup flow would have recorded three distinct ids here. + assert len(lookup_context_ids) == 1 + assert None not in lookup_context_ids diff --git a/tests/unit/test_loop_lag_monitor.py b/tests/unit/test_loop_lag_monitor.py new file mode 100644 index 0000000000..881d2acdc2 --- /dev/null +++ b/tests/unit/test_loop_lag_monitor.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import asyncio +import logging +import time + +import pytest + +from app.core.resilience import loop_lag_monitor + +pytestmark = pytest.mark.unit + + +class _StubGauge: + def __init__(self) -> None: + self.values: list[float] = [] + + def set(self, value: float) -> None: + self.values.append(value) + + +class _StubCounter: + def __init__(self) -> None: + self.count = 0 + + def inc(self, amount: float = 1) -> None: + self.count += amount + + +@pytest.fixture +def stub_metrics(monkeypatch): + gauge = _StubGauge() + counter = _StubCounter() + monkeypatch.setattr(loop_lag_monitor.prometheus_metrics, "event_loop_lag_seconds", gauge) + monkeypatch.setattr(loop_lag_monitor.prometheus_metrics, "event_loop_lag_warnings_total", counter) + monkeypatch.setattr(loop_lag_monitor, "_SAMPLE_INTERVAL_SECONDS", 0.01) + return gauge, counter + + +async def _run_monitor_briefly(*, warn_threshold_seconds: float, block_seconds: float) -> asyncio.Task[None]: + task = asyncio.create_task( + loop_lag_monitor.run_event_loop_lag_monitor(warn_threshold_seconds=warn_threshold_seconds) + ) + # Let the monitor enter its first sleep, then starve the loop synchronously + # so the sleep resumes late — exactly what a callback storm looks like. + await asyncio.sleep(0) + if block_seconds: + time.sleep(block_seconds) + await asyncio.sleep(0.05) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + return task + + +async def test_starved_loop_emits_warning_and_metrics(stub_metrics, caplog): + gauge, counter = stub_metrics + with caplog.at_level(logging.WARNING, logger=loop_lag_monitor.logger.name): + await _run_monitor_briefly(warn_threshold_seconds=0.05, block_seconds=0.15) + assert any(v >= 0.05 for v in gauge.values) + assert counter.count >= 1 + assert any("event_loop_lag" in record.message for record in caplog.records) + + +async def test_healthy_loop_stays_quiet(stub_metrics, caplog): + gauge, counter = stub_metrics + with caplog.at_level(logging.WARNING, logger=loop_lag_monitor.logger.name): + await _run_monitor_briefly(warn_threshold_seconds=0.5, block_seconds=0.0) + assert gauge.values, "gauge should be sampled even when healthy" + assert counter.count == 0 + assert not [r for r in caplog.records if "event_loop_lag" in r.message] + + +async def test_warning_log_is_rate_limited(stub_metrics, caplog): + _, counter = stub_metrics + task = asyncio.create_task(loop_lag_monitor.run_event_loop_lag_monitor(warn_threshold_seconds=0.05)) + with caplog.at_level(logging.WARNING, logger=loop_lag_monitor.logger.name): + await asyncio.sleep(0) + time.sleep(0.1) + await asyncio.sleep(0.05) + time.sleep(0.1) + await asyncio.sleep(0.05) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + warning_lines = [r for r in caplog.records if "event_loop_lag" in r.message] + assert len(warning_lines) == 1, "second spike within the window must be suppressed" + assert counter.count >= 2, "counter still tracks every over-threshold sample" diff --git a/tests/unit/test_model_registry.py b/tests/unit/test_model_registry.py index 6514132b86..2d15149a43 100644 --- a/tests/unit/test_model_registry.py +++ b/tests/unit/test_model_registry.py @@ -27,7 +27,7 @@ } # The 21-plan list upstream advertises for GPT-5.6 -# (codex-rs/models-manager/models.json at rust-v0.144.1). +# (codex-rs/models-manager/models.json at rust-v0.145.0). EXPECTED_GPT56_MODEL_PLANS = { "business", "edu", @@ -242,7 +242,7 @@ def test_bootstrap_models_include_representative_upstream_metadata(): sol = models["gpt-5.6-sol"] assert sol.display_name == "GPT-5.6-Sol" - assert sol.context_window == 372_000 + assert sol.context_window == 272_000 assert sol.default_reasoning_level == "low" assert [level.effort for level in sol.supported_reasoning_levels] == [ "low", @@ -253,6 +253,7 @@ def test_bootstrap_models_include_representative_upstream_metadata(): "ultra", ] assert sol.raw["additional_speed_tiers"] == ["fast"] + assert "ultrafast" not in str(sol.raw["service_tiers"]) terra = models["gpt-5.6-terra"] assert terra.display_name == "GPT-5.6-Terra" @@ -271,10 +272,14 @@ def test_bootstrap_models_include_representative_upstream_metadata(): assert luna.default_reasoning_level == "medium" assert [level.effort for level in luna.supported_reasoning_levels] == ["low", "medium", "high", "xhigh", "max"] - # Upstream-exact GPT-5.6 raw metadata (codex-rs/models-manager/models.json - # at rust-v0.144.1). + # Reproducible upstream catalog evidence: + # codex-rs/models-manager/models.json at rust-v0.145.0, except + # ``max_context_window``: raised to 872000 in openai/codex commit + # 2eee483e49f88b868f67364134a658b3298e6c14 (openai/codex#39102), which no + # rust-v* release tag carries yet. for gpt56 in (sol, terra, luna): assert gpt56.minimal_client_version == "0.144.0" + assert gpt56.context_window == 272_000 assert gpt56.raw["tool_mode"] == "code_mode_only" assert gpt56.raw["use_responses_lite"] is True assert gpt56.raw["apply_patch_tool_type"] == "freeform" @@ -287,7 +292,13 @@ def test_bootstrap_models_include_representative_upstream_metadata(): assert gpt56.raw["include_skills_usage_instructions"] is False assert gpt56.raw["experimental_supported_tools"] == [] assert gpt56.raw["supports_search_tool"] is True - assert gpt56.raw["max_context_window"] == 372_000 + # The upstream ceiling is decoupled from the default input budget, so + # the ``_bootstrap_model`` synthesis (max == context_window) must not + # win for these entries. + max_context_window = gpt56.raw["max_context_window"] + assert isinstance(max_context_window, int) + assert max_context_window == 872_000 + assert max_context_window > gpt56.context_window assert gpt56.raw["service_tiers"] == [ {"id": "priority", "name": "Fast", "description": "1.5x speed, increased usage"} ] diff --git a/tests/unit/test_model_sources_catalog.py b/tests/unit/test_model_sources_catalog.py index fb25a3c94d..db7e63a2b8 100644 --- a/tests/unit/test_model_sources_catalog.py +++ b/tests/unit/test_model_sources_catalog.py @@ -9,8 +9,10 @@ from app.modules.model_sources.catalog import ( DEFAULT_SOURCE_CONTEXT_WINDOW, source_model_audio_cost_usd, + source_model_reasoning_levels, source_model_request_overrides, source_model_supported_tool_types, + source_model_supports_reasoning, source_models_to_upstream_models, ) @@ -220,3 +222,207 @@ def test_source_models_force_codex_lb_provider_metadata() -> None: assert len(models) == 1 assert models[0].raw["model_provider"] == "codex-lb" + + +def _reasoning_source(raw_metadata_json: str | None) -> ModelSource: + return ModelSource( + id="src_reasoning", + name="Reasoning", + kind=MODEL_SOURCE_KIND_OPENAI_COMPATIBLE, + base_url="http://127.0.0.1:8000/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=True, + supports_audio_transcriptions=False, + models=[ + ModelSourceModel( + model="reasoning-model", + is_enabled=True, + supports_streaming=True, + raw_metadata_json=raw_metadata_json, + ) + ], + ) + + +def test_source_model_without_metadata_advertises_no_reasoning_levels() -> None: + [model] = source_models_to_upstream_models([_reasoning_source(None)]) + assert model.supported_reasoning_levels == () + assert model.default_reasoning_level is None + assert model.supports_reasoning_summaries is False + + +def test_source_model_reasoning_levels_accept_effort_slugs() -> None: + raw = json.dumps( + { + "supports_reasoning": True, + "supported_reasoning_levels": ["low", "medium", "high", "xhigh"], + "default_reasoning_level": "high", + } + ) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert [level.effort for level in model.supported_reasoning_levels] == [ + "low", + "medium", + "high", + "xhigh", + ] + assert model.default_reasoning_level == "high" + + +def test_source_model_reasoning_levels_accept_objects_and_summaries() -> None: + raw = json.dumps( + { + "supports_reasoning": True, + "supported_reasoning_levels": [ + {"effort": "low", "description": "Low effort"}, + {"effort": "max", "description": "Max effort"}, + ], + "default_reasoning_level": "max", + "supports_reasoning_summaries": True, + } + ) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert [(level.effort, level.description) for level in model.supported_reasoning_levels] == [ + ("low", "Low effort"), + ("max", "Max effort"), + ] + assert model.default_reasoning_level == "max" + assert model.supports_reasoning_summaries is True + + +def test_source_model_reasoning_levels_ignore_invalid_entries_and_defaults() -> None: + raw = json.dumps( + { + "supports_reasoning": True, + "supported_reasoning_levels": ["low", "low", {"description": "no effort key"}, 7, {"effort": "high"}], + # Not one of the advertised efforts, so it must not be surfaced. + "default_reasoning_level": "ultra", + } + ) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert [level.effort for level in model.supported_reasoning_levels] == ["low", "high"] + assert model.default_reasoning_level is None + + +def test_source_model_reasoning_levels_ignore_non_list_metadata() -> None: + raw = json.dumps({"supports_reasoning": True, "supported_reasoning_levels": "high"}) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert model.supported_reasoning_levels == () + + +def test_source_model_reasoning_levels_are_normalized_and_deduplicated() -> None: + """Efforts are normalized and deduplicated, but not filtered by vocabulary. + + Backends disagree on which efforts exist, so an effort this proxy has + never heard of is still the operator's to declare; only shape is checked. + """ + raw = json.dumps( + { + "supports_reasoning": True, + "supported_reasoning_levels": [" Low ", "HIGH", " ", "low", "provider-specific"], + "default_reasoning_level": " HIGH ", + } + ) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert [level.effort for level in model.supported_reasoning_levels] == [ + "low", + "high", + "provider-specific", + ] + assert model.default_reasoning_level == "high" + + +def test_source_model_can_declare_none_as_a_reasoning_level() -> None: + """``none`` is a real effort on GLM and Model Studio (see #1660). + + It is also already first-class for API-key enforced efforts, so dropping + it from source catalogs would have made the two vocabularies disagree. + """ + raw = json.dumps( + { + "supports_reasoning": True, + "supported_reasoning_levels": ["none", "high", "max"], + "default_reasoning_level": "none", + } + ) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert [level.effort for level in model.supported_reasoning_levels] == ["none", "high", "max"] + assert model.default_reasoning_level == "none" + + +def test_source_model_default_level_outside_declared_set_is_dropped() -> None: + raw = json.dumps( + {"supports_reasoning": True, "supported_reasoning_levels": ["low"], "default_reasoning_level": "max"} + ) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert model.default_reasoning_level is None + + +def test_declared_levels_do_not_imply_the_reasoning_opt_in() -> None: + """Levels describe *which* efforts an opted-in backend takes, not whether + reasoning is allowed. The dashboard's Reasoning switch is the only opt-in, + and the catalog derivation is gated on it too, so a model with the switch + off advertises nothing rather than advertising an inert capability.""" + raw = json.dumps({"supported_reasoning_levels": ["low", "high"]}) + source = _reasoning_source(raw) + assert source_model_supports_reasoning(source, "reasoning-model") is False + [model] = source_models_to_upstream_models([source]) + assert model.supported_reasoning_levels == () + assert model.default_reasoning_level is None + assert source_model_reasoning_levels(source, "reasoning-model") == () + + +def test_declared_summaries_do_not_imply_the_reasoning_opt_in() -> None: + """Summary support is gated by the same switch, for the same reason.""" + summaries_only = _reasoning_source(json.dumps({"supports_reasoning_summaries": True})) + assert source_model_supports_reasoning(summaries_only, "reasoning-model") is False + [model] = source_models_to_upstream_models([summaries_only]) + assert model.supports_reasoning_summaries is False + + +def test_the_reasoning_switch_gates_every_surface() -> None: + """The Codex catalog, the chat gate and the restore must never disagree. + + They are read by different call sites, so this pins them together: with + the switch off nothing is advertised or restorable, with it on the + operator's declared levels reach all three. + """ + declared = {"supported_reasoning_levels": ["low", "high"], "supports_reasoning_summaries": True} + off = _reasoning_source(json.dumps(declared)) + on = _reasoning_source(json.dumps({"supports_reasoning": True, **declared})) + + [off_model] = source_models_to_upstream_models([off]) + assert off_model.supported_reasoning_levels == () + assert off_model.supports_reasoning_summaries is False + assert source_model_supports_reasoning(off, "reasoning-model") is False + assert source_model_reasoning_levels(off, "reasoning-model") == () + + [on_model] = source_models_to_upstream_models([on]) + assert [level.effort for level in on_model.supported_reasoning_levels] == ["low", "high"] + assert on_model.supports_reasoning_summaries is True + assert source_model_supports_reasoning(on, "reasoning-model") is True + assert [level.effort for level in source_model_reasoning_levels(on, "reasoning-model")] == ["low", "high"] + + +def test_no_declared_levels_keeps_the_explicit_reasoning_opt_in() -> None: + assert source_model_supports_reasoning(_reasoning_source(None), "reasoning-model") is False + explicit = _reasoning_source(json.dumps({"supports_reasoning": True})) + assert source_model_supports_reasoning(explicit, "reasoning-model") is True + + +def test_source_model_reasoning_levels_accessor_matches_the_catalog() -> None: + raw = json.dumps({"supports_reasoning": True, "supported_reasoning_levels": ["minimal", "low"]}) + source = _reasoning_source(raw) + assert [level.effort for level in source_model_reasoning_levels(source, "reasoning-model")] == [ + "minimal", + "low", + ] + assert source_model_reasoning_levels(source, "unknown-model") == () + + +def test_declared_summaries_imply_the_chat_path_reasoning_opt_in() -> None: + """``supports_reasoning_summaries`` is surfaced as ``supports_reasoning`` on + /v1/models, so declaring it alone must not leave the chat path stripping.""" + summaries_only = _reasoning_source(json.dumps({"supports_reasoning": True, "supports_reasoning_summaries": True})) + assert source_model_supports_reasoning(summaries_only, "reasoning-model") is True diff --git a/tests/unit/test_multipart_content_encoding_middleware.py b/tests/unit/test_multipart_content_encoding_middleware.py index d427ae557e..5fac0d9889 100644 --- a/tests/unit/test_multipart_content_encoding_middleware.py +++ b/tests/unit/test_multipart_content_encoding_middleware.py @@ -6,7 +6,6 @@ import pytest from httpx import ASGITransport, AsyncByteStream, AsyncClient from starlette.datastructures import Headers -from starlette.middleware.base import BaseHTTPMiddleware from starlette.requests import Request from starlette.types import Message, Scope @@ -22,6 +21,7 @@ ) from app.core.middleware.path_rewrite import BackendApiCodexV1AliasMiddleware from app.core.middleware.request_body_limit import RequestBodyLimitMiddleware +from app.core.middleware.request_decompression import RequestDecompressionMiddleware from app.main import create_app pytestmark = pytest.mark.unit @@ -360,9 +360,7 @@ def test_production_middleware_order_composes_route_and_generic_ingress_guards() ) limit_index = next(index for index, item in enumerate(middleware) if item.cls is RequestBodyLimitMiddleware) decompression_index = next( - index - for index, item in enumerate(middleware) - if item.cls is BaseHTTPMiddleware and item.kwargs.get("dispatch").__name__ == "request_decompression_middleware" + index for index, item in enumerate(middleware) if item.cls is RequestDecompressionMiddleware ) assert alias_index < multipart_index < limit_index < decompression_index diff --git a/tests/unit/test_openai_errors.py b/tests/unit/test_openai_errors.py index a6068aa6e5..ac0dc547dd 100644 --- a/tests/unit/test_openai_errors.py +++ b/tests/unit/test_openai_errors.py @@ -57,10 +57,15 @@ def test_previous_response_not_found_classifier_covers_openai_shapes(): param=None, message="Invalid `previous_response_id`.", ) + assert is_previous_response_not_found_error( + code="invalid_request_error", + param=None, + message="Invalid `previous_response_id`", + ) assert is_previous_response_not_found_error( code="invalid_request_error", param="previous_response_id", - message="Invalid 'previous_response_id'.", + message="Invalid `previous_response_id`.", ) assert not is_previous_response_not_found_error( code="invalid_request_error", @@ -75,17 +80,17 @@ def test_previous_response_not_found_classifier_covers_openai_shapes(): assert not is_previous_response_not_found_error( code="invalid_request_error", param=None, - message="Invalid previous_response_id because another field is malformed.", + message="Invalid request payload.", ) assert not is_previous_response_not_found_error( code="invalid_request_error", param=None, - message="Invalid 'previous_response_id.", + message="Invalid `previous_response_id`...", ) assert not is_previous_response_not_found_error( - code="invalid_request_error", + code=None, param=None, - message='Invalid `previous_response_id".', + message="Invalid `previous_response_id`.", ) diff --git a/tests/unit/test_openai_requests.py b/tests/unit/test_openai_requests.py index 1afe4e5632..45b0b0e452 100644 --- a/tests/unit/test_openai_requests.py +++ b/tests/unit/test_openai_requests.py @@ -1,24 +1,34 @@ from __future__ import annotations import json +import re +from copy import deepcopy from typing import Mapping, cast import pytest +from hypothesis import given, settings +from hypothesis import strategies as st from pydantic import ValidationError from app.core.openai.exceptions import ClientPayloadError from app.core.openai.requests import ( _ESTIMATED_CHARS_PER_TOKEN, _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS, + _UNSUPPORTED_UPSTREAM_FIELDS, ResponsesCompactRequest, ResponsesRequest, + _estimated_json_array_item_tokens, _estimated_json_tokens, _input_image_file_reference, + _sanitize_input_items, + _strip_unsupported_fields, + _trim_compact_input_for_upstream, extract_input_file_ids, extract_input_image_file_references, ) from app.core.openai.v1_requests import V1ResponsesCompactRequest, V1ResponsesRequest from app.core.types import JsonValue +from tests.unit.hypothesis_strategies import json_arrays, json_directive_types, json_objects, json_values def test_responses_requires_instructions(): @@ -115,17 +125,68 @@ def test_known_unsupported_upstream_fields_are_stripped(): assert dumped["custom_field"] == "kept" -def test_responses_preserves_service_tier(): +@given(json_arrays) +@settings(deadline=None) +def test_sanitize_input_items_is_idempotent_for_json(input_items): + original = deepcopy(input_items) + try: + sanitized = _sanitize_input_items(input_items) + except ValueError: + # Tool items without a usable call ID are deliberately rejected. + return + + assert input_items == original + assert _sanitize_input_items(deepcopy(sanitized)) == sanitized + + +@given( + role=st.sampled_from(["system", "developer"]), + item_type=json_directive_types, + extra=json_objects, +) +@settings(deadline=None) +def test_sanitize_input_items_preserves_typed_directives(role, item_type, extra): + directive = dict(extra) + directive.update({"role": role, "type": item_type}) + + assert _sanitize_input_items([directive]) == [directive] + + +@given(payload=json_objects.map(lambda value: {key: item for key, item in value.items() if key != "input"})) +@settings(deadline=None) +def test_strip_unsupported_fields_is_idempotent(payload): + payload = cast(dict[str, JsonValue], payload) + first = _strip_unsupported_fields(deepcopy(payload)) + second = _strip_unsupported_fields(deepcopy(first)) + + assert second == first + assert _UNSUPPORTED_UPSTREAM_FIELDS.isdisjoint(first) + + +@given(namespace=json_values) +@settings(deadline=None) +def test_strip_unsupported_fields_namespace_flag_controls_replayed_calls(namespace): + payload = cast(dict[str, JsonValue], {"input": [{"type": "function_call", "namespace": namespace}]}) + + preserved = _strip_unsupported_fields(deepcopy(payload), strip_replayed_tool_call_namespaces=False) + stripped = _strip_unsupported_fields(deepcopy(payload)) + + assert preserved["input"] == payload["input"] + assert stripped["input"] == [{"type": "function_call"}] + + +@pytest.mark.parametrize("service_tier", ["priority", "ultrafast"]) +def test_responses_preserves_service_tier(service_tier: str): payload = { "model": "gpt-5.1", "instructions": "hi", "input": [], - "service_tier": "priority", + "service_tier": service_tier, } request = ResponsesRequest.model_validate(payload) dumped = request.to_payload() - assert dumped["service_tier"] == "priority" + assert dumped["service_tier"] == service_tier def test_responses_normalizes_fast_service_tier_to_priority_for_upstream(): @@ -405,6 +466,41 @@ def test_openai_compatible_reasoning_aliases_are_normalized(): assert "reasoningSummary" not in dumped +@pytest.mark.parametrize("request_type", [ResponsesRequest, ResponsesCompactRequest]) +@pytest.mark.parametrize("alias", ["reasoningEffort", "reasoning_effort", "thinking"]) +def test_reasoning_aliases_are_preserved_until_wire_serialization(request_type, alias): + request = request_type.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "hi", + "input": [], + alias: "ultra", + } + ) + + assert request.reasoning is None + assert (request.model_extra or {})[alias] == "ultra" + dumped = request.to_payload() + assert dumped["reasoning"] == {"effort": "ultra"} + assert alias not in dumped + + +def test_source_forwarding_preserves_provider_thinking_object(): + thinking = {"type": "enabled", "budget": 4096, "budget_tokens": 2048, "vendor_hint": "keep"} + request = ResponsesRequest.model_validate( + { + "model": "source-model", + "instructions": "hi", + "input": [], + "thinking": thinking, + } + ) + + forwarded = request.model_dump_for_forwarding() + assert forwarded["thinking"] == thinking + assert "reasoning" not in forwarded + + def test_provider_thinking_aliases_are_normalized(): payload = { "model": "gpt-5.1", @@ -425,7 +521,7 @@ def test_provider_thinking_string_alias_accepts_catalog_advertised_efforts(): # GPT-5.6 catalog entries advertise ``max`` and ``ultra`` # (codex-rs/models-manager/models.json at rust-v0.144.1); the string-form # thinking alias must accept every catalog-advertised effort. - for effort in ("low", "medium", "high", "xhigh", "max", "ultra"): + for effort in ("minimal", "low", "medium", "high", "xhigh", "max", "ultra"): payload = { "model": "gpt-5.6-sol", "instructions": "hi", @@ -484,16 +580,17 @@ def test_openai_compatible_top_level_verbosity_is_normalized(): assert "verbosity" not in dumped -def test_v1_responses_preserves_service_tier(): +@pytest.mark.parametrize("service_tier", ["priority", "ultrafast"]) +def test_v1_responses_preserves_service_tier(service_tier: str): payload = { "model": "gpt-5.1", "input": "hello", - "service_tier": "priority", + "service_tier": service_tier, } request = V1ResponsesRequest.model_validate(payload).to_responses_request() dumped = request.to_payload() - assert dumped["service_tier"] == "priority" + assert dumped["service_tier"] == service_tier def test_v1_responses_normalizes_fast_service_tier_to_priority_for_upstream(): @@ -1163,6 +1260,163 @@ def test_compact_many_small_items_include_array_wire_framing_in_budget(): assert wire_bytes <= _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS * _ESTIMATED_CHARS_PER_TOKEN +@given(input_items=json_arrays) +@settings(max_examples=30, deadline=None) +def test_compact_trim_leaves_budget_fitting_json_unchanged(input_items): + if _estimated_json_tokens(input_items) > _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS: + return + + payload = cast(dict[str, JsonValue], {"input": deepcopy(input_items)}) + original = deepcopy(payload["input"]) + + _trim_compact_input_for_upstream(payload) + + assert payload["input"] == original + + +@given(size=st.integers(min_value=400_000, max_value=500_000)) +@settings(max_examples=8, deadline=None) +def test_compact_trim_keeps_budget_order_and_is_stable(size): + input_items = [ + {"id": "head", "role": "user", "content": "head"}, + {"id": "middle", "role": "assistant", "content": "x" * size}, + {"id": "latest", "role": "user", "content": "latest"}, + ] + payload = cast(dict[str, JsonValue], {"input": input_items}) + + _trim_compact_input_for_upstream(payload) + trimmed_input = cast(list[JsonValue], deepcopy(payload["input"])) + + assert _estimated_json_tokens(trimmed_input) <= _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS + retained_ids = [item["id"] for item in trimmed_input if isinstance(item, dict) and isinstance(item.get("id"), str)] + assert retained_ids == ["head", "latest"] + + _trim_compact_input_for_upstream(payload) + assert payload["input"] == trimmed_input + + +@given(size=st.integers(min_value=400_000, max_value=500_000)) +@settings(max_examples=8, deadline=None) +def test_compact_trim_marker_accounts_for_omitted_middle_item(size): + input_items = [ + {"id": "head", "role": "user", "content": "head"}, + {"id": "middle", "role": "assistant", "content": "x" * size}, + {"id": "latest", "role": "user", "content": "latest"}, + ] + payload = cast(dict[str, JsonValue], {"input": input_items}) + + _trim_compact_input_for_upstream(payload) + trimmed_input = cast(list[JsonValue], payload["input"]) + + marker = next( + item + for item in trimmed_input + if isinstance(item, dict) and "[compact trim] Omitted " in str(item.get("content")) + ) + marker_text = str(marker["content"]) + match = re.search(r"Omitted (\d+) input items \(~(\d+) estimated tokens\)", marker_text) + + assert match is not None + assert match.groups() == ("1", str(_estimated_json_array_item_tokens(cast(JsonValue, input_items[1])))) + + +@given( + pair=st.sampled_from( + [ + ("function_call", "function_call_output"), + ("custom_tool_call", "custom_tool_call_output"), + ("apply_patch_call", "apply_patch_call_output"), + ] + ), + filler_size=st.integers(min_value=300_000, max_value=400_000), +) +@settings(max_examples=8, deadline=None) +def test_compact_trim_keeps_generated_tool_pairs(pair, filler_size): + call_type, output_type = pair + call = { + "type": call_type, + "name": "exec_command", + "call_id": "call-generated", + "arguments" if call_type == "function_call" else "input": "{}", + } + if call_type == "apply_patch_call": + call = { + "type": call_type, + "call_id": "call-generated", + "operation": {"patch": "noop"}, + } + output = {"type": output_type, "call_id": "call-generated", "output": "result"} + payload = cast( + dict[str, JsonValue], + { + "input": [ + {"role": "assistant", "content": "x" * filler_size}, + call, + output, + ] + }, + ) + + _trim_compact_input_for_upstream(payload) + trimmed_input = cast(list[JsonValue], payload["input"]) + + assert call in trimmed_input + assert output in trimmed_input + assert _estimated_json_tokens(trimmed_input) <= _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS + + +@given(anchor=st.sampled_from(["goal", "plan"]), filler_size=st.integers(350_000, 450_000)) +@settings(max_examples=8, deadline=None) +def test_compact_trim_keeps_generated_state_anchor(anchor, filler_size): + anchor_text = ( + 'continue the goal' + if anchor == "goal" + else "# Plan Mode" + ) + anchor_item = { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": anchor_text}], + } + payload = cast( + dict[str, JsonValue], + { + "input": [ + {"role": "user", "content": "head"}, + {"role": "assistant", "content": "x" * filler_size}, + anchor_item, + {"role": "user", "content": "latest"}, + ] + }, + ) + + _trim_compact_input_for_upstream(payload) + trimmed_input = cast(list[JsonValue], payload["input"]) + + assert anchor_item in trimmed_input + assert _estimated_json_tokens(trimmed_input) <= _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS + + +@given(size=st.integers(min_value=400_000, max_value=500_000)) +@settings(max_examples=8, deadline=None) +def test_compact_trim_rejects_generated_oversized_latest_item(size): + payload = cast( + dict[str, JsonValue], + { + "input": [ + {"role": "assistant", "content": "head"}, + {"role": "user", "content": "x" * size}, + ] + }, + ) + + with pytest.raises(ClientPayloadError) as raised: + _trim_compact_input_for_upstream(payload) + + assert raised.value.param == "input" + assert raised.value.code == "responses_compact_input_too_large" + + def test_compact_trims_oversized_input_by_estimated_tokens_with_head_tail_and_marker(): input_items = [ {"role": "user", "content": "initial goal and instructions"}, diff --git a/tests/unit/test_proxy_api_responses_contract.py b/tests/unit/test_proxy_api_responses_contract.py index 5dea95fd97..9ba8426fb8 100644 --- a/tests/unit/test_proxy_api_responses_contract.py +++ b/tests/unit/test_proxy_api_responses_contract.py @@ -165,6 +165,48 @@ async def fail_release(value: object) -> None: assert "Failed to release API key reservation after rate-limit header failure" in caplog.text +@pytest.mark.asyncio +async def test_rate_limit_header_failure_uses_reservation_cleanup_retry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + reservation = object() + header_failure = RuntimeError("rate-limit header failure") + released: list[str] = [] + + async def fail_headers(*_args: object) -> dict[str, str]: + raise header_failure + + async def record_release( + value: object, + *, + action: str, + scheduler: object, + request_id: str, + ) -> None: + del value, scheduler, request_id + released.append(action) + + monkeypatch.setattr(proxy_api_module, "_rate_limit_headers_for_request", fail_headers) + monkeypatch.setattr(proxy_api_module, "_release_reservation_best_effort", record_release) + cleanup = proxy_api_module._ResponsesReservationCleanup( + owns_reservation=True, + reservation=cast(Any, reservation), + scheduler=None, + request_id="req_header_cleanup", + ) + + with pytest.raises(RuntimeError) as caught: + await proxy_api_module._rate_limit_headers_with_reservation_cleanup( + cast(Any, object()), + None, + cast(Any, reservation), + reservation_cleanup=cleanup, + ) + + assert caught.value is header_failure + assert released == ["rate limit headers"] + + def test_strip_blank_reasoning_comment_preserves_unmatched_whitespace_and_inline_comments() -> None: assert proxy_api_module._strip_blank_html_comment_lines("Need more steps\n") == "Need more steps\n" assert proxy_api_module._strip_blank_html_comment_lines("Hard break \n") == "Hard break \n" @@ -381,6 +423,26 @@ def test_compact_response_output_item_preserves_summary_item_id() -> None: } +def test_compact_response_output_item_drops_invalid_id_prefix() -> None: + payload = CompactResponsePayload.model_validate( + { + "object": "response.compaction", + "output": [ + { + "id": "msg_compact_context", + "type": "compaction", + "encrypted_content": "COMPACT_CONTEXT", + } + ], + } + ) + + assert proxy_api_module._compact_response_output_item(payload) == { + "type": "compaction", + "encrypted_content": "COMPACT_CONTEXT", + } + + def test_compact_response_id_generates_unique_fallback(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(proxy_api_module, "get_request_id", lambda: None) payload = CompactResponsePayload.model_validate({"object": "response.compaction"}) @@ -1764,6 +1826,38 @@ async def test_normalize_public_stream_passes_canonical_unmutated_blocks_verbati assert delta in blocks +@pytest.mark.asyncio +async def test_normalize_public_stream_passes_raw_utf8_verbatim_blocks_byte_identically() -> None: + """Upstream-verbatim delta blocks (raw UTF-8, upstream key spacing — not + the ensure_ascii canonical re-encode) still satisfy the identity + pass-through gate: it compares parsed-payload object identity plus the + `event:` framing prefix, never re-serialized bytes.""" + created = proxy_api_module.format_sse_event( + {"type": "response.created", "response": {"id": "resp_utf8", "output": []}} + ) + verbatim_delta = ( + "event: response.output_text.delta\n" + 'data: {"type": "response.output_text.delta", "item_id": "msg_1", "output_index": 0, "delta": "안녕"}\n\n' + ) + completed_payload: dict[str, Any] = { + "type": "response.completed", + "response": { + "id": "resp_utf8", + "output": [{"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "안녕"}]}], + }, + } + completed = proxy_api_module.format_sse_event(completed_payload) + + blocks = [ + block + async for block in proxy_api_module._normalize_public_responses_stream( + _iter_blocks(created, verbatim_delta, completed) + ) + ] + + assert verbatim_delta in blocks + + @pytest.mark.asyncio async def test_normalize_public_stream_reframes_data_only_blocks_with_event_name() -> None: """A data-only block (e.g. bridge-rewritten terminal event) must regain diff --git a/tests/unit/test_proxy_api_websocket_auth.py b/tests/unit/test_proxy_api_websocket_auth.py index 7a896cad75..13517982fd 100644 --- a/tests/unit/test_proxy_api_websocket_auth.py +++ b/tests/unit/test_proxy_api_websocket_auth.py @@ -15,6 +15,7 @@ import app.core.auth.dependencies as auth_dependencies import app.core.request_locality as request_locality import app.modules.proxy.api as proxy_api_module +import app.modules.proxy.request_policy as proxy_request_policy from app.core.clients.proxy import ProxyResponseError from app.core.errors import openai_error from app.core.exceptions import ProxyAuthError @@ -393,7 +394,7 @@ async def test_stream_responses_prefers_forwarded_downstream_turn_state(monkeypa def fake_apply_api_key_enforcement(_payload, _api_key, *, prohibit_fast_mode=False): assert prohibit_fast_mode is False - return None + return proxy_request_policy.ApiKeyEnforcementResult(False, None) def fake_validate_model_access(_api_key, _model): return None @@ -557,7 +558,7 @@ async def test_stream_responses_does_not_release_forwarded_reservation_on_intern def fake_apply_api_key_enforcement(_payload, _api_key, *, prohibit_fast_mode=False): assert prohibit_fast_mode is False - return None + return proxy_request_policy.ApiKeyEnforcementResult(False, None) def fake_validate_model_access(_api_key, _model): return None @@ -630,6 +631,35 @@ def test_public_previous_response_not_found_error_is_masked_to_stream_incomplete assert "resp_missing" not in masked.model_dump_json() +def test_public_previous_response_not_found_can_enable_client_full_history_recovery( + monkeypatch: pytest.MonkeyPatch, +) -> None: + envelope = proxy_api_module.OpenAIErrorEnvelopeModel( + error=proxy_api_module.OpenAIError( + message="Previous response with id 'resp_missing' not found.", + type="invalid_request_error", + code="previous_response_not_found", + param="previous_response_id", + ) + ) + monkeypatch.setattr( + proxy_api_module, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="client_full_history_once" + ), + ) + + status_code, preserved = proxy_api_module._mask_previous_response_not_found_error( + envelope, + default_status=400, + allow_client_full_history_once=True, + ) + + assert status_code == 400 + assert preserved == envelope + + def test_public_previous_response_invalid_request_param_is_masked_to_stream_incomplete(): envelope = proxy_api_module.OpenAIErrorEnvelopeModel( error=proxy_api_module.OpenAIError( diff --git a/tests/unit/test_proxy_errors.py b/tests/unit/test_proxy_errors.py index 083d6b437f..7d24526fd4 100644 --- a/tests/unit/test_proxy_errors.py +++ b/tests/unit/test_proxy_errors.py @@ -1,16 +1,86 @@ from __future__ import annotations import json +from collections.abc import AsyncIterator +from types import SimpleNamespace import pytest from starlette.requests import Request from app.core.clients.proxy import ProxyResponseError, _error_event_from_response, _error_payload_from_response +from app.core.exceptions import ProxyRateLimitError +from app.core.openai.requests import ResponsesRequest +from app.modules.proxy import api as proxy_api from app.modules.proxy.api import _logged_error_json_response, _stream_response_error_events pytestmark = pytest.mark.unit +def test_http_bridge_recovery_eligibility_accepts_turn_state_anchor_without_previous_response( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + proxy_api.proxy_service_module, + "get_settings", + lambda: SimpleNamespace(http_responses_session_bridge_operation_ledger_enabled=True), + ) + payload = ResponsesRequest(model="gpt-5.6", instructions="", input="retry") + + assert ( + proxy_api._http_bridge_recovery_request_eligible( + payload, + bridge_active=True, + headers={"x-codex-turn-state": "turn-1"}, + ) + is True + ) + assert ( + proxy_api._http_bridge_recovery_request_eligible( + payload, + bridge_active=True, + headers={}, + ) + is False + ) + + +def test_http_bridge_indefinite_recovery_defers_predecessor_proof_to_submit_path( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + proxy_api.proxy_service_module, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_operation_ledger_enabled=True, + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery", + ), + ) + fresh_turn = ResponsesRequest(model="gpt-5.6", instructions="", input="retry") + anchored_turn = ResponsesRequest( + model="gpt-5.6", + instructions="", + input="retry", + previous_response_id="resp_parent", + ) + + assert ( + proxy_api._http_bridge_recovery_request_eligible( + fresh_turn, + bridge_active=True, + headers={"x-codex-turn-state": "turn-1"}, + ) + is True + ) + assert ( + proxy_api._http_bridge_recovery_request_eligible( + anchored_turn, + bridge_active=True, + headers={"x-codex-turn-state": "turn-1"}, + ) + is True + ) + + def test_logged_error_json_response_preserves_upstream_diagnostic_markers(): message = "Provider Exception: failed while reading /tmp/upstream-cache" request = Request({"type": "http", "method": "POST", "path": "/v1/responses", "headers": []}) @@ -76,6 +146,165 @@ async def stream(): assert events[0].startswith("retry: 2000\n") +@pytest.mark.asyncio +async def test_indefinite_recovery_does_not_retry_after_downstream_event(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + proxy_api, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery" + ), + ) + + async def recovery() -> AsyncIterator[str]: + raise AssertionError("recovery must not run after a downstream event") + yield "" + + async def stream(): + yield 'data: {"type":"response.created"}\n\n' + raise ProxyResponseError( + 502, + {"error": {"code": "stream_incomplete", "message": "closed", "type": "server_error"}}, + ) + + events = [ + event + async for event in _stream_response_error_events( + stream(), + owns_reservation=False, + reservation=None, + recovery_stream_factory=recovery, + ) + ] + + assert len(events) == 2 + assert "response.created" in events[0] + assert "response.failed" in events[1] + + +@pytest.mark.asyncio +async def test_indefinite_recovery_converts_retry_reservation_failure_to_sse(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + proxy_api, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery" + ), + ) + monkeypatch.setattr(proxy_api.asyncio, "sleep", lambda _delay: _completed_asyncio_sleep()) + + async def stream(): + if False: + yield "" + raise ProxyResponseError( + 502, + {"error": {"code": "stream_incomplete", "message": "closed", "type": "server_error"}}, + ) + + async def recovery_stream(): + raise ProxyRateLimitError("quota exhausted") + yield "" + + events = [ + event + async for event in _stream_response_error_events( + stream(), + owns_reservation=False, + reservation=None, + recovery_stream_factory=lambda: recovery_stream(), + ) + ] + + assert any("rate_limit_exceeded" in event and "response.failed" in event for event in events) + + +@pytest.mark.asyncio +async def test_indefinite_recovery_converts_unexpected_admission_failure_to_sse( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr( + proxy_api, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery" + ), + ) + monkeypatch.setattr(proxy_api.asyncio, "sleep", lambda _delay: _completed_asyncio_sleep()) + + async def stream(): + if False: + yield "" + raise ProxyResponseError( + 502, + {"error": {"code": "stream_incomplete", "message": "closed", "type": "server_error"}}, + ) + + async def recovery_stream(): + raise RuntimeError("durable admission database unavailable") + yield "" + + events = [ + event + async for event in _stream_response_error_events( + stream(), + owns_reservation=False, + reservation=None, + recovery_stream_factory=lambda: recovery_stream(), + ) + ] + + assert any("bridge_recovery_admission_failed" in event and "response.failed" in event for event in events) + + +@pytest.mark.asyncio +async def test_indefinite_recovery_stops_after_retry_output_then_transport_error( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr( + proxy_api, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery" + ), + ) + monkeypatch.setattr(proxy_api.asyncio, "sleep", lambda _delay: _completed_asyncio_sleep()) + attempts = 0 + + async def stream(): + if False: + yield "" + raise ProxyResponseError( + 502, + {"error": {"code": "stream_incomplete", "message": "closed", "type": "server_error"}}, + ) + + async def recovery_stream(): + nonlocal attempts + attempts += 1 + yield 'data: {"type":"response.created"}\n\n' + raise ProxyResponseError( + 502, + {"error": {"code": "upstream_request_timeout", "message": "stalled", "type": "server_error"}}, + ) + + events = [ + event + async for event in _stream_response_error_events( + stream(), + owns_reservation=False, + reservation=None, + recovery_stream_factory=lambda: recovery_stream(), + ) + ] + + assert attempts == 1 + assert any('"type":"response.created"' in event for event in events) + + +async def _completed_asyncio_sleep(_delay: float = 0.0) -> None: + return None + + def _payload_error_code(payload) -> str | None: return payload["error"].get("code") diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 7702506297..ce9033e736 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -9,24 +9,25 @@ import subprocess import time from collections import deque +from collections.abc import Callable from contextlib import nullcontext from dataclasses import replace from datetime import UTC, datetime, timedelta, timezone from types import SimpleNamespace -from typing import Any, cast +from typing import Any, Mapping, cast from unittest.mock import AsyncMock, Mock import aiohttp import anyio import pytest from fastapi import WebSocket +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from websockets.exceptions import ConnectionClosedError from websockets.frames import Close from app.core.auth.refresh import RefreshError from app.core.clients.proxy import CODEX_RESPONSES_LITE_WEBSOCKET_METADATA_KEY, ProxyResponseError from app.core.clients.proxy_websocket import ( - UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE, CodexUpstreamWebSocket, UpstreamWebSocket, @@ -36,9 +37,9 @@ ) from app.core.config.settings import Settings from app.core.errors import openai_error -from app.core.types import JsonValue from app.core.utils.request_id import get_request_id, reset_request_scope_id, set_request_scope_id -from app.db.models import AccountStatus, HttpBridgeSessionState +from app.db.models import AccountStatus, Base, HttpBridgeSessionState +from app.modules.proxy import affinity as proxy_affinity from app.modules.proxy import http_bridge_forwarding as http_bridge_forwarding_module from app.modules.proxy import service as proxy_service from app.modules.proxy._service import support as proxy_support_module @@ -59,9 +60,10 @@ DurableBridgeAliasRegistration, DurableBridgeAliasRegistrationReceipt, ) +from app.modules.proxy.durable_bridge_runtime import http_bridge_owner_process_epoch +from app.modules.proxy.http_bridge_event_batcher import TerminalOperationEventAppendResult from app.modules.proxy.http_bridge_forwarding import OwnerForwardRelayFailure from app.modules.proxy.load_balancer import CONTINUITY_OWNER_UNAVAILABLE, CatalogOmissionQuotaAdmission -from app.modules.proxy.response_transition_manifest import build_response_transition_manifest pytestmark = pytest.mark.unit @@ -119,33 +121,6 @@ def test_http_bridge_dead_owner_epoch_uses_standard_previous_response_not_found_ assert proxy_error.payload["error"]["param"] == "previous_response_id" -def test_full_resend_suffix_shape_diagnostic_is_bounded_and_content_free() -> None: - input_items: list[JsonValue] = [ - {"role": "user", "content": "stored secret"}, - {"type": "reasoning", "encrypted_content": "opaque secret"}, - { - "type": "agent_message", - "author": "/root/private_agent", - "recipient": "/root", - "content": [{"type": "input_text", "text": "private result"}], - }, - {"type": "message", "role": "user", "content": "private retry"}, - {"type": "unknown-user-controlled-type", "secret": "not logged"}, - {"type": ["unhashable", "type"], "secret": "not logged"}, - {"type": "message", "role": {"unhashable": "role"}, "secret": "not logged"}, - *[{"type": "input_text", "text": f"private-{index}"} for index in range(6)], - ] - - shape = http_bridge_streaming_module._full_resend_suffix_shape_for_observability( - input_items, - stored_count=1, - ) - - assert shape == "reasoning>agent_message>user>other>other>other>input_part>input_part>more" - assert "private" not in shape - assert "unknown-user-controlled-type" not in shape - - def test_http_bridge_rejected_dead_owner_recovery_code_is_not_emitted_from_app() -> None: result = subprocess.run( ["git", "grep", "-n", "bridge_continuity_recovery_required", "--", "app/"], @@ -157,32 +132,6 @@ def test_http_bridge_rejected_dead_owner_recovery_code_is_not_emitted_from_app() assert result.returncode == 1, result.stdout -def test_custom_semantic_rebase_request_paths_are_disabled_by_default() -> None: - lookup = Mock() - - assert http_bridge_streaming_module._DURABLE_RECOVERY_MARKER_REQUEST_PATH_ENABLED is False - assert http_bridge_streaming_module._ROWLESS_SEMANTIC_REBASE_REQUEST_PATH_ENABLED is False - assert http_bridge_streaming_module._durable_recovery_marker_request_path_active(lookup) is False - lookup.recovery_is_required_for_latest_anchor.assert_not_called() - - -@pytest.mark.parametrize( - ("detail", "expected"), - [ - ("stream_idle_timeout", "repeated_zero_event_idle_timeout"), - ("missing_response_created_timeout", "repeated_zero_event_idle_timeout"), - ("stream_incomplete", "repeated_zero_event_stream_incomplete"), - ("clean_close", None), - (None, None), - ], -) -def test_http_bridge_anchor_poison_detail_matches_upstream_contract( - detail: str | None, - expected: str | None, -) -> None: - assert http_bridge_retry_circuit_module._http_bridge_anchor_poison_detail(detail) == expected - - @pytest.fixture(autouse=True) def _share_proxy_dashboard_settings(monkeypatch: pytest.MonkeyPatch) -> None: class _SettingsCache: @@ -213,8533 +162,8275 @@ def _without_installation_metadata(text: str) -> dict[str, Any]: return payload -def _make_app_settings(*, bridge_enabled: bool = True, **overrides: Any) -> Settings: - return Settings(http_responses_session_bridge_enabled=bridge_enabled, **overrides) - - -def _make_bridge_session( - *, - key: proxy_service._HTTPBridgeSessionKey | None = None, - key_value: str = "bridge-test", - pending_requests: deque[proxy_service._WebSocketRequestState] | None = None, - queued_request_count: int = 0, -) -> proxy_service._HTTPBridgeSession: - session_key = key or proxy_service._HTTPBridgeSessionKey("session_header", key_value, None) - return proxy_service._HTTPBridgeSession( - key=session_key, - headers={"x-codex-session-id": key_value}, - affinity=proxy_service._AffinityPolicy( - key=key_value, - kind=proxy_service.StickySessionKind.CODEX_SESSION, - ), - request_model="gpt-5.2", - account=cast(Any, SimpleNamespace(id="acc-bridge", status=AccountStatus.ACTIVE, plan_type="plus")), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=pending_requests or deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=queued_request_count, - last_used_at=1.0, - idle_ttl_seconds=120.0, - ) - - -@pytest.mark.parametrize( - ("namespace", "expected"), - [ - ("collaboration", True), - (" ", False), - (7, False), - ], -) -def test_http_bridge_account_neutral_replay_validates_namespaced_tool_call_history( - namespace: JsonValue, - expected: bool, -) -> None: - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.6-sol", - "instructions": "", - "input": [ - {"role": "user", "content": "old request"}, - { - "type": "function_call", - "namespace": namespace, - "call_id": "call_1", - "name": "spawn_agent", - "arguments": "{}", - }, - {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, - {"role": "user", "content": "next request"}, - ], - } - ) - - assert http_bridge_streaming_module._http_bridge_payload_is_account_neutral_fresh_replay(payload) is expected +def test_http_bridge_operation_metadata_is_stable_and_non_destructive() -> None: + payload = {"type": "response.create", "previous_response_id": "resp_parent", "input": "continue"} + text = json.dumps(payload) + with_operation = http_bridge_request_submit_module._text_with_operation_id(text, "op_test") + decoded = json.loads(with_operation) + assert decoded["previous_response_id"] == "resp_parent" + assert decoded["client_metadata"] == {"codex_lb_operation_id": "op_test"} + assert http_bridge_request_submit_module._text_with_operation_id(with_operation, "op_test") == with_operation + supplied = '{"type":"response.create","client_metadata":{"codex_lb_operation_id":"caller-value"}}' + supplied_result = json.loads(http_bridge_request_submit_module._text_with_operation_id(supplied, "op_test")) + assert supplied_result["client_metadata"]["codex_lb_operation_id"] == "op_test" + normalized = http_bridge_request_submit_module._text_without_operation_id(supplied) + assert json.loads(normalized) == {"type": "response.create"} -def _make_eventless_http_bridge_owner( - *, - request_id: str = "req-eventless-owner", - sent_at: float = 100.0, -) -> proxy_service._WebSocketRequestState: - return proxy_service._WebSocketRequestState( - request_id=request_id, - model="gpt-5.6-sol", +def test_http_bridge_inserts_previous_response_id_for_hard_turn_advance() -> None: + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-next", + model="gpt-5.6", service_tier=None, - reasoning_effort="high", + reasoning_effort=None, api_key_reservation=None, - started_at=-10_000.0, - transport="http", - response_create_gate=asyncio.Semaphore(0), - response_create_gate_acquired=True, - awaiting_response_created=True, - response_create_sent_at=sent_at, - event_queue=asyncio.Queue(), + started_at=0.0, + hard_continuity_anchor=True, + ) + completed_operation = SimpleNamespace( + state="completed", + event_spool_complete=True, + response_id="resp-prior-turn", ) - -class _SilentEventlessUpstream: - """Upstream double that never produces a response event, for eventless-timeout tests.""" - - def __init__(self) -> None: - self.first_receive_started = asyncio.Event() - self.receive_cancelled = asyncio.Event() - self.closed = False - - async def receive(self) -> UpstreamWebSocketMessage: - self.first_receive_started.set() - try: - await asyncio.Event().wait() - except asyncio.CancelledError: - self.receive_cancelled.set() - raise - raise AssertionError("unreachable") - - async def send_text(self, text: str) -> None: - pass - - async def close(self) -> None: - self.closed = True - - -def test_http_bridge_eventless_precreated_deadline_uses_current_send_and_client_safe_cap() -> None: - request_state = _make_eventless_http_bridge_owner() - + response_id = http_bridge_request_submit_module._http_bridge_terminal_hard_turn_response_id( + request_state, + completed_operation, + ) + assert response_id == "resp-prior-turn" assert ( - http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( + http_bridge_request_submit_module._http_bridge_terminal_hard_turn_response_id( request_state, - stuck_gate_retire_after_seconds=300.0, + SimpleNamespace( + state="completed", + event_spool_complete=False, + response_id="resp-completed-but-unsynced", + ), ) - == 160.0 + == "resp-completed-but-unsynced" ) assert ( - http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( - request_state, - stuck_gate_retire_after_seconds=30.0, - ) - == 130.0 + json.loads( + http_bridge_request_submit_module._text_with_previous_response_id( + '{"type":"response.create","input":"same"}', + response_id, + ) + )["previous_response_id"] + == "resp-prior-turn" ) - request_state.latency_first_upstream_event_ms = 25 - request_state.last_upstream_activity_at = 150.0 + request_state.replay_count = 1 assert ( - http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( + http_bridge_request_submit_module._http_bridge_terminal_hard_turn_response_id( request_state, - stuck_gate_retire_after_seconds=300.0, + completed_operation, ) - == 160.0 + is None ) - -@pytest.mark.parametrize( - ("field_name", "field_value"), - [ - ("response_id", "resp-created"), - ("latency_response_created_ms", 12), - ("downstream_visible", True), - ("last_downstream_sequence_number", 0), - ("awaiting_response_created", False), - ("response_create_gate_acquired", False), - ("response_create_gate", None), - ("response_create_sent_at", None), - ], -) -def test_http_bridge_eventless_precreated_deadline_requires_narrow_owner_evidence( - field_name: str, - field_value: object, -) -> None: - request_state = _make_eventless_http_bridge_owner() - setattr(request_state, field_name, field_value) - + request_state.replay_count = 0 + request_state.previous_response_id = "resp-prior-turn" assert ( - http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( + http_bridge_request_submit_module._http_bridge_terminal_hard_turn_response_id( request_state, - stuck_gate_retire_after_seconds=300.0, + SimpleNamespace( + state="completed", + event_spool_complete=True, + response_id="resp-second-turn", + ), + allow_anchored_continuation=True, + ) + == "resp-second-turn" + ) + assert ( + http_bridge_request_submit_module._http_bridge_terminal_hard_turn_response_id( + replace(request_state, previous_response_id=None), + SimpleNamespace( + state="incomplete", + event_spool_complete=True, + response_id="resp-incomplete-turn", + ), ) is None ) -def test_http_bridge_eventless_precreated_deadline_survives_reasoning_prelude_without_created() -> None: - request_state = _make_eventless_http_bridge_owner() - request_state.last_upstream_activity_at = 150.0 - request_state.upstream_model_output_seen = True - request_state.deferred_reasoning_downstream_texts.append( - 'data: {"type":"response.output_item.added","item":{"type":"reasoning"}}\n\n' - ) - - assert ( - http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( - request_state, - stuck_gate_retire_after_seconds=300.0, - ) - == 150.0 + http_bridge_helpers_module._HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS +def test_http_bridge_operation_fingerprint_strips_account_installation_metadata() -> None: + request = ( + '{"type":"response.create","previous_response_id":"resp_parent",' + '"client_metadata":{"x-codex-installation-id":"account-a",' + '"x-codex-turn-metadata":"{\\"installation_id\\":\\"account-a\\",\\"turn_id\\":\\"t1\\"}",' + '"caller":"stable"}}' ) + normalized = json.loads(http_bridge_request_submit_module._text_without_account_installation_id(request)) + assert normalized == { + "type": "response.create", + "previous_response_id": "resp_parent", + "client_metadata": { + "x-codex-turn-metadata": '{"turn_id":"t1"}', + "caller": "stable", + }, + } @pytest.mark.asyncio -async def test_process_http_bridge_upstream_text_anchors_deferred_reasoning_prelude_without_created() -> None: +async def test_submit_hard_turn_walks_completed_operation_chain_before_recording( + monkeypatch: pytest.MonkeyPatch, +) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - request_state = _make_eventless_http_bridge_owner(sent_at=time.monotonic() - 2.0) - session = _make_bridge_session(pending_requests=deque([request_state]), queued_request_count=1) + session = _make_bridge_session(key_value="hard-turn-chain") + session.durable_session_id = "durable-hard-turn-chain" + session.durable_owner_epoch = 4 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-chain", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + hard_continuity_anchor=True, + request_text='{"type":"response.create","input":"same"}', + transport="http", + skip_request_log=True, + ) + completed_operations = iter( + [ + SimpleNamespace(state="completed", event_spool_complete=True, response_id="resp-1"), + SimpleNamespace(state="completed", event_spool_complete=True, response_id="resp-2"), + SimpleNamespace(state="completed", event_spool_complete=True, response_id="resp-3"), + None, + ] + ) + recorded: dict[str, Any] = {} + initial_fingerprint = http_bridge_request_submit_module._http_bridge_operation_fingerprint( + session_id=session.durable_session_id, + api_key_scope="api-key-scope", + request_state=request_state, + text_data=request_state.request_text or "{}", + ) - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "response.output_item.added", - "item": {"type": "reasoning", "id": "rs_1"}, - }, - separators=(",", ":"), + async def get_operation_by_fingerprint(**_kwargs: Any) -> Any: + return next(completed_operations) + + async def get_operation(**_kwargs: Any) -> None: + return None + + async def record_operation(**kwargs: Any) -> Any: + recorded.update(kwargs) + raise RuntimeError("stop after operation identity assertion") + + service._durable_bridge = cast( + Any, + SimpleNamespace( + get_operation_by_fingerprint=get_operation_by_fingerprint, + get_operation=get_operation, + record_operation=record_operation, + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery", + http_responses_session_bridge_instance_id="instance-hard-turn-chain", ), ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_allowed", AsyncMock(return_value=True)) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=0.0)) - assert request_state.response_event_count == 0 - assert request_state.response_id is None - assert request_state.downstream_visible is False - assert request_state.upstream_model_output_seen is True - assert request_state.last_upstream_activity_at is not None - sent_at = request_state.response_create_sent_at - assert sent_at is not None - assert request_state.last_upstream_activity_at > sent_at - assert len(request_state.deferred_reasoning_downstream_texts) == 1 - assert ( - http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( - request_state, - stuck_gate_retire_after_seconds=300.0, + with pytest.raises(ProxyResponseError) as exc_info: + await service._submit_http_bridge_request_with_handoff( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + request_scope_id="scope-hard-turn-chain", + owned_unanchored_handoff=False, ) - == request_state.last_upstream_activity_at - + http_bridge_helpers_module._HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS - ) + + assert exc_info.value.payload["error"]["code"] == "bridge_continuity_persistence_failed" + assert json.loads(recorded["request_text"])["previous_response_id"] == "resp-3" + assert recorded["parent_response_id"] == "resp-3" + assert recorded["request_fingerprint"] != initial_fingerprint + assert json.loads(request_state.request_text or "{}")["previous_response_id"] == "resp-3" @pytest.mark.asyncio -async def test_http_bridge_send_replaces_timestamp_and_wakes_existing_reader( +async def test_submit_hard_turn_walks_race_path_chain_before_recording( monkeypatch: pytest.MonkeyPatch, ) -> None: - request_state = _make_eventless_http_bridge_owner(sent_at=1.0) - session = _make_bridge_session() - seen_sent_ats: list[float | None] = [] - - async def send_text(_text: str) -> None: - seen_sent_ats.append(request_state.response_create_sent_at) - - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace(send_text=send_text, close=AsyncMock()), - ) - monotonic_values = iter((100.0, 200.0)) - monotonic = lambda: next(monotonic_values) # noqa: E731 - monkeypatch.setattr( - http_bridge_request_submit_module, - "_service_time", - lambda: SimpleNamespace(monotonic=monotonic), + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="hard-turn-race-chain") + session.durable_session_id = "durable-hard-turn-race-chain" + session.durable_owner_epoch = 4 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-race-chain", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + hard_continuity_anchor=True, + request_text='{"type":"response.create","input":"same"}', + transport="http", + skip_request_log=True, ) - - await http_bridge_request_submit_module._send_http_bridge_request_text_with_archive_id( - session, - request_state, - "first", + operation_lookups = iter( + [ + None, + SimpleNamespace(state="completed", event_spool_complete=True, response_id="resp-2"), + None, + ] ) - session.upstream_reader_wakeup.clear() - await http_bridge_request_submit_module._send_http_bridge_request_text_with_archive_id( - session, - request_state, - "second", + latest_completed = iter( + [ + SimpleNamespace(state="completed", event_spool_complete=True, response_id="resp-1"), + None, + ] ) + recorded: dict[str, Any] = {} - assert seen_sent_ats == [100.0, 200.0] - assert request_state.response_create_sent_at == 200.0 - assert session.upstream_reader_wakeup.is_set() is True + async def get_operation_by_fingerprint(**_kwargs: Any) -> Any: + return next(operation_lookups) + async def get_operation(**_kwargs: Any) -> None: + return None -@pytest.mark.asyncio -@pytest.mark.parametrize("failure", [RuntimeError("send failed"), asyncio.CancelledError()]) -async def test_http_bridge_failed_send_disarms_eventless_deadline( - monkeypatch: pytest.MonkeyPatch, - failure: BaseException, -) -> None: - request_state = _make_eventless_http_bridge_owner(sent_at=1.0) - session = _make_bridge_session() + async def get_latest_completed_operation(**_kwargs: Any) -> Any: + return next(latest_completed) - async def send_text(_text: str) -> None: - assert request_state.response_create_sent_at == 100.0 - raise failure + async def record_operation(**kwargs: Any) -> Any: + recorded.update(kwargs) + raise RuntimeError("stop after operation identity assertion") - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace(send_text=send_text, close=AsyncMock()), + service._durable_bridge = cast( + Any, + SimpleNamespace( + get_operation_by_fingerprint=get_operation_by_fingerprint, + get_operation=get_operation, + get_latest_completed_operation=get_latest_completed_operation, + record_operation=record_operation, + ), ) monkeypatch.setattr( - http_bridge_request_submit_module, - "_service_time", - lambda: SimpleNamespace(monotonic=lambda: 100.0), + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery", + http_responses_session_bridge_instance_id="instance-hard-turn-race-chain", + ), ) - session.upstream_reader_wakeup.clear() + monkeypatch.setattr(service, "_http_bridge_precreated_retry_allowed", AsyncMock(return_value=True)) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=0.0)) - with pytest.raises(type(failure)): - await http_bridge_request_submit_module._send_http_bridge_request_text_with_archive_id( + with pytest.raises(ProxyResponseError) as exc_info: + await service._submit_http_bridge_request_with_handoff( session, - request_state, - "request", - ) - - assert request_state.response_create_sent_at is None - assert session.upstream_reader_wakeup.is_set() is True - assert ( - http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( - request_state, - stuck_gate_retire_after_seconds=300.0, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + request_scope_id="scope-hard-turn-race-chain", + owned_unanchored_handoff=False, ) - is None - ) + assert exc_info.value.payload["error"]["code"] == "bridge_continuity_persistence_failed" + assert json.loads(recorded["request_text"])["previous_response_id"] == "resp-2" + assert recorded["parent_response_id"] == "resp-2" + assert json.loads(request_state.request_text or "{}")["previous_response_id"] == "resp-2" + assert request_state.proxy_injected_previous_response_id is True -def _make_account_neutral_replay_session_key( - nonce: str, - api_key_id: str | None = None, -) -> proxy_service._HTTPBridgeSessionKey: - kind, key = make_http_bridge_account_neutral_replay_key(nonce) - return proxy_service._HTTPBridgeSessionKey(kind, key, api_key_id) +def test_ambiguous_continuation_recovery_is_opt_in_and_requires_unobserved_anchor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + request_state = proxy_service._WebSocketRequestState( + request_id="req-recovery", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + previous_response_id="resp-parent", + response_event_count=0, + response_id=None, + fresh_upstream_request_is_retry_safe=False, + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace(http_responses_session_bridge_ambiguous_continuation_recovery_mode="fail_closed"), + ) + assert http_bridge_streaming_module._http_bridge_client_full_history_recovery_enabled(request_state) is False -def test_forwarded_fork_keeps_authenticated_original_unanchored_state() -> None: - assert http_bridge_helpers_module._http_bridge_request_needs_unanchored_handoff( - proxy_service._HTTPBridgeSessionKey( - "internal_unanchored_parallel", - "fork-key", - None, + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="client_full_history_once" ), - "http_turn_generated", - None, - True, - True, ) + assert http_bridge_streaming_module._http_bridge_client_full_history_recovery_enabled(request_state) is True + request_state.propagate_http_errors = True + assert http_bridge_request_submit_module._http_bridge_client_full_history_recovery_enabled(request_state) is True + request_state.response_event_count = 1 + assert http_bridge_streaming_module._http_bridge_client_full_history_recovery_enabled(request_state) is False + assert http_bridge_request_submit_module._http_bridge_client_full_history_recovery_enabled(request_state) is False -def test_verified_replay_model_fork_preserves_recovery_kind() -> None: - replay_key = _make_account_neutral_replay_session_key("replay-parent") - - fork_key = http_bridge_helpers_module._http_bridge_incompatible_model_fork_key( - key=replay_key, - existing_model="gpt-5.6-sol", - request_model="gpt-5.6-terra", - request_scope_id="request-model-transition", +def test_hard_continuity_operation_fence_requires_server_recovery_mode( + monkeypatch: pytest.MonkeyPatch, +) -> None: + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-fence", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + hard_continuity_anchor=True, ) - - assert fork_key is not None - assert is_http_bridge_account_neutral_replay( - kind=fork_key.affinity_kind, - key=fork_key.affinity_key, + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace(http_responses_session_bridge_ambiguous_continuation_recovery_mode="fail_closed"), ) - assert fork_key != replay_key - - -@pytest.mark.asyncio -async def test_legacy_forward_anchor_lookup_accepts_registered_turn_state_alias() -> None: - key = proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None) - lookup = proxy_service.DurableBridgeLookup( - session_id="durable-1", - canonical_kind="session_header", - canonical_key="sid-123", - api_key_scope="__anonymous__", - account_id="acc-1", - owner_instance_id="instance-b", - owner_epoch=2, - lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_client", - latest_response_id="resp-1", + assert ( + http_bridge_request_submit_module._http_bridge_operation_fence_for_hard_continuity_enabled(request_state) + is False ) - durable_bridge = SimpleNamespace(lookup_turn_state_target=AsyncMock(return_value=lookup)) - - resolved = await http_bridge_streaming_module._legacy_forward_anchor_lookup( - durable_bridge=durable_bridge, - bridge_session_key=key, - turn_state="http_turn_client", - api_key=None, - previous_response_id=None, - forwarded_request=True, - forwarded_legacy_signature=True, + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery" + ), ) - - assert resolved is lookup - durable_bridge.lookup_turn_state_target.assert_awaited_once_with( - turn_state="http_turn_client", - api_key_id=None, + assert ( + http_bridge_request_submit_module._http_bridge_operation_fence_for_hard_continuity_enabled(request_state) + is True + ) + first_fingerprint = http_bridge_request_submit_module._http_bridge_operation_fingerprint( + session_id="durable-a", + api_key_scope="key-scope", + request_state=request_state, + text_data='{"type":"response.create","input":"same"}', + ) + second_fingerprint = http_bridge_request_submit_module._http_bridge_operation_fingerprint( + session_id="durable-b", + api_key_scope="key-scope", + request_state=request_state, + text_data='{"type":"response.create","input":"same"}', + ) + assert first_fingerprint != second_fingerprint + request_state.hard_continuity_anchor = False + assert ( + http_bridge_request_submit_module._http_bridge_operation_fence_for_hard_continuity_enabled(request_state) + is False ) -@pytest.mark.asyncio -async def test_legacy_forward_anchor_lookup_rejects_unknown_generated_turn_state() -> None: - durable_bridge = SimpleNamespace(lookup_turn_state_target=AsyncMock(return_value=None)) - - with pytest.raises(ProxyResponseError) as exc_info: - await http_bridge_streaming_module._legacy_forward_anchor_lookup( - durable_bridge=durable_bridge, - bridge_session_key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), - turn_state="http_turn_generated", - api_key=None, - previous_response_id=None, - forwarded_request=True, - forwarded_legacy_signature=True, - ) - - assert exc_info.value.status_code == 409 - assert exc_info.value.payload["error"]["code"] == "bridge_forward_upgrade_required" - - -@pytest.mark.asyncio -async def test_current_origin_legacy_owner_lookup_rejects_unknown_turn_state_alias() -> None: - durable_bridge = SimpleNamespace(lookup_turn_state_target=AsyncMock(return_value=None)) - - with pytest.raises(ProxyResponseError) as exc_info: - await http_bridge_streaming_module._current_origin_legacy_owner_anchor_lookup( - durable_bridge=durable_bridge, - bridge_session_key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), - turn_state="http_turn_unknown", - api_key=None, - previous_response_id=None, - forwarded_request=False, - ) - - assert exc_info.value.status_code == 409 - assert exc_info.value.payload["error"]["code"] == "bridge_forward_upgrade_required" - durable_bridge.lookup_turn_state_target.assert_awaited_once_with( - turn_state="http_turn_unknown", - api_key_id=None, +def test_http_bridge_durable_recovery_requires_predecessor_anchor() -> None: + fresh_turn = proxy_service._WebSocketRequestState( + request_id="req-fresh-recovery-proof", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + operation_registered=True, + operation_id="op-fresh", ) + anchored_turn = replace( + fresh_turn, + operation_parent_response_id="resp-parent", + ) + + assert http_bridge_streaming_module._http_bridge_durable_recovery_predecessor_proven(fresh_turn) is False + assert http_bridge_streaming_module._http_bridge_durable_recovery_predecessor_proven(anchored_turn) is True @pytest.mark.asyncio -async def test_submit_http_bridge_request_cancellation_releases_published_handoff( +@pytest.mark.parametrize("anchored", [False, True]) +async def test_stream_via_http_bridge_marks_recovery_only_after_parent_proof( monkeypatch: pytest.MonkeyPatch, + anchored: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - session.unanchored_reservation_id = "scope-cancelled-submit" + turn_state = "turn-recovery-proof" + payload_data: dict[str, Any] = {"model": "gpt-5.6", "instructions": "", "input": "retry"} + payload = proxy_service.ResponsesRequest.model_validate(payload_data) request_state = proxy_service._WebSocketRequestState( - request_id="req-cancelled-submit", - model="gpt-5.6-sol", + request_id=f"req-recovery-proof-{anchored}", + model="gpt-5.6", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, - awaiting_response_created=True, - event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', + started_at=time.monotonic(), transport="http", - skip_request_log=True, + previous_response_id=None, + hard_continuity_anchor=True, ) - monkeypatch.setattr(service, "_maybe_prewarm_http_bridge_session", AsyncMock()) - await session.pending_lock.acquire() - request_scope_token = set_request_scope_id("scope-cancelled-submit") - try: - submit_task = asyncio.create_task( - service._submit_http_bridge_request( - session, - request_state=request_state, - text_data=request_state.request_text or "{}", - queue_limit=8, - ) - ) - await asyncio.sleep(0) - submit_task.cancel() - with pytest.raises(asyncio.CancelledError): - await submit_task - finally: - session.pending_lock.release() - reset_request_scope_id(request_scope_token) - - assert session.unanchored_reservation_id is None - - -@pytest.mark.asyncio -async def test_submit_http_bridge_request_early_failure_releases_published_handoff() -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - session.closed = True - session.unanchored_reservation_id = "scope-closed-submit" - request_state = proxy_service._WebSocketRequestState( - request_id="req-closed-submit", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - awaiting_response_created=True, - event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', - transport="http", - skip_request_log=True, + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("turn_state_header", turn_state, None), + key_value=turn_state, ) - request_scope_token = set_request_scope_id("scope-closed-submit") - try: - with pytest.raises(proxy_service.ProxyResponseError): - await service._submit_http_bridge_request( - session, - request_state=request_state, - text_data=request_state.request_text or "{}", - queue_limit=8, - ) - finally: - reset_request_scope_id(request_scope_token) + session.durable_session_id = "durable-recovery-proof" + session.durable_owner_epoch = 1 + session.closed = True - assert session.unanchored_reservation_id is None + def fake_prepare( + _payload: proxy_service.ResponsesRequest, + _headers: Mapping[str, str], + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del api_key, api_key_reservation, request_id, client_ip + return request_state, '{"type":"response.create"}' + async def fail_eventlessly(*_args: Any, **_kwargs: Any): + raise AssertionError("submit should fail before the upstream event reader is entered") + yield "" # pragma: no cover -@pytest.mark.asyncio -@pytest.mark.parametrize("submit_succeeds", [True, False]) -async def test_http_bridge_submit_transfers_settlement_ownership_only_after_success( - monkeypatch: pytest.MonkeyPatch, - submit_succeeds: bool, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - reservation = proxy_service.ApiKeyUsageReservationData( - reservation_id="resv-bridge-submit-owner", - key_id="key-bridge-submit-owner", - model="gpt-5.6-sol", + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery", + ), ) - lifecycle = proxy_support_module._DeferredAccountBackoffLifecycle(reservation=reservation) - request_state = proxy_service._WebSocketRequestState( - request_id="req-bridge-submit-owner", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort=None, - api_key_reservation=reservation, - started_at=time.monotonic(), - event_queue=asyncio.Queue(), - deferred_account_backoff_lifecycle=lifecycle, + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), ) - - async def submit(*_args: object, **_kwargs: object) -> None: - if not submit_succeeds: - raise RuntimeError("send failed before submit returned") - assert request_state.event_queue is not None - request_state.event_queue.put_nowait(None) - - monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock(return_value=False)) - monkeypatch.setattr(service, "_maybe_release_idle_http_bridge_session_lease", AsyncMock(return_value=False)) - stream = service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data="{}", - queue_limit=4, - propagate_http_errors=True, - downstream_turn_state=None, - request_deadline=time.monotonic() + 10, + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service._durable_bridge, "release_live_session", AsyncMock(return_value=None)) + monkeypatch.setattr(service._durable_bridge, "reset_operation_event_spool", AsyncMock(return_value=True)) + completed_operations: list[Any] = ( + [SimpleNamespace(state="completed", event_spool_complete=True, response_id="resp-parent"), None] + if anchored + else [None] ) - if submit_succeeds: - assert [event async for event in stream] == [] - else: - with pytest.raises(RuntimeError, match="send failed"): - async for _ in stream: - pass + async def get_operation_by_fingerprint(**_kwargs: Any) -> Any: + return completed_operations.pop(0) - assert lifecycle.settlement_owned is submit_succeeds + async def get_operation(**_kwargs: Any) -> None: + return None + async def record_operation(**kwargs: Any) -> Any: + request_state.operation_id = kwargs["operation_id"] + return SimpleNamespace( + created=True, + operation_id=kwargs["operation_id"], + state="submitted", + event_spool_complete=False, + response_id=None, + ) -@pytest.mark.asyncio -async def test_http_bridge_request_cleanup_releases_pre_submit_handoff( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="scope-pre-submit") - session.unanchored_reservation_id = "scope-pre-submit" - service._http_bridge_sessions[session.key] = session - runtime_config = SimpleNamespace( - enabled=True, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - prompt_cache_idle_ttl_seconds=120.0, + service._durable_bridge = cast( + Any, + SimpleNamespace( + get_operation_by_fingerprint=get_operation_by_fingerprint, + get_operation=get_operation, + record_operation=record_operation, + lookup_request_targets=AsyncMock(return_value=None), + release_live_session=AsyncMock(return_value=None), + reset_operation_event_spool=AsyncMock(return_value=True), + ), ) + service._http_bridge_sessions[session.key] = session + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-bridge")) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_allowed", AsyncMock(return_value=True)) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=0.0)) + monkeypatch.setattr(service, "_retry_http_bridge_request_on_fresh_upstream", AsyncMock(return_value=False)) - async def fail_before_submit(*args: object, **kwargs: object): - del args, kwargs - raise RuntimeError("payload preparation failed") + async def submit_then_fail( + _session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + propagate_http_errors: bool, + downstream_turn_state: str | None, + request_deadline: float | None = None, + ): + del propagate_http_errors, downstream_turn_state, request_deadline + await service._submit_http_bridge_request_with_handoff( + _session, + request_state=request_state, + text_data=text_data, + queue_limit=queue_limit, + request_scope_id=request_state.request_id, + owned_unanchored_handoff=False, + ) yield "" - monkeypatch.setattr( - http_bridge_streaming_module, - "_service_get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=SimpleNamespace())), - ) - monkeypatch.setattr(http_bridge_streaming_module, "_service_get_settings", _make_app_settings) - monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_runtime_config", lambda *args: runtime_config) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_stream_via_http_bridge", fail_before_submit) - payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.6-sol", "instructions": "test", "input": "hello"} - ) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", submit_then_fail) - request_scope_token = set_request_scope_id("scope-pre-submit") - try: - with pytest.raises(RuntimeError, match="payload preparation failed"): - async for _ in service._stream_http_bridge_or_retry( - payload, - {}, - codex_session_affinity=True, - propagate_http_errors=True, - openai_cache_affinity=False, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - ): - pass - finally: - reset_request_scope_id(request_scope_token) + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_via_http_bridge( + payload, + headers={"x-codex-turn-state": turn_state}, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=900.0, + max_sessions=8, + queue_limit=4, + ): + pass - assert session.unanchored_reservation_id is None + assert getattr(exc_info.value, "http_bridge_durable_recovery_eligible", False) is anchored + if anchored: + assert request_state.previous_response_id == "resp-parent" + assert request_state.operation_parent_response_id == "resp-parent" @pytest.mark.asyncio -async def test_http_bridge_owned_terminal_lifecycle_skips_outer_startup_release( +async def test_hard_continuity_operation_replay_requires_matching_unknown_fence( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - reservation = proxy_service.ApiKeyUsageReservationData( - reservation_id="resv-bridge-owned", - key_id="key-bridge-owned", - model="gpt-5.6-sol", + session = _make_bridge_session(key_value="hard-fence") + session.durable_session_id = "durable-hard-fence" + session.durable_owner_epoch = 3 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-fence-replay", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + hard_continuity_anchor=True, ) - account = cast(Any, SimpleNamespace(id="acc-bridge-owned")) - runtime_config = SimpleNamespace( - enabled=True, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - prompt_cache_idle_ttl_seconds=120.0, + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery" + ), ) + operation = SimpleNamespace( + session_id="durable-hard-fence", + state="unknown", + event_spool_complete=False, + ) + service._durable_bridge = SimpleNamespace(get_operation_by_fingerprint=AsyncMock(return_value=operation)) - async def terminal_before_finalizer(*_args: object, **kwargs: object): - tracker = cast( - proxy_support_module._DeferredAccountBackoffTracker, - kwargs["deferred_account_backoff_tracker"], - ) - lifecycle = proxy_support_module._DeferredAccountBackoffLifecycle( - reservation=reservation, - pending_backoffs={account.id: account}, - settlement_owned=True, + assert ( + await service._http_bridge_operation_fenced_continuity_replay_allowed( + session, + request_state=request_state, + text_data='{"type":"response.create","input":"retry"}', ) - tracker.current_lifecycle = lifecycle - yield 'data: {"type":"response.completed"}\n\n' - - monkeypatch.setattr( - http_bridge_streaming_module, - "_service_get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=SimpleNamespace())), - ) - monkeypatch.setattr(http_bridge_streaming_module, "_service_get_settings", _make_app_settings) - monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_runtime_config", lambda *_args: runtime_config) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_stream_via_http_bridge", terminal_before_finalizer) - release_reservation = AsyncMock() - drain_backoffs = AsyncMock() - monkeypatch.setattr(service, "_release_websocket_reservation", release_reservation) - monkeypatch.setattr(service, "_drain_deferred_account_error_backoffs", drain_backoffs) - payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.6-sol", "instructions": "test", "input": "hello"} + is True ) - chunks = [ - chunk - async for chunk in service._stream_http_bridge_or_retry( - payload, - {}, - codex_session_affinity=True, - propagate_http_errors=True, - openai_cache_affinity=False, - api_key=None, - api_key_reservation=reservation, - suppress_text_done_events=False, + operation.session_id = "different-session" + assert ( + await service._http_bridge_operation_fenced_continuity_replay_allowed( + session, + request_state=request_state, + text_data='{"type":"response.create","input":"retry"}', ) - ] + is False + ) - assert chunks == ['data: {"type":"response.completed"}\n\n'] - release_reservation.assert_not_awaited() - drain_backoffs.assert_not_awaited() +def _make_app_settings(*, bridge_enabled: bool = True, **overrides: Any) -> Settings: + return Settings(http_responses_session_bridge_enabled=bridge_enabled, **overrides) -@pytest.mark.asyncio -@pytest.mark.parametrize("release_fails", [False, True]) -async def test_http_bridge_startup_fallback_releases_current_lifecycle_before_backoff( - monkeypatch: pytest.MonkeyPatch, - release_fails: bool, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - original_reservation = proxy_service.ApiKeyUsageReservationData( - reservation_id="resv-bridge-original", - key_id="key-bridge-startup", - model="gpt-5.6-sol", - ) - current_reservation = proxy_service.ApiKeyUsageReservationData( - reservation_id="resv-bridge-current", - key_id="key-bridge-startup", - model="gpt-5.6-sol", - ) - account = cast(Any, SimpleNamespace(id="acc-bridge-startup")) - runtime_config = SimpleNamespace( - enabled=True, + +def _make_bridge_session( + *, + key: proxy_service._HTTPBridgeSessionKey | None = None, + key_value: str = "bridge-test", + pending_requests: deque[proxy_service._WebSocketRequestState] | None = None, + queued_request_count: int = 0, +) -> proxy_service._HTTPBridgeSession: + session_key = key or proxy_service._HTTPBridgeSessionKey("session_header", key_value, None) + return proxy_service._HTTPBridgeSession( + key=session_key, + headers={"x-codex-session-id": key_value}, + affinity=proxy_service._AffinityPolicy( + key=key_value, + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + request_model="gpt-5.2", + account=cast( + Any, + SimpleNamespace( + id="acc-bridge", + chatgpt_account_id="workspace-bridge", + status=AccountStatus.ACTIVE, + plan_type="plus", + ), + ), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=pending_requests or deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=queued_request_count, + last_used_at=1.0, idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - prompt_cache_idle_ttl_seconds=120.0, ) - tracker_seen: proxy_support_module._DeferredAccountBackoffTracker | None = None - order: list[str] = [] - - async def fail_before_submit(*_args: object, **kwargs: object): - nonlocal tracker_seen - tracker_seen = cast( - proxy_support_module._DeferredAccountBackoffTracker, - kwargs["deferred_account_backoff_tracker"], - ) - tracker_seen.current_lifecycle = proxy_support_module._DeferredAccountBackoffLifecycle( - reservation=current_reservation, - pending_backoffs={account.id: account}, - ) - raise RuntimeError("startup failed") - yield "" - - async def release_reservation(reservation: object) -> None: - assert reservation is current_reservation - order.append("release") - if release_fails: - raise RuntimeError("release failed") - async def drain_backoffs(pending: dict[str, object]) -> None: - assert order == ["release"] - order.append("backoff") - pending.clear() - monkeypatch.setattr( - http_bridge_streaming_module, - "_service_get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=SimpleNamespace())), - ) - monkeypatch.setattr(http_bridge_streaming_module, "_service_get_settings", _make_app_settings) - monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_runtime_config", lambda *_args: runtime_config) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_stream_via_http_bridge", fail_before_submit) - monkeypatch.setattr(service, "_release_websocket_reservation", release_reservation) - monkeypatch.setattr(service, "_drain_deferred_account_error_backoffs", drain_backoffs) +def test_http_bridge_account_neutral_replay_rejects_namespaced_tool_call_history() -> None: payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.6-sol", "instructions": "test", "input": "hello"} + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [ + {"role": "user", "content": "old request"}, + { + "type": "function_call", + "namespace": "collaboration", + "call_id": "call_1", + "name": "spawn_agent", + "arguments": "{}", + }, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + {"role": "user", "content": "next request"}, + ], + } ) - with pytest.raises(RuntimeError, match="startup failed"): - async for _ in service._stream_http_bridge_or_retry( - payload, - {}, - codex_session_affinity=True, - propagate_http_errors=True, - openai_cache_affinity=False, - api_key=None, - api_key_reservation=original_reservation, - suppress_text_done_events=False, - ): - pass + assert http_bridge_streaming_module._http_bridge_payload_is_account_neutral_fresh_replay(payload) is False - assert tracker_seen is not None - assert order == (["release"] if release_fails else ["release", "backoff"]) - assert tracker_seen.current_lifecycle is not None - assert tracker_seen.current_lifecycle.settlement_confirmed is not release_fails - assert bool(tracker_seen.current_lifecycle.pending_backoffs) is release_fails + +def _make_eventless_http_bridge_owner( + *, + request_id: str = "req-eventless-owner", + sent_at: float = 100.0, +) -> proxy_service._WebSocketRequestState: + return proxy_service._WebSocketRequestState( + request_id=request_id, + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort="high", + api_key_reservation=None, + started_at=-10_000.0, + transport="http", + response_create_gate=asyncio.Semaphore(0), + response_create_gate_acquired=True, + awaiting_response_created=True, + response_create_sent_at=sent_at, + event_queue=asyncio.Queue(), + ) @pytest.mark.asyncio -async def test_durable_turn_state_fence_rejection_rolls_back_local_alias( +@pytest.mark.parametrize("include_sibling", [False, True]) +async def test_http_bridge_eventless_anchored_precreated_retry_stays_fail_closed( monkeypatch: pytest.MonkeyPatch, + include_sibling: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - session.durable_session_id = "durable-session" - session.durable_owner_epoch = 3 - service._http_bridge_sessions[session.key] = session - service._durable_bridge = SimpleNamespace( - register_turn_state=AsyncMock(return_value=DurableBridgeAliasRegistration.OWNER_FENCED) - ) - monkeypatch.setattr(http_bridge_helpers_module, "get_settings", _make_app_settings) - - await service._register_http_bridge_turn_state(session, "turn-rejected") - - alias_key = proxy_service._http_bridge_turn_state_alias_key("turn-rejected", session.key.api_key_id) - assert "turn-rejected" not in session.downstream_turn_state_aliases - assert session.downstream_turn_state is None - assert alias_key not in service._http_bridge_turn_state_index + owner = _make_eventless_http_bridge_owner() + owner.request_text = '{"type":"response.create","input":"continue"}' + owner.previous_response_id = "resp-parent" + pending_requests = deque([owner]) + queued_request_count = 1 + if include_sibling: + sibling = proxy_service._WebSocketRequestState( + request_id="req-created-sibling", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + response_id="resp-created-sibling", + ) + pending_requests.append(sibling) + queued_request_count = 2 -@pytest.mark.asyncio -@pytest.mark.parametrize("existing_kind", ["prompt_cache", "session_header", "turn_state_header"]) -async def test_verified_replay_turn_alias_rebind_cannot_be_stolen_by_old_session( - existing_kind: str, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - turn_state = "http_turn_recovered" - old_session = _make_bridge_session(key=proxy_service._HTTPBridgeSessionKey(existing_kind, "old-owner", None)) - old_session.downstream_turn_state = turn_state - old_session.downstream_turn_state_aliases.add(turn_state) - recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("new-owner")) - recovery.durable_session_id = "durable-new-owner" - recovery.durable_owner_epoch = 2 - recovery.headers = { - "session_id": "retired", - "session-id": "retired", - "thread-id": "retired", - "x-codex-conversation-id": "retired", - "x-codex-session-id": "retired", - "x-codex-turn-state": turn_state, - } - service._http_bridge_sessions[old_session.key] = old_session - service._http_bridge_sessions[recovery.key] = recovery - service._durable_bridge = SimpleNamespace( - register_turn_state=AsyncMock(return_value=DurableBridgeAliasRegistration.REGISTERED) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "hard-anchor", None), + pending_requests=pending_requests, + queued_request_count=queued_request_count, ) - alias_key = proxy_service._http_bridge_turn_state_alias_key(turn_state, None) - service._http_bridge_turn_state_index[alias_key] = old_session.key + session.last_upstream_close_code = 1011 + session.upstream = cast(UpstreamWebSocket, SimpleNamespace(send_text=AsyncMock(), close=AsyncMock())) + reconnect = AsyncMock() + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) - await service._register_http_bridge_turn_state(recovery, turn_state) + assert await service._retry_http_bridge_precreated_request(session) is False + reconnect.assert_not_awaited() - assert service._http_bridge_turn_state_index[alias_key] == recovery.key - assert turn_state not in old_session.downstream_turn_state_aliases - assert old_session.downstream_turn_state is None - assert recovery.headers == {} - # A stale local alias set must not reclaim a different live owner's lane. - old_session.downstream_turn_state_aliases.add(turn_state) - http_bridge_helpers_module._register_http_bridge_turn_state_aliases_locked(service, old_session) - assert service._http_bridge_turn_state_index[alias_key] == recovery.key +class _SilentEventlessUpstream: + """Upstream double that never produces a response event, for eventless-timeout tests.""" + def __init__(self) -> None: + self.first_receive_started = asyncio.Event() + self.receive_cancelled = asyncio.Event() + self.closed = False -@pytest.mark.asyncio -async def test_verified_replay_turn_alias_does_not_replace_unrelated_internal_lane() -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - turn_state = "http_turn_conflict" - existing = _make_bridge_session( - key=proxy_service._HTTPBridgeSessionKey("internal_request_parallel", "other-lane", None) - ) - recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("recovery-lane")) - service._http_bridge_sessions[existing.key] = existing - service._http_bridge_sessions[recovery.key] = recovery - alias_key = proxy_service._http_bridge_turn_state_alias_key(turn_state, None) - service._http_bridge_turn_state_index[alias_key] = existing.key + async def receive(self) -> UpstreamWebSocketMessage: + self.first_receive_started.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + self.receive_cancelled.set() + raise + raise AssertionError("unreachable") - await service._register_http_bridge_turn_state(recovery, turn_state) + async def send_text(self, text: str) -> None: + pass - assert service._http_bridge_turn_state_index[alias_key] == existing.key - assert turn_state not in recovery.downstream_turn_state_aliases + async def close(self) -> None: + self.closed = True -@pytest.mark.asyncio -@pytest.mark.parametrize("alias_kind", ["turn_state", "previous_response"]) -async def test_verified_replay_alias_requires_durable_identity(alias_kind: str) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - alias = "http_turn_missing_durable_identity" if alias_kind == "turn_state" else "resp_missing_durable_identity" - predecessor = _make_bridge_session( - key=proxy_service._HTTPBridgeSessionKey("session_header", "missing-durable-predecessor", None) +def test_http_bridge_eventless_precreated_deadline_uses_current_send_and_client_safe_cap() -> None: + request_state = _make_eventless_http_bridge_owner() + + assert ( + http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( + request_state, + stuck_gate_retire_after_seconds=300.0, + ) + == 160.0 + ) + assert ( + http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( + request_state, + stuck_gate_retire_after_seconds=30.0, + ) + == 130.0 ) - recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("missing-durable-recovery")) - service._http_bridge_sessions[predecessor.key] = predecessor - service._http_bridge_sessions[recovery.key] = recovery - if alias_kind == "turn_state": - predecessor.downstream_turn_state = alias - predecessor.downstream_turn_state_aliases.add(alias) - alias_key = proxy_service._http_bridge_turn_state_alias_key(alias, None) - service._http_bridge_turn_state_index[alias_key] = predecessor.key - registered = await service._register_http_bridge_turn_state(recovery, alias) - assert alias not in recovery.downstream_turn_state_aliases - assert alias in predecessor.downstream_turn_state_aliases - assert service._http_bridge_turn_state_index[alias_key] == predecessor.key - else: - predecessor.previous_response_ids.add(alias) - alias_key = proxy_service._http_bridge_previous_response_alias_key(alias, None) - service._http_bridge_previous_response_index[alias_key] = predecessor.key - registered = await service._register_http_bridge_previous_response_id(recovery, alias) - assert alias not in recovery.previous_response_ids - assert alias in predecessor.previous_response_ids - assert service._http_bridge_previous_response_index[alias_key] == predecessor.key - assert registered is False + request_state.latency_first_upstream_event_ms = 25 + request_state.last_upstream_activity_at = 150.0 + assert ( + http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( + request_state, + stuck_gate_retire_after_seconds=300.0, + ) + == 160.0 + ) -@pytest.mark.asyncio -@pytest.mark.parametrize("existing_kind", ["prompt_cache", "session_header", "turn_state_header"]) -async def test_verified_replay_response_alias_rebind_cannot_be_stolen_by_old_session( - existing_kind: str, +@pytest.mark.parametrize( + ("field_name", "field_value"), + [ + ("response_id", "resp-created"), + ("latency_response_created_ms", 12), + ("downstream_visible", True), + ("last_downstream_sequence_number", 0), + ("awaiting_response_created", False), + ("response_create_gate_acquired", False), + ("response_create_gate", None), + ("response_create_sent_at", None), + ], +) +def test_http_bridge_eventless_precreated_deadline_requires_narrow_owner_evidence( + field_name: str, + field_value: object, ) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - response_id = "resp_recovered" - old_session = _make_bridge_session(key=proxy_service._HTTPBridgeSessionKey(existing_kind, "old-owner", None)) - old_session.previous_response_ids.add(response_id) - recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("new-owner")) - recovery.durable_session_id = "durable-new-owner" - recovery.durable_owner_epoch = 2 - service._http_bridge_sessions[old_session.key] = old_session - service._http_bridge_sessions[recovery.key] = recovery - service._durable_bridge = SimpleNamespace( - register_previous_response_id=AsyncMock(return_value=DurableBridgeAliasRegistration.REGISTERED) - ) - alias_key = proxy_service._http_bridge_previous_response_alias_key(response_id, None) - service._http_bridge_previous_response_index[alias_key] = old_session.key + request_state = _make_eventless_http_bridge_owner() + setattr(request_state, field_name, field_value) - await service._register_http_bridge_previous_response_id(recovery, response_id) + assert ( + http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( + request_state, + stuck_gate_retire_after_seconds=300.0, + ) + is None + ) - assert service._http_bridge_previous_response_index[alias_key] == recovery.key - assert response_id not in old_session.previous_response_ids - assert response_id in recovery.previous_response_ids - await service._register_http_bridge_previous_response_id(old_session, response_id) +def test_http_bridge_eventless_precreated_deadline_survives_reasoning_prelude_without_created() -> None: + request_state = _make_eventless_http_bridge_owner() + request_state.last_upstream_activity_at = 150.0 + request_state.upstream_model_output_seen = True + request_state.deferred_reasoning_downstream_texts.append( + 'data: {"type":"response.output_item.added","item":{"type":"reasoning"}}\n\n' + ) - assert service._http_bridge_previous_response_index[alias_key] == recovery.key - assert response_id not in old_session.previous_response_ids + assert ( + http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( + request_state, + stuck_gate_retire_after_seconds=300.0, + ) + == 150.0 + http_bridge_helpers_module._HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS + ) @pytest.mark.asyncio -@pytest.mark.parametrize("alias_kind", ["turn_state", "previous_response"]) -@pytest.mark.parametrize("durable_outcome", ["registered", "protected", "exception"]) -async def test_durable_verified_replay_alias_is_published_only_after_fenced_write( - alias_kind: str, - durable_outcome: str, -) -> None: +async def test_process_http_bridge_upstream_text_anchors_deferred_reasoning_prelude_without_created() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - alias = "http_turn_atomic_rebind" if alias_kind == "turn_state" else "resp_atomic_rebind" - predecessor = _make_bridge_session( - key=proxy_service._HTTPBridgeSessionKey("session_header", "atomic-predecessor", None) - ) - recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("atomic-recovery")) - recovery.durable_session_id = "durable-atomic-recovery" - recovery.durable_owner_epoch = 2 - service._http_bridge_sessions[predecessor.key] = predecessor - service._http_bridge_sessions[recovery.key] = recovery - if alias_kind == "turn_state": - predecessor.downstream_turn_state = alias - predecessor.downstream_turn_state_aliases.add(alias) - alias_key = proxy_service._http_bridge_turn_state_alias_key(alias, None) - service._http_bridge_turn_state_index[alias_key] = predecessor.key - else: - predecessor.previous_response_ids.add(alias) - alias_key = proxy_service._http_bridge_previous_response_alias_key(alias, None) - service._http_bridge_previous_response_index[alias_key] = predecessor.key - - write_started = asyncio.Event() - release_write = asyncio.Event() - - async def persist_alias(**_kwargs: Any) -> DurableBridgeAliasRegistration: - write_started.set() - await release_write.wait() - if durable_outcome == "exception": - raise RuntimeError("durable alias write failed") - if durable_outcome == "protected": - return DurableBridgeAliasRegistration.ALIAS_PROTECTED - return DurableBridgeAliasRegistration.REGISTERED - + request_state = _make_eventless_http_bridge_owner(sent_at=time.monotonic() - 2.0) + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + request_state.response_create_attempt = attempt + session = _make_bridge_session(pending_requests=deque([request_state]), queued_request_count=1) + lookup_retry_circuit = AsyncMock(return_value=None) + persist_retry_circuit = AsyncMock(return_value=None) service._durable_bridge = SimpleNamespace( - register_turn_state=persist_alias, - register_previous_response_id=persist_alias, + lookup_retry_circuit=lookup_retry_circuit, + persist_retry_circuit=persist_retry_circuit, ) - if alias_kind == "turn_state": - registration = asyncio.create_task(service._register_http_bridge_turn_state(recovery, alias)) - else: - registration = asyncio.create_task(service._register_http_bridge_previous_response_id(recovery, alias)) - try: - await asyncio.wait_for(write_started.wait(), 1.0) + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "response.output_item.added", + "item": {"type": "reasoning", "id": "rs_1"}, + }, + separators=(",", ":"), + ), + ) - if alias_kind == "turn_state": - assert service._http_bridge_turn_state_index[alias_key] == predecessor.key - assert alias in predecessor.downstream_turn_state_aliases - assert alias not in recovery.downstream_turn_state_aliases - else: - assert service._http_bridge_previous_response_index[alias_key] == predecessor.key - assert alias in predecessor.previous_response_ids - assert alias not in recovery.previous_response_ids + assert request_state.response_event_count == 0 + assert request_state.response_id is None + assert request_state.downstream_visible is False + assert request_state.upstream_model_output_seen is True + assert attempt.response_observed is True + assert request_state.last_upstream_activity_at is not None + sent_at = request_state.response_create_sent_at + assert sent_at is not None + assert request_state.last_upstream_activity_at > sent_at + assert len(request_state.deferred_reasoning_downstream_texts) == 1 + assert ( + http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( + request_state, + stuck_gate_retire_after_seconds=300.0, + ) + == request_state.last_upstream_activity_at + + http_bridge_helpers_module._HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS + ) + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (request_state,) + ) + assert selection.kind == "settled" + assert selection.attempt is attempt + assert ( + await service._record_http_bridge_retry_circuit_failure_for_attempt_selection( + session, + detail="stream_idle_timeout", + selection=selection, + ) + is None + ) + assert session.key not in cast(Any, service)._http_bridge_retry_circuits + lookup_retry_circuit.assert_not_awaited() + persist_retry_circuit.assert_not_awaited() - release_write.set() - await asyncio.wait_for(registration, 1.0) - expected_owner = recovery if durable_outcome == "registered" else predecessor - if alias_kind == "turn_state": - assert service._http_bridge_turn_state_index[alias_key] == expected_owner.key - assert (alias in recovery.downstream_turn_state_aliases) is (durable_outcome == "registered") - assert (alias in predecessor.downstream_turn_state_aliases) is (durable_outcome != "registered") - else: - assert service._http_bridge_previous_response_index[alias_key] == expected_owner.key - assert (alias in recovery.previous_response_ids) is (durable_outcome == "registered") - assert (alias in predecessor.previous_response_ids) is (durable_outcome != "registered") - finally: - release_write.set() - if not registration.done(): - registration.cancel() - await asyncio.gather(registration, return_exceptions=True) +@pytest.mark.asyncio +async def test_http_bridge_send_replaces_timestamp_and_wakes_existing_reader( + monkeypatch: pytest.MonkeyPatch, +) -> None: + request_state = _make_eventless_http_bridge_owner(sent_at=1.0) + session = _make_bridge_session() + seen_sent_ats: list[float | None] = [] + async def send_text(_text: str) -> None: + seen_sent_ats.append(request_state.response_create_sent_at) -@pytest.mark.asyncio -@pytest.mark.parametrize("alias_kind", ["turn_state", "previous_response"]) -async def test_durable_verified_replay_alias_writes_do_not_serialize_unrelated_sessions(alias_kind: str) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - alias = "http_turn_serial_rebind" if alias_kind == "turn_state" else "resp_serial_rebind" - predecessor = _make_bridge_session( - key=proxy_service._HTTPBridgeSessionKey("session_header", "serial-predecessor", None) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=AsyncMock()), ) - recoveries = [ - _make_bridge_session(key=_make_account_neutral_replay_session_key(f"serial-recovery-{index}")) - for index in range(2) - ] - for index, recovery in enumerate(recoveries): - recovery.durable_session_id = f"durable-serial-recovery-{index}" - recovery.durable_owner_epoch = 2 - service._http_bridge_sessions[recovery.key] = recovery - service._http_bridge_sessions[predecessor.key] = predecessor - if alias_kind == "turn_state": - predecessor.downstream_turn_state = alias - predecessor.downstream_turn_state_aliases.add(alias) - alias_key = proxy_service._http_bridge_turn_state_alias_key(alias, None) - service._http_bridge_turn_state_index[alias_key] = predecessor.key - else: - predecessor.previous_response_ids.add(alias) - alias_key = proxy_service._http_bridge_previous_response_alias_key(alias, None) - service._http_bridge_previous_response_index[alias_key] = predecessor.key - - write_started = [asyncio.Event(), asyncio.Event()] - release_write = [asyncio.Event(), asyncio.Event()] - durable_write_order: list[str] = [] - - async def persist_alias(**kwargs: Any) -> DurableBridgeAliasRegistration: - write_index = len(durable_write_order) - durable_write_order.append(kwargs["session_id"]) - write_started[write_index].set() - await release_write[write_index].wait() - return ( - DurableBridgeAliasRegistration.REGISTERED - if write_index == 0 - else DurableBridgeAliasRegistration.ALIAS_PROTECTED - ) - - service._durable_bridge = SimpleNamespace( - register_turn_state=persist_alias, - register_previous_response_id=persist_alias, + monotonic_values = iter((100.0, 200.0)) + monotonic = lambda: next(monotonic_values) # noqa: E731 + monkeypatch.setattr( + http_bridge_request_submit_module, + "_service_time", + lambda: SimpleNamespace(monotonic=monotonic), ) - async def register(recovery: proxy_service._HTTPBridgeSession) -> bool: - if alias_kind == "turn_state": - return await service._register_http_bridge_turn_state(recovery, alias) - return await service._register_http_bridge_previous_response_id(recovery, alias) - - registrations = [asyncio.create_task(register(recoveries[0]))] - try: - await asyncio.wait_for(write_started[0].wait(), 1.0) - registrations.append(asyncio.create_task(register(recoveries[1]))) - await asyncio.wait_for(write_started[1].wait(), 1.0) - - release_write[0].set() - first_registered = await asyncio.wait_for(asyncio.shield(registrations[0]), 1.0) - if alias_kind == "turn_state": - assert service._http_bridge_turn_state_index[alias_key] == recoveries[0].key - else: - assert service._http_bridge_previous_response_index[alias_key] == recoveries[0].key - - release_write[1].set() - second_registered = await asyncio.wait_for(registrations[1], 1.0) + await http_bridge_request_submit_module._send_http_bridge_request_text_with_archive_id( + session, + request_state, + "first", + ) + first_attempt = request_state.response_create_attempt + session.upstream_reader_wakeup.clear() + await http_bridge_request_submit_module._send_http_bridge_request_text_with_archive_id( + session, + request_state, + "second", + ) - assert durable_write_order == [recovery.durable_session_id for recovery in recoveries] - assert [first_registered, second_registered] == [True, False] - if alias_kind == "turn_state": - assert service._http_bridge_turn_state_index[alias_key] == recoveries[0].key - assert alias in recoveries[0].downstream_turn_state_aliases - assert alias not in recoveries[1].downstream_turn_state_aliases - else: - assert service._http_bridge_previous_response_index[alias_key] == recoveries[0].key - assert alias in recoveries[0].previous_response_ids - assert alias not in recoveries[1].previous_response_ids - finally: - for release in release_write: - release.set() - for registration in registrations: - if not registration.done(): - registration.cancel() - await asyncio.gather(*registrations, return_exceptions=True) + assert seen_sent_ats == [100.0, 200.0] + assert request_state.response_create_sent_at == 200.0 + assert first_attempt is not None + assert first_attempt.ordinal == 1 + assert request_state.response_create_attempt is not first_attempt + assert request_state.response_create_attempt is not None + assert request_state.response_create_attempt.ordinal == 2 + assert session.upstream_reader_wakeup.is_set() is True @pytest.mark.asyncio -async def test_durable_verified_replay_alias_writes_serialize_within_one_session() -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("serial-one-session")) - recovery.durable_session_id = "durable-serial-one-session" - recovery.durable_owner_epoch = 2 - service._http_bridge_sessions[recovery.key] = recovery - writes_started = [asyncio.Event(), asyncio.Event()] - release_writes = [asyncio.Event(), asyncio.Event()] - write_count = 0 +@pytest.mark.parametrize("failure", [RuntimeError("send failed"), asyncio.CancelledError()]) +async def test_http_bridge_failed_send_disarms_eventless_deadline( + monkeypatch: pytest.MonkeyPatch, + failure: BaseException, +) -> None: + request_state = _make_eventless_http_bridge_owner(sent_at=1.0) + session = _make_bridge_session() - async def persist_alias(**_kwargs: Any) -> DurableBridgeAliasRegistration: - nonlocal write_count - write_index = write_count - write_count += 1 - writes_started[write_index].set() - await release_writes[write_index].wait() - return DurableBridgeAliasRegistration.REGISTERED + async def send_text(_text: str) -> None: + assert request_state.response_create_sent_at == 100.0 + raise failure - service._durable_bridge = SimpleNamespace( - register_turn_state=persist_alias, - register_previous_response_id=persist_alias, + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=AsyncMock()), ) - turn_registration = asyncio.create_task( - service._register_http_bridge_turn_state(recovery, "http_turn_serial_one_session") + monkeypatch.setattr( + http_bridge_request_submit_module, + "_service_time", + lambda: SimpleNamespace(monotonic=lambda: 100.0), ) - response_registration: asyncio.Task[bool] | None = None - try: - await asyncio.wait_for(writes_started[0].wait(), 1.0) - response_registration = asyncio.create_task( - service._register_http_bridge_previous_response_id(recovery, "resp_serial_one_session") + session.upstream_reader_wakeup.clear() + + with pytest.raises(type(failure)): + await http_bridge_request_submit_module._send_http_bridge_request_text_with_archive_id( + session, + request_state, + "request", ) - await asyncio.sleep(0) - assert writes_started[1].is_set() is False - release_writes[0].set() - assert await asyncio.wait_for(turn_registration, 1.0) is True - await asyncio.wait_for(writes_started[1].wait(), 1.0) - release_writes[1].set() - assert await asyncio.wait_for(response_registration, 1.0) is True - finally: - for release_write in release_writes: - release_write.set() - tasks = [turn_registration] - if response_registration is not None: - tasks.append(response_registration) - for task in tasks: - if not task.done(): - task.cancel() - await asyncio.gather(*tasks, return_exceptions=True) + assert request_state.response_create_sent_at is None + assert request_state.response_create_attempt is not None + assert request_state.response_create_attempt.disarmed is True + assert session.upstream_reader_wakeup.is_set() is True + assert ( + http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( + request_state, + stuck_gate_retire_after_seconds=300.0, + ) + is None + ) @pytest.mark.asyncio -@pytest.mark.parametrize("alias_kind", ["turn_state", "previous_response"]) -async def test_durable_active_recovery_alias_protection_prevents_superseding_replica_publication( - alias_kind: str, +async def test_http_bridge_send_started_callback_runs_after_exact_frame_preflight( + monkeypatch: pytest.MonkeyPatch, ) -> None: - services = [proxy_service.ProxyService(cast(Any, nullcontext())) for _ in range(2)] - recoveries = [ - _make_bridge_session(key=_make_account_neutral_replay_session_key(f"replica-recovery-{index}")) - for index in range(2) - ] - alias = "http_turn_replica_race" if alias_kind == "turn_state" else "resp_replica_race" - for index, (service, recovery) in enumerate(zip(services, recoveries, strict=True)): - recovery.account = cast( - Any, - SimpleNamespace(id=f"acc-replica-{index}", status=AccountStatus.ACTIVE, plan_type="plus"), - ) - recovery.durable_session_id = f"durable-replica-recovery-{index}" - recovery.durable_owner_epoch = 2 - service._http_bridge_sessions[recovery.key] = recovery + request_state = _make_eventless_http_bridge_owner() + session = _make_bridge_session() + send_started = Mock() + send_text = AsyncMock() + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=AsyncMock()), + ) - first_write_committed = asyncio.Event() - release_first_writer = asyncio.Event() - durable_owner: list[str] = [] + def fail_preflight(_request_state: object, _text_data: str) -> None: + raise proxy_service.ProxyResponseError(400, {"error": {"code": "payload_too_large"}}) - async def persist_first(**kwargs: Any) -> DurableBridgeAliasRegistration: - durable_owner[:] = [kwargs["session_id"]] - first_write_committed.set() - await release_first_writer.wait() - return DurableBridgeAliasRegistration.REGISTERED + monkeypatch.setattr( + http_bridge_request_submit_module, + "_enforce_http_bridge_response_create_text_size", + fail_preflight, + ) - async def persist_second(**kwargs: Any) -> DurableBridgeAliasRegistration: - del kwargs - assert durable_owner == [recoveries[0].durable_session_id] - return DurableBridgeAliasRegistration.ALIAS_PROTECTED + with pytest.raises(proxy_service.ProxyResponseError): + await http_bridge_request_submit_module._send_http_bridge_request_text_with_archive_id( + session, + request_state, + "request", + on_send_started=send_started, + ) - services[0]._durable_bridge = SimpleNamespace( - register_turn_state=persist_first, - register_previous_response_id=persist_first, + send_started.assert_not_called() + send_text.assert_not_awaited() + + monkeypatch.setattr( + http_bridge_request_submit_module, + "_enforce_http_bridge_response_create_text_size", + lambda _request_state, _text_data: None, ) - services[1]._durable_bridge = SimpleNamespace( - register_turn_state=persist_second, - register_previous_response_id=persist_second, + await http_bridge_request_submit_module._send_http_bridge_request_text_with_archive_id( + session, + request_state, + "request", + on_send_started=send_started, ) - async def register(service: proxy_service.ProxyService, recovery: proxy_service._HTTPBridgeSession) -> bool: - if alias_kind == "turn_state": - return await service._register_http_bridge_turn_state(recovery, alias) - return await service._register_http_bridge_previous_response_id(recovery, alias) + send_started.assert_called_once() + send_text.assert_awaited_once() - first_registration = asyncio.create_task(register(services[0], recoveries[0])) - try: - await asyncio.wait_for(first_write_committed.wait(), 1.0) - second_registered = await asyncio.wait_for(register(services[1], recoveries[1]), 1.0) - release_first_writer.set() - first_registered = await asyncio.wait_for(first_registration, 1.0) - assert first_registered is True - assert second_registered is False - assert durable_owner == [recoveries[0].durable_session_id] - if alias_kind == "turn_state": - alias_key = proxy_service._http_bridge_turn_state_alias_key(alias, None) - assert services[0]._http_bridge_turn_state_index[alias_key] == recoveries[0].key - assert alias in recoveries[0].downstream_turn_state_aliases - assert alias_key not in services[1]._http_bridge_turn_state_index - assert alias not in recoveries[1].downstream_turn_state_aliases - else: - alias_key = proxy_service._http_bridge_previous_response_alias_key(alias, None) - assert services[0]._http_bridge_previous_response_index[alias_key] == recoveries[0].key - assert alias in recoveries[0].previous_response_ids - assert alias_key not in services[1]._http_bridge_previous_response_index - assert alias not in recoveries[1].previous_response_ids - finally: - release_first_writer.set() - if not first_registration.done(): - first_registration.cancel() - await asyncio.gather(first_registration, return_exceptions=True) +def _make_account_neutral_replay_session_key( + nonce: str, + api_key_id: str | None = None, +) -> proxy_service._HTTPBridgeSessionKey: + kind, key = make_http_bridge_account_neutral_replay_key(nonce) + return proxy_service._HTTPBridgeSessionKey(kind, key, api_key_id) -@pytest.mark.asyncio -async def test_durable_protected_alias_rejection_preserves_sibling_session() -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="shared-owner") - session.durable_session_id = "durable-shared-owner" - session.durable_owner_epoch = 3 - session.downstream_turn_state = "http_turn_recovered_elsewhere" - session.downstream_turn_state_aliases.update({"http_turn_recovered_elsewhere", "http_turn_sibling"}) - service._http_bridge_sessions[session.key] = session - recovered_alias_key = proxy_service._http_bridge_turn_state_alias_key("http_turn_recovered_elsewhere", None) - sibling_alias_key = proxy_service._http_bridge_turn_state_alias_key("http_turn_sibling", None) - service._http_bridge_turn_state_index[recovered_alias_key] = session.key - service._http_bridge_turn_state_index[sibling_alias_key] = session.key - service._durable_bridge = SimpleNamespace( - register_turn_state=AsyncMock(return_value=DurableBridgeAliasRegistration.ALIAS_PROTECTED) +def test_forwarded_fork_keeps_authenticated_original_unanchored_state() -> None: + assert http_bridge_helpers_module._http_bridge_request_needs_unanchored_handoff( + proxy_service._HTTPBridgeSessionKey( + "internal_unanchored_parallel", + "fork-key", + None, + ), + "http_turn_generated", + None, + True, + True, ) - await service._register_http_bridge_turn_state(session, "http_turn_recovered_elsewhere") - assert session.closed is False - assert service._http_bridge_sessions[session.key] is session - assert "http_turn_recovered_elsewhere" not in session.downstream_turn_state_aliases - assert recovered_alias_key not in service._http_bridge_turn_state_index - assert "http_turn_sibling" in session.downstream_turn_state_aliases - assert service._http_bridge_turn_state_index[sibling_alias_key] == session.key +def test_verified_replay_model_fork_preserves_recovery_kind() -> None: + replay_key = _make_account_neutral_replay_session_key("replay-parent") + + fork_key = http_bridge_helpers_module._http_bridge_incompatible_model_fork_key( + key=replay_key, + existing_model="gpt-5.6-sol", + request_model="gpt-5.6-terra", + request_scope_id="request-model-transition", + ) + + assert fork_key is not None + assert is_http_bridge_account_neutral_replay( + kind=fork_key.affinity_kind, + key=fork_key.affinity_key, + ) + assert fork_key != replay_key @pytest.mark.asyncio -async def test_durable_response_fence_rejection_rolls_back_local_alias( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - session.durable_session_id = "durable-session" - session.durable_owner_epoch = 3 - service._http_bridge_sessions[session.key] = session - service._durable_bridge = SimpleNamespace( - register_previous_response_id=AsyncMock(return_value=DurableBridgeAliasRegistration.OWNER_FENCED) +async def test_legacy_forward_anchor_lookup_accepts_registered_turn_state_alias() -> None: + key = proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None) + lookup = proxy_service.DurableBridgeLookup( + session_id="durable-1", + canonical_kind="session_header", + canonical_key="sid-123", + api_key_scope="__anonymous__", + account_id="acc-1", + owner_instance_id="instance-b", + owner_epoch=2, + lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_client", + latest_response_id="resp-1", ) - monkeypatch.setattr(http_bridge_helpers_module, "get_settings", _make_app_settings) + durable_bridge = SimpleNamespace(lookup_turn_state_target=AsyncMock(return_value=lookup)) - await service._register_http_bridge_previous_response_id(session, "resp-rejected") + resolved = await http_bridge_streaming_module._legacy_forward_anchor_lookup( + durable_bridge=durable_bridge, + bridge_session_key=key, + turn_state="http_turn_client", + api_key=None, + previous_response_id=None, + forwarded_request=True, + forwarded_legacy_signature=True, + ) - alias_key = proxy_service._http_bridge_previous_response_alias_key("resp-rejected", session.key.api_key_id) - assert "resp-rejected" not in session.previous_response_ids - assert alias_key not in service._http_bridge_previous_response_index + assert resolved is lookup + durable_bridge.lookup_turn_state_target.assert_awaited_once_with( + turn_state="http_turn_client", + api_key_id=None, + ) @pytest.mark.asyncio -async def test_durable_alias_fence_rejection_rolls_back_after_same_session_epoch_refresh( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - session.durable_session_id = "durable-session" - session.durable_owner_epoch = 3 - service._http_bridge_sessions[session.key] = session +async def test_legacy_forward_anchor_lookup_rejects_unknown_generated_turn_state() -> None: + durable_bridge = SimpleNamespace(lookup_turn_state_target=AsyncMock(return_value=None)) - async def reject_turn_state(**_kwargs: Any) -> DurableBridgeAliasRegistration: - session.durable_owner_epoch = 4 - return DurableBridgeAliasRegistration.OWNER_FENCED + with pytest.raises(ProxyResponseError) as exc_info: + await http_bridge_streaming_module._legacy_forward_anchor_lookup( + durable_bridge=durable_bridge, + bridge_session_key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), + turn_state="http_turn_generated", + api_key=None, + previous_response_id=None, + forwarded_request=True, + forwarded_legacy_signature=True, + ) - async def reject_previous_response(**_kwargs: Any) -> DurableBridgeAliasRegistration: - session.durable_owner_epoch = 5 - return DurableBridgeAliasRegistration.OWNER_FENCED + assert exc_info.value.status_code == 409 + assert exc_info.value.payload["error"]["code"] == "bridge_forward_upgrade_required" - service._durable_bridge = SimpleNamespace( - register_turn_state=reject_turn_state, - register_previous_response_id=reject_previous_response, - ) - monkeypatch.setattr(http_bridge_helpers_module, "get_settings", _make_app_settings) - await service._register_http_bridge_turn_state(session, "turn-rejected-after-refresh") - await service._register_http_bridge_previous_response_id(session, "resp-rejected-after-refresh") +@pytest.mark.asyncio +async def test_current_origin_legacy_owner_lookup_rejects_unknown_turn_state_alias() -> None: + durable_bridge = SimpleNamespace(lookup_turn_state_target=AsyncMock(return_value=None)) - turn_alias_key = proxy_service._http_bridge_turn_state_alias_key( - "turn-rejected-after-refresh", session.key.api_key_id - ) - response_alias_key = proxy_service._http_bridge_previous_response_alias_key( - "resp-rejected-after-refresh", session.key.api_key_id + with pytest.raises(ProxyResponseError) as exc_info: + await http_bridge_streaming_module._current_origin_legacy_owner_anchor_lookup( + durable_bridge=durable_bridge, + bridge_session_key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), + turn_state="http_turn_unknown", + api_key=None, + previous_response_id=None, + forwarded_request=False, + ) + + assert exc_info.value.status_code == 409 + assert exc_info.value.payload["error"]["code"] == "bridge_forward_upgrade_required" + durable_bridge.lookup_turn_state_target.assert_awaited_once_with( + turn_state="http_turn_unknown", + api_key_id=None, ) - assert "turn-rejected-after-refresh" not in session.downstream_turn_state_aliases - assert session.downstream_turn_state is None - assert turn_alias_key not in service._http_bridge_turn_state_index - assert "resp-rejected-after-refresh" not in session.previous_response_ids - assert response_alias_key not in service._http_bridge_previous_response_index @pytest.mark.asyncio -async def test_stale_turn_state_rejection_preserves_newer_same_session_registration( +async def test_submit_http_bridge_request_cancellation_releases_published_handoff( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) session = _make_bridge_session() - session.durable_session_id = "durable-session" - session.durable_owner_epoch = 3 - service._http_bridge_sessions[session.key] = session - first_started = asyncio.Event() - release_first = asyncio.Event() - - async def register_turn_state(*, owner_epoch: int, **_kwargs: Any) -> DurableBridgeAliasRegistration: - if owner_epoch == 3: - first_started.set() - await release_first.wait() - return DurableBridgeAliasRegistration.OWNER_FENCED - assert owner_epoch == 4 - return DurableBridgeAliasRegistration.REGISTERED - - service._durable_bridge = SimpleNamespace(register_turn_state=register_turn_state) - monkeypatch.setattr(http_bridge_helpers_module, "get_settings", _make_app_settings) - - stale_registration = asyncio.create_task(service._register_http_bridge_turn_state(session, "turn-race")) + session.unanchored_reservation_id = "scope-cancelled-submit" + request_state = proxy_service._WebSocketRequestState( + request_id="req-cancelled-submit", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', + transport="http", + skip_request_log=True, + ) + monkeypatch.setattr(service, "_maybe_prewarm_http_bridge_session", AsyncMock()) + await session.pending_lock.acquire() + request_scope_token = set_request_scope_id("scope-cancelled-submit") try: - await asyncio.wait_for(first_started.wait(), 1.0) - session.durable_owner_epoch = 4 - await service._register_http_bridge_turn_state(session, "turn-race") - release_first.set() - await asyncio.wait_for(stale_registration, 1.0) - - alias_key = proxy_service._http_bridge_turn_state_alias_key("turn-race", session.key.api_key_id) - assert "turn-race" in session.downstream_turn_state_aliases - assert session.downstream_turn_state == "turn-race" - assert service._http_bridge_turn_state_index[alias_key] == session.key + submit_task = asyncio.create_task( + service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + ) + ) + await asyncio.sleep(0) + submit_task.cancel() + with pytest.raises(asyncio.CancelledError): + await submit_task finally: - release_first.set() - if not stale_registration.done(): - stale_registration.cancel() - await asyncio.gather(stale_registration, return_exceptions=True) + session.pending_lock.release() + reset_request_scope_id(request_scope_token) + + assert session.unanchored_reservation_id is None @pytest.mark.asyncio -async def test_stale_response_rejection_preserves_newer_same_session_registration( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_submit_http_bridge_request_early_failure_releases_published_handoff() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) session = _make_bridge_session() - session.durable_session_id = "durable-session" - session.durable_owner_epoch = 3 - service._http_bridge_sessions[session.key] = session - first_started = asyncio.Event() - release_first = asyncio.Event() - - async def register_previous_response_id(*, owner_epoch: int, **_kwargs: Any) -> DurableBridgeAliasRegistration: - if owner_epoch == 3: - first_started.set() - await release_first.wait() - return DurableBridgeAliasRegistration.OWNER_FENCED - assert owner_epoch == 4 - return DurableBridgeAliasRegistration.REGISTERED - - service._durable_bridge = SimpleNamespace(register_previous_response_id=register_previous_response_id) - monkeypatch.setattr(http_bridge_helpers_module, "get_settings", _make_app_settings) - - stale_registration = asyncio.create_task(service._register_http_bridge_previous_response_id(session, "resp-race")) + session.closed = True + session.unanchored_reservation_id = "scope-closed-submit" + request_state = proxy_service._WebSocketRequestState( + request_id="req-closed-submit", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', + transport="http", + skip_request_log=True, + ) + request_scope_token = set_request_scope_id("scope-closed-submit") try: - await asyncio.wait_for(first_started.wait(), 1.0) - session.durable_owner_epoch = 4 - await service._register_http_bridge_previous_response_id(session, "resp-race") - release_first.set() - await asyncio.wait_for(stale_registration, 1.0) - - alias_key = proxy_service._http_bridge_previous_response_alias_key("resp-race", session.key.api_key_id) - assert "resp-race" in session.previous_response_ids - assert service._http_bridge_previous_response_index[alias_key] == session.key + with pytest.raises(proxy_service.ProxyResponseError): + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + ) finally: - release_first.set() - if not stale_registration.done(): - stale_registration.cancel() - await asyncio.gather(stale_registration, return_exceptions=True) - - -@pytest.mark.asyncio -async def test_durable_alias_fence_rejection_preserves_new_local_generation( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - stale_session = _make_bridge_session() - stale_session.durable_session_id = "durable-stale" - stale_session.durable_owner_epoch = 3 - current_session = _make_bridge_session() - current_session.downstream_turn_state_aliases.add("turn-current") - current_session.previous_response_ids.add("resp-current") - service._http_bridge_sessions[current_session.key] = current_session - turn_alias_key = proxy_service._http_bridge_turn_state_alias_key("turn-current", current_session.key.api_key_id) - response_alias_key = proxy_service._http_bridge_previous_response_alias_key( - "resp-current", current_session.key.api_key_id - ) - service._http_bridge_turn_state_index[turn_alias_key] = current_session.key - service._http_bridge_previous_response_index[response_alias_key] = current_session.key - service._durable_bridge = SimpleNamespace( - register_turn_state=AsyncMock(return_value=DurableBridgeAliasRegistration.OWNER_FENCED), - register_previous_response_id=AsyncMock(return_value=DurableBridgeAliasRegistration.OWNER_FENCED), - ) - monkeypatch.setattr(http_bridge_helpers_module, "get_settings", _make_app_settings) - - await service._register_http_bridge_turn_state(stale_session, "turn-current") - await service._register_http_bridge_previous_response_id(stale_session, "resp-current") - - assert "turn-current" not in stale_session.downstream_turn_state_aliases - assert "resp-current" not in stale_session.previous_response_ids - assert service._http_bridge_turn_state_index[turn_alias_key] == current_session.key - assert service._http_bridge_previous_response_index[response_alias_key] == current_session.key - + reset_request_scope_id(request_scope_token) -def test_codex_prewarm_eligibility_is_enabled_flag_alone() -> None: - assert proxy_service._http_bridge_prewarm_enabled( - _make_app_settings(http_responses_session_bridge_codex_prewarm_enabled=True) - ) - assert not proxy_service._http_bridge_prewarm_enabled(_make_app_settings()) + assert session.unanchored_reservation_id is None @pytest.mark.asyncio -async def test_maybe_prewarm_http_bridge_session_not_applicable_when_disabled( +async def test_submit_http_bridge_request_prewarm_failure_retires_owned_handoff( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - state = proxy_service._WebSocketRequestState( - request_id="req-prewarm-disabled", - model="gpt-5.2", + session = _make_bridge_session() + session.unanchored_reservation_id = "scope-prewarm-failure" + session.upstream_control.retire_after_drain = True + request_state = proxy_service._WebSocketRequestState( + request_id="req-prewarm-failure", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, - request_text=json.dumps({"input": "x" * 50000}), + started_at=time.monotonic(), + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', transport="http", + skip_request_log=True, ) - session = _make_bridge_session() - session.codex_session = True - session.last_used_at = -180.0 + retire = AsyncMock(return_value=True) + monkeypatch.setattr(service, "_ensure_http_bridge_session_stream_lease_locked", AsyncMock()) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_allowed", AsyncMock(return_value=True)) monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: _make_app_settings(), + service, + "_maybe_prewarm_http_bridge_session", + AsyncMock(side_effect=RuntimeError("prewarm failed")), ) + monkeypatch.setattr(service, "_retire_http_bridge_after_drain_if_ready", retire) - await service._maybe_prewarm_http_bridge_session( - session, - request_state=state, - text_data=state.request_text or "{}", - ) + request_scope_token = set_request_scope_id("scope-prewarm-failure") + try: + with pytest.raises(RuntimeError, match="prewarm failed"): + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + ) + finally: + reset_request_scope_id(request_scope_token) - assert state.prewarm_status == "not_applicable" - assert session.prewarmed is False + assert session.unanchored_reservation_id is None + assert session.admission_waiter_count == 0 + retire.assert_awaited_once_with(session) @pytest.mark.asyncio -async def test_http_bridge_activity_snapshot_counts_pending_and_inflight_sessions(): - service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) +async def test_submit_http_bridge_request_keeps_reserved_detached_lane_after_replacement( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("session_header", "sid-detached-submit", None) + predecessor = _make_bridge_session(key=key, key_value=key.affinity_key) + replacement = _make_bridge_session(key=key, key_value=key.affinity_key) + send_text = AsyncMock() + predecessor.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=AsyncMock()), + ) + predecessor.unanchored_reservation_id = "scope-detached-submit" + service._http_bridge_sessions[key] = predecessor + monkeypatch.setattr(service, "_http_bridge_precreated_retry_allowed", AsyncMock(return_value=True)) request_state = proxy_service._WebSocketRequestState( - request_id="req-drain-status", - model="gpt-5.5", + request_id="req-detached-submit", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, - response_id=None, + started_at=time.monotonic(), awaiting_response_created=True, - event_queue=None, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"continue"}', transport="http", skip_request_log=True, ) - session = _make_bridge_session( - pending_requests=deque([request_state]), - queued_request_count=2, - ) - service._http_bridge_sessions[session.key] = session - service._http_bridge_inflight_sessions[ - proxy_service._HTTPBridgeSessionKey("session_header", "inflight-drain-status", None) - ] = asyncio.Future() - snapshot = service.http_bridge_activity_snapshot_nowait() + async def replace_during_prewarm(*_args: object, **_kwargs: object) -> None: + predecessor.upstream_control.reconnect_requested = True + predecessor.upstream_control.retire_after_drain = True + async with service._http_bridge_lock: + assert ( + service._detach_http_bridge_session_locked( + key, + expected_session=predecessor, + mark_closed=False, + ) + is predecessor + ) + service._http_bridge_sessions[key] = replacement - assert snapshot == { - "http_bridge_live_sessions": 1, - "http_bridge_pending_or_queued_requests": 2, - "http_bridge_pending_unknown_sessions": 0, - "http_bridge_inflight_session_creates": 1, - "http_bridge_inflight_session_create_oldest_age_seconds": 0, - "http_bridge_stale_inflight_session_creates": 0, - "http_bridge_cleaned_inflight_session_creates": 0, - "http_bridge_background_cleanup_tasks": 0, - "http_bridge_active": True, - "http_bridge_restart_blocking": True, - } + monkeypatch.setattr(service, "_maybe_prewarm_http_bridge_session", replace_during_prewarm) + request_scope_token = set_request_scope_id("scope-detached-submit") + try: + await service._submit_http_bridge_request( + predecessor, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + ) + finally: + reset_request_scope_id(request_scope_token) + + send_text.assert_awaited_once_with(request_state.request_text) + assert predecessor.unanchored_reservation_id is None + assert service._http_bridge_sessions[key] is replacement + assert service._http_bridge_detached_sessions[id(predecessor)] is predecessor + + assert await service._detach_http_bridge_request(predecessor, request_state=request_state) is True @pytest.mark.asyncio -async def test_http_bridge_activity_snapshot_counts_closed_admission_waiter_as_restart_blocking() -> None: - service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) - session = _make_bridge_session(queued_request_count=1) - session.closed = True - session.admission_waiter_count = 1 - service._http_bridge_sessions[session.key] = session +async def test_release_handoffs_retires_ready_detached_generation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="detached-released-handoff") + session.unanchored_reservation_id = "scope-detached-release" + session.upstream_control.reconnect_requested = True + session.upstream_control.retire_after_drain = True + service._http_bridge_detached_sessions[id(session)] = session + retire = AsyncMock(return_value=True) + monkeypatch.setattr(service, "_retire_http_bridge_after_drain_if_ready", retire) - snapshot = service.http_bridge_activity_snapshot_nowait() + await http_bridge_helpers_module._release_http_bridge_unanchored_handoffs_for_request( + service, + request_scope_id="scope-detached-release", + ) - assert snapshot["http_bridge_live_sessions"] == 0 - assert snapshot["http_bridge_pending_or_queued_requests"] == 1 - assert snapshot["http_bridge_active"] is True - assert snapshot["http_bridge_restart_blocking"] is True + assert session.unanchored_reservation_id is None + retire.assert_awaited_once_with(session) @pytest.mark.asyncio -async def test_response_create_gate_timeout_retires_old_pending_without_upstream_event( +@pytest.mark.parametrize("submit_succeeds", [True, False]) +async def test_http_bridge_submit_transfers_settlement_ownership_only_after_success( monkeypatch: pytest.MonkeyPatch, + submit_succeeds: bool, ) -> None: - settings = _make_app_settings( - proxy_admission_wait_timeout_seconds=0.001, - http_responses_session_bridge_stuck_gate_retire_after_seconds=300.0, - ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) - service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + service = proxy_service.ProxyService(cast(Any, nullcontext())) session = _make_bridge_session() - service._http_bridge_sessions[session.key] = session - await session.response_create_gate.acquire() - old_pending = proxy_service._WebSocketRequestState( - request_id="req-old-pending", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic() - 301.0, - transport="http", - response_create_gate_acquired=True, - awaiting_response_created=True, - # A downstream keepalive is not evidence that upstream created a - # response; this request is still safe to retire after the stale window. - downstream_visible=True, + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv-bridge-submit-owner", + key_id="key-bridge-submit-owner", + model="gpt-5.6-sol", ) - waiter = proxy_service._WebSocketRequestState( - request_id="req-visible-waiter", - model="gpt-5.2", + lifecycle = proxy_support_module._DeferredAccountBackoffLifecycle(reservation=reservation) + request_state = proxy_service._WebSocketRequestState( + request_id="req-bridge-submit-owner", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, - api_key_reservation=None, + api_key_reservation=reservation, started_at=time.monotonic(), - transport="http", - downstream_visible=True, + event_queue=asyncio.Queue(), + deferred_account_backoff_lifecycle=lifecycle, ) - async with session.pending_lock: - session.pending_requests.append(old_pending) - session.queued_request_count = 1 - - retire_calls: list[str] = [] - - async def fake_retire( - retire_session: proxy_service._HTTPBridgeSession, - *, - detail: str, - ) -> None: - retire_calls.append(detail) - retire_session.closed = True - monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", fake_retire) + async def submit(*_args: object, **_kwargs: object) -> None: + if not submit_succeeds: + raise RuntimeError("send failed before submit returned") + assert request_state.event_queue is not None + request_state.event_queue.put_nowait(None) - try: - with pytest.raises(ProxyResponseError) as exc_info: - await service._acquire_request_state_response_create_admission( - waiter, - response_create_gate=session.response_create_gate, - account_id=session.account.id, - surface="http_bridge", - bridge_session=session, - ) - finally: - if session.response_create_gate.locked(): - session.response_create_gate.release() - - assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" - assert retire_calls == ["response_create_gate_timeout_stuck_pending"] - assert session.closed is True - assert waiter.response_create_gate is None - assert waiter.response_create_gate_acquired is False - - -def test_stale_gate_cleanup_keeps_draining_sibling_active() -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - now = time.monotonic() - stale = proxy_service._WebSocketRequestState( - request_id="req-stale-gate-holder", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=now - 301.0, - transport="http", - ) - draining = proxy_service._WebSocketRequestState( - request_id="req-draining-terminal-sibling", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=now - 301.0, - transport="http", - draining_until_terminal=True, + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_maybe_release_idle_http_bridge_session_lease", AsyncMock(return_value=False)) + stream = service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data="{}", + queue_limit=4, + propagate_http_errors=True, + downstream_turn_state=None, + request_deadline=time.monotonic() + 10, ) - stale_states, should_retire = service._classify_http_bridge_stale_gate_holders( - [stale, draining], - now=now, - threshold_seconds=300.0, - session_closed=False, - ) + if submit_succeeds: + assert [event async for event in stream] == [] + else: + with pytest.raises(RuntimeError, match="send failed"): + async for _ in stream: + pass - assert stale_states == [stale] - assert should_retire is False + assert lifecycle.settlement_owned is submit_succeeds @pytest.mark.asyncio -async def test_response_create_gate_timeout_retires_closed_anchored_pending_without_upstream_event( +async def test_http_bridge_request_cleanup_releases_pre_submit_handoff( monkeypatch: pytest.MonkeyPatch, ) -> None: - settings = _make_app_settings( - proxy_admission_wait_timeout_seconds=0.001, - http_responses_session_bridge_stuck_gate_retire_after_seconds=300.0, - ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) - service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) - session = _make_bridge_session() - session.closed = True + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="scope-pre-submit") + session.unanchored_reservation_id = "scope-pre-submit" service._http_bridge_sessions[session.key] = session - await session.response_create_gate.acquire() - old_pending = proxy_service._WebSocketRequestState( - request_id="req-closed-anchored-pending", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic() - 301.0, - transport="http", - previous_response_id="resp-anchored", - hard_continuity_anchor=True, - response_create_gate_acquired=True, - awaiting_response_created=True, - ) - waiter = proxy_service._WebSocketRequestState( - request_id="req-closed-anchored-waiter", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - transport="http", - downstream_visible=True, + runtime_config = SimpleNamespace( + enabled=True, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + prompt_cache_idle_ttl_seconds=120.0, ) - async with session.pending_lock: - session.pending_requests.append(old_pending) - session.queued_request_count = 1 - retire_calls: list[str] = [] - - async def fake_retire( - retire_session: proxy_service._HTTPBridgeSession, - *, - detail: str, - ) -> None: - retire_calls.append(detail) - retire_session.closed = True + async def fail_before_submit(*args: object, **kwargs: object): + del args, kwargs + raise RuntimeError("payload preparation failed") + yield "" - monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", fake_retire) + monkeypatch.setattr( + http_bridge_streaming_module, + "_service_get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=SimpleNamespace())), + ) + monkeypatch.setattr(http_bridge_streaming_module, "_service_get_settings", _make_app_settings) + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_runtime_config", lambda *args: runtime_config) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_stream_via_http_bridge", fail_before_submit) + payload = proxy_service.ResponsesRequest.model_validate( + {"model": "gpt-5.6-sol", "instructions": "test", "input": "hello"} + ) + request_scope_token = set_request_scope_id("scope-pre-submit") try: - with pytest.raises(ProxyResponseError) as exc_info: - await service._acquire_request_state_response_create_admission( - waiter, - response_create_gate=session.response_create_gate, - account_id=session.account.id, - surface="http_bridge", - bridge_session=session, - ) + with pytest.raises(RuntimeError, match="payload preparation failed"): + async for _ in service._stream_http_bridge_or_retry( + payload, + {}, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + ): + pass finally: - if session.response_create_gate.locked(): - session.response_create_gate.release() + reset_request_scope_id(request_scope_token) - assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" - assert retire_calls == ["response_create_gate_timeout_stuck_pending"] + assert session.unanchored_reservation_id is None @pytest.mark.asyncio -async def test_response_create_gate_timeout_retires_old_precreated_request_after_rate_limit_telemetry( +async def test_http_bridge_owned_terminal_lifecycle_skips_outer_startup_release( monkeypatch: pytest.MonkeyPatch, ) -> None: - settings = _make_app_settings( - proxy_admission_wait_timeout_seconds=0.001, - http_responses_session_bridge_stuck_gate_retire_after_seconds=300.0, - ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) - service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) - session = _make_bridge_session() - service._http_bridge_sessions[session.key] = session - await session.response_create_gate.acquire() - old_pending = proxy_service._WebSocketRequestState( - request_id="req-old-pending-after-telemetry", + service = proxy_service.ProxyService(cast(Any, nullcontext())) + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv-bridge-owned", + key_id="key-bridge-owned", model="gpt-5.6-sol", - service_tier=None, - reasoning_effort="high", - api_key_reservation=None, - started_at=time.monotonic() - 301.0, - transport="http", - response_create_gate=session.response_create_gate, - response_create_gate_acquired=True, - awaiting_response_created=True, - downstream_visible=False, - event_queue=asyncio.Queue(), ) - waiter = proxy_service._WebSocketRequestState( - request_id="req-visible-waiter-after-telemetry", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort="high", - api_key_reservation=None, - started_at=time.monotonic(), - transport="http", - downstream_visible=True, + account = cast(Any, SimpleNamespace(id="acc-bridge-owned")) + runtime_config = SimpleNamespace( + enabled=True, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + prompt_cache_idle_ttl_seconds=120.0, ) - async with session.pending_lock: - session.pending_requests.append(old_pending) - session.queued_request_count = 1 - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "codex.rate_limits", - "plan_type": "pro", - "rate_limits": {"allowed": True, "limit_reached": False}, - }, - separators=(",", ":"), - ), + async def terminal_before_finalizer(*_args: object, **kwargs: object): + tracker = cast( + proxy_support_module._DeferredAccountBackoffTracker, + kwargs["deferred_account_backoff_tracker"], + ) + lifecycle = proxy_support_module._DeferredAccountBackoffLifecycle( + reservation=reservation, + pending_backoffs={account.id: account}, + settlement_owned=True, + ) + tracker.current_lifecycle = lifecycle + yield 'data: {"type":"response.completed"}\n\n' + + monkeypatch.setattr( + http_bridge_streaming_module, + "_service_get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=SimpleNamespace())), + ) + monkeypatch.setattr(http_bridge_streaming_module, "_service_get_settings", _make_app_settings) + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_runtime_config", lambda *_args: runtime_config) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_stream_via_http_bridge", terminal_before_finalizer) + release_reservation = AsyncMock() + drain_backoffs = AsyncMock() + monkeypatch.setattr(service, "_release_websocket_reservation", release_reservation) + monkeypatch.setattr(service, "_drain_deferred_account_error_backoffs", drain_backoffs) + payload = proxy_service.ResponsesRequest.model_validate( + {"model": "gpt-5.6-sol", "instructions": "test", "input": "hello"} ) - assert old_pending.latency_first_upstream_event_ms is not None - assert old_pending.latency_response_created_ms is None - assert old_pending.awaiting_response_created is True - assert old_pending.response_id is None - assert old_pending.downstream_visible is False - assert session.response_create_gate.locked() is True - - retire_calls: list[str] = [] - - async def fake_retire( - retire_session: proxy_service._HTTPBridgeSession, - *, - detail: str, - ) -> None: - retire_calls.append(detail) - retire_session.closed = True - - monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", fake_retire) - - try: - with pytest.raises(ProxyResponseError) as exc_info: - await service._acquire_request_state_response_create_admission( - waiter, - response_create_gate=session.response_create_gate, - account_id=session.account.id, - surface="http_bridge", - bridge_session=session, - ) - finally: - if session.response_create_gate.locked(): - session.response_create_gate.release() + chunks = [ + chunk + async for chunk in service._stream_http_bridge_or_retry( + payload, + {}, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=reservation, + suppress_text_done_events=False, + ) + ] - assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" - assert retire_calls == ["response_create_gate_timeout_stuck_pending"] - assert session.closed is True + assert chunks == ['data: {"type":"response.completed"}\n\n'] + release_reservation.assert_not_awaited() + drain_backoffs.assert_not_awaited() @pytest.mark.asyncio -@pytest.mark.parametrize( - ( - "awaiting_response_created", - "downstream_visible", - "latency_first_upstream_event_ms", - "latency_response_created_ms", - "response_event_count", - ), - [ - (False, True, 100, 100, 1), - ], -) -async def test_response_create_gate_timeout_does_not_retire_active_response_progress( +@pytest.mark.parametrize("release_fails", [False, True]) +async def test_http_bridge_startup_fallback_releases_current_lifecycle_before_backoff( monkeypatch: pytest.MonkeyPatch, - awaiting_response_created: bool, - downstream_visible: bool, - latency_first_upstream_event_ms: int, - latency_response_created_ms: int | None, - response_event_count: int, + release_fails: bool, ) -> None: - settings = _make_app_settings( - proxy_admission_wait_timeout_seconds=0.001, - http_responses_session_bridge_stuck_gate_retire_after_seconds=300.0, - ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) - service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) - session = _make_bridge_session() - service._http_bridge_sessions[session.key] = session - await session.response_create_gate.acquire() - active_stream = proxy_service._WebSocketRequestState( - request_id="req-active-visible", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic() - 301.0, - transport="http", - response_create_gate_acquired=True, - awaiting_response_created=awaiting_response_created, - downstream_visible=downstream_visible, - latency_first_upstream_event_ms=latency_first_upstream_event_ms, - latency_response_created_ms=latency_response_created_ms, - response_event_count=response_event_count, + service = proxy_service.ProxyService(cast(Any, nullcontext())) + original_reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv-bridge-original", + key_id="key-bridge-startup", + model="gpt-5.6-sol", ) - stale_pending = proxy_service._WebSocketRequestState( - request_id="req-stale-unanchored", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic() - 301.0, - transport="http", - response_create_gate_acquired=True, - awaiting_response_created=True, + current_reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv-bridge-current", + key_id="key-bridge-startup", + model="gpt-5.6-sol", ) - waiter = proxy_service._WebSocketRequestState( - request_id="req-visible-waiter", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - transport="http", - downstream_visible=True, + account = cast(Any, SimpleNamespace(id="acc-bridge-startup")) + runtime_config = SimpleNamespace( + enabled=True, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + prompt_cache_idle_ttl_seconds=120.0, ) - async with session.pending_lock: - session.pending_requests.append(active_stream) - session.pending_requests.append(stale_pending) - session.queued_request_count = 2 - - retire_calls: list[str] = [] + tracker_seen: proxy_support_module._DeferredAccountBackoffTracker | None = None + order: list[str] = [] - async def fake_retire( - retire_session: proxy_service._HTTPBridgeSession, - *, - detail: str, - ) -> None: - retire_calls.append(detail) - retire_session.closed = True + async def fail_before_submit(*_args: object, **kwargs: object): + nonlocal tracker_seen + tracker_seen = cast( + proxy_support_module._DeferredAccountBackoffTracker, + kwargs["deferred_account_backoff_tracker"], + ) + tracker_seen.current_lifecycle = proxy_support_module._DeferredAccountBackoffLifecycle( + reservation=current_reservation, + pending_backoffs={account.id: account}, + ) + raise RuntimeError("startup failed") + yield "" - monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", fake_retire) + async def release_reservation(reservation: object) -> None: + assert reservation is current_reservation + order.append("release") + if release_fails: + raise RuntimeError("release failed") - try: - with pytest.raises(ProxyResponseError) as exc_info: - await service._acquire_request_state_response_create_admission( - waiter, - response_create_gate=session.response_create_gate, - account_id=session.account.id, - surface="http_bridge", - bridge_session=session, - ) - finally: - if session.response_create_gate.locked(): - session.response_create_gate.release() + async def drain_backoffs(pending: dict[str, object]) -> None: + assert order == ["release"] + order.append("backoff") + pending.clear() - assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" - assert retire_calls == [] - assert session.closed is False + monkeypatch.setattr( + http_bridge_streaming_module, + "_service_get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=SimpleNamespace())), + ) + monkeypatch.setattr(http_bridge_streaming_module, "_service_get_settings", _make_app_settings) + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_runtime_config", lambda *_args: runtime_config) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_stream_via_http_bridge", fail_before_submit) + monkeypatch.setattr(service, "_release_websocket_reservation", release_reservation) + monkeypatch.setattr(service, "_drain_deferred_account_error_backoffs", drain_backoffs) + payload = proxy_service.ResponsesRequest.model_validate( + {"model": "gpt-5.6-sol", "instructions": "test", "input": "hello"} + ) + with pytest.raises(RuntimeError, match="startup failed"): + async for _ in service._stream_http_bridge_or_retry( + payload, + {}, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=original_reservation, + suppress_text_done_events=False, + ): + pass -def test_http_bridge_pending_state_with_recent_events_but_no_created_is_not_stale() -> None: - # Reattached streams can deliver events whose response.created was lost - # (observed events=54, created=None in prod on 2026-07-20). Recent upstream - # activity must keep the stream alive while the create gate remains held. - request_state = proxy_service._WebSocketRequestState( - request_id="req-events-no-created", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic() - 301.0, - transport="http", - response_create_gate_acquired=True, - awaiting_response_created=True, - latency_first_upstream_event_ms=25, - response_event_count=54, - last_upstream_activity_at=time.monotonic(), - ) + assert tracker_seen is not None + assert order == (["release"] if release_fails else ["release", "backoff"]) + assert tracker_seen.current_lifecycle is not None + assert tracker_seen.current_lifecycle.settlement_confirmed is not release_fails + assert bool(tracker_seen.current_lifecycle.pending_backoffs) is release_fails - assert ( - http_bridge_helpers_module._http_bridge_pending_state_is_stale( - request_state, - now=time.monotonic(), - threshold_seconds=300.0, - ) - is False - ) - request_state.last_upstream_activity_at = time.monotonic() - 301.0 - assert ( - http_bridge_helpers_module._http_bridge_pending_state_is_stale( - request_state, - now=time.monotonic(), - threshold_seconds=300.0, - ) - is True +@pytest.mark.asyncio +async def test_durable_turn_state_fence_rejection_rolls_back_local_alias( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + session.durable_session_id = "durable-session" + session.durable_owner_epoch = 3 + service._http_bridge_sessions[session.key] = session + service._durable_bridge = SimpleNamespace( + register_turn_state=AsyncMock(return_value=DurableBridgeAliasRegistration.OWNER_FENCED) ) + monkeypatch.setattr(http_bridge_helpers_module, "get_settings", _make_app_settings) + await service._register_http_bridge_turn_state(session, "turn-rejected") -def test_http_bridge_pending_state_with_first_event_latency_only_is_stale() -> None: - request_state = proxy_service._WebSocketRequestState( - request_id="req-sparse-active", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic() - 301.0, - transport="http", - response_create_gate_acquired=True, - awaiting_response_created=True, - latency_first_upstream_event_ms=10, - downstream_visible=False, - ) - - assert ( - http_bridge_helpers_module._http_bridge_pending_state_is_stale( - request_state, - now=time.monotonic(), - threshold_seconds=300.0, - ) - is True - ) + alias_key = proxy_service._http_bridge_turn_state_alias_key("turn-rejected", session.key.api_key_id) + assert "turn-rejected" not in session.downstream_turn_state_aliases + assert session.downstream_turn_state is None + assert alias_key not in service._http_bridge_turn_state_index -@pytest.mark.parametrize( - ("previous_response_id", "session_id", "hard_continuity_anchor"), - [ - ("resp-anchored", None, False), - (None, "turn-anchored", True), - ], -) -def test_http_bridge_pending_state_with_continuity_anchor_is_not_stale( - previous_response_id: str | None, - session_id: str | None, - hard_continuity_anchor: bool, +@pytest.mark.asyncio +@pytest.mark.parametrize("existing_kind", ["prompt_cache", "session_header", "turn_state_header"]) +async def test_verified_replay_turn_alias_rebind_cannot_be_stolen_by_old_session( + existing_kind: str, ) -> None: - request_state = proxy_service._WebSocketRequestState( - request_id="req-anchored-pending", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic() - 301.0, - transport="http", - response_create_gate_acquired=True, - awaiting_response_created=True, - previous_response_id=previous_response_id, - session_id=session_id, - hard_continuity_anchor=hard_continuity_anchor, - ) - - assert ( - http_bridge_helpers_module._http_bridge_pending_state_is_stale( - request_state, - now=time.monotonic(), - threshold_seconds=300.0, - ) - is False + service = proxy_service.ProxyService(cast(Any, nullcontext())) + turn_state = "http_turn_recovered" + old_session = _make_bridge_session(key=proxy_service._HTTPBridgeSessionKey(existing_kind, "old-owner", None)) + old_session.downstream_turn_state = turn_state + old_session.downstream_turn_state_aliases.add(turn_state) + recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("new-owner")) + recovery.durable_session_id = "durable-new-owner" + recovery.durable_owner_epoch = 2 + recovery.headers = { + "session_id": "retired", + "session-id": "retired", + "thread-id": "retired", + "x-codex-conversation-id": "retired", + "x-codex-session-id": "retired", + "x-codex-turn-state": turn_state, + } + service._http_bridge_sessions[old_session.key] = old_session + service._http_bridge_sessions[recovery.key] = recovery + service._durable_bridge = SimpleNamespace( + register_turn_state=AsyncMock(return_value=DurableBridgeAliasRegistration.REGISTERED) ) + alias_key = proxy_service._http_bridge_turn_state_alias_key(turn_state, None) + service._http_bridge_turn_state_index[alias_key] = old_session.key + await service._register_http_bridge_turn_state(recovery, turn_state) -@pytest.mark.parametrize( - ("previous_response_id", "session_id", "hard_continuity_anchor"), - [ - ("resp-anchored-silent", None, False), - (None, "turn-anchored-silent", True), - ], -) -def test_http_bridge_pending_state_with_continuity_anchor_is_stale_after_extended_silence( - previous_response_id: str | None, - session_id: str | None, - hard_continuity_anchor: bool, -) -> None: - request_state = proxy_service._WebSocketRequestState( - request_id="req-anchored-silent-pending", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic() - 601.0, - transport="http", - response_create_gate_acquired=True, - awaiting_response_created=True, - previous_response_id=previous_response_id, - session_id=session_id, - hard_continuity_anchor=hard_continuity_anchor, - ) + assert service._http_bridge_turn_state_index[alias_key] == recovery.key + assert turn_state not in old_session.downstream_turn_state_aliases + assert old_session.downstream_turn_state is None + assert recovery.headers == {} - assert ( - http_bridge_helpers_module._http_bridge_pending_state_is_stale( - request_state, - now=time.monotonic(), - threshold_seconds=300.0, - ) - is True - ) + # A stale local alias set must not reclaim a different live owner's lane. + old_session.downstream_turn_state_aliases.add(turn_state) + http_bridge_helpers_module._register_http_bridge_turn_state_aliases_locked(service, old_session) + assert service._http_bridge_turn_state_index[alias_key] == recovery.key -@pytest.mark.parametrize( - ("previous_response_id", "session_id", "hard_continuity_anchor"), - [ - ("resp-closed-anchored", None, False), - (None, "turn-closed-anchored", True), - ], -) -def test_http_bridge_closed_session_pending_anchor_is_stale( - previous_response_id: str | None, - session_id: str | None, - hard_continuity_anchor: bool, -) -> None: - request_state = proxy_service._WebSocketRequestState( - request_id="req-closed-anchored-pending", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic() - 301.0, - transport="http", - response_create_gate_acquired=True, - awaiting_response_created=True, - previous_response_id=previous_response_id, - session_id=session_id, - hard_continuity_anchor=hard_continuity_anchor, +@pytest.mark.asyncio +async def test_verified_replay_turn_alias_does_not_replace_unrelated_internal_lane() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + turn_state = "http_turn_conflict" + existing = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("internal_request_parallel", "other-lane", None) ) + recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("recovery-lane")) + service._http_bridge_sessions[existing.key] = existing + service._http_bridge_sessions[recovery.key] = recovery + alias_key = proxy_service._http_bridge_turn_state_alias_key(turn_state, None) + service._http_bridge_turn_state_index[alias_key] = existing.key - assert ( - http_bridge_helpers_module._http_bridge_pending_state_is_stale( - request_state, - now=time.monotonic(), - threshold_seconds=300.0, - session_closed=True, - ) - is True - ) + await service._register_http_bridge_turn_state(recovery, turn_state) + assert service._http_bridge_turn_state_index[alias_key] == existing.key + assert turn_state not in recovery.downstream_turn_state_aliases -def test_http_bridge_pending_state_with_plain_session_header_is_stale() -> None: - request_state = proxy_service._WebSocketRequestState( - request_id="req-session-header-pending", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic() - 301.0, - transport="http", - session_id="session-header-only", - response_create_gate_acquired=True, - awaiting_response_created=True, - ) - assert ( - http_bridge_helpers_module._http_bridge_pending_state_is_stale( - request_state, - now=time.monotonic(), - threshold_seconds=300.0, - ) - is True +@pytest.mark.asyncio +@pytest.mark.parametrize("alias_kind", ["turn_state", "previous_response"]) +async def test_verified_replay_alias_requires_durable_identity(alias_kind: str) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + alias = "http_turn_missing_durable_identity" if alias_kind == "turn_state" else "resp_missing_durable_identity" + predecessor = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "missing-durable-predecessor", None) ) + recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("missing-durable-recovery")) + service._http_bridge_sessions[predecessor.key] = predecessor + service._http_bridge_sessions[recovery.key] = recovery + if alias_kind == "turn_state": + predecessor.downstream_turn_state = alias + predecessor.downstream_turn_state_aliases.add(alias) + alias_key = proxy_service._http_bridge_turn_state_alias_key(alias, None) + service._http_bridge_turn_state_index[alias_key] = predecessor.key + registered = await service._register_http_bridge_turn_state(recovery, alias) + assert alias not in recovery.downstream_turn_state_aliases + assert alias in predecessor.downstream_turn_state_aliases + assert service._http_bridge_turn_state_index[alias_key] == predecessor.key + else: + predecessor.previous_response_ids.add(alias) + alias_key = proxy_service._http_bridge_previous_response_alias_key(alias, None) + service._http_bridge_previous_response_index[alias_key] = predecessor.key + registered = await service._register_http_bridge_previous_response_id(recovery, alias) + assert alias not in recovery.previous_response_ids + assert alias in predecessor.previous_response_ids + assert service._http_bridge_previous_response_index[alias_key] == predecessor.key + assert registered is False -def test_http_bridge_synthesized_downstream_turn_state_is_not_hard_anchor() -> None: - request_state = proxy_service._WebSocketRequestState( - request_id="req-synth-turn-state", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - transport="http", - ) - http_bridge_streaming_module._apply_http_bridge_downstream_turn_state( - request_state, - downstream_turn_state="synthesized-turn-state", - incoming_turn_state_header=None, +@pytest.mark.asyncio +@pytest.mark.parametrize("existing_kind", ["prompt_cache", "session_header", "turn_state_header"]) +async def test_verified_replay_response_alias_rebind_cannot_be_stolen_by_old_session( + existing_kind: str, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + response_id = "resp_recovered" + old_session = _make_bridge_session(key=proxy_service._HTTPBridgeSessionKey(existing_kind, "old-owner", None)) + old_session.previous_response_ids.add(response_id) + recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("new-owner")) + recovery.durable_session_id = "durable-new-owner" + recovery.durable_owner_epoch = 2 + service._http_bridge_sessions[old_session.key] = old_session + service._http_bridge_sessions[recovery.key] = recovery + service._durable_bridge = SimpleNamespace( + register_previous_response_id=AsyncMock(return_value=DurableBridgeAliasRegistration.REGISTERED) ) + alias_key = proxy_service._http_bridge_previous_response_alias_key(response_id, None) + service._http_bridge_previous_response_index[alias_key] = old_session.key - assert request_state.session_id == "synthesized-turn-state" - assert request_state.hard_continuity_anchor is False + await service._register_http_bridge_previous_response_id(recovery, response_id) + assert service._http_bridge_previous_response_index[alias_key] == recovery.key + assert response_id not in old_session.previous_response_ids + assert response_id in recovery.previous_response_ids -@pytest.mark.parametrize( - ("incoming_turn_state_header", "previous_response_id"), - [ - ("client-turn-state", None), - (None, "resp-continuation"), - ], -) -def test_http_bridge_real_continuity_sets_hard_anchor( - incoming_turn_state_header: str | None, - previous_response_id: str | None, + await service._register_http_bridge_previous_response_id(old_session, response_id) + + assert service._http_bridge_previous_response_index[alias_key] == recovery.key + assert response_id not in old_session.previous_response_ids + + +@pytest.mark.asyncio +@pytest.mark.parametrize("alias_kind", ["turn_state", "previous_response"]) +@pytest.mark.parametrize("durable_outcome", ["registered", "protected", "exception"]) +async def test_durable_verified_replay_alias_is_published_only_after_fenced_write( + alias_kind: str, + durable_outcome: str, ) -> None: - request_state = proxy_service._WebSocketRequestState( - request_id="req-real-anchor", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - transport="http", - previous_response_id=previous_response_id, + service = proxy_service.ProxyService(cast(Any, nullcontext())) + alias = "http_turn_atomic_rebind" if alias_kind == "turn_state" else "resp_atomic_rebind" + predecessor = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "atomic-predecessor", None) ) + recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("atomic-recovery")) + recovery.durable_session_id = "durable-atomic-recovery" + recovery.durable_owner_epoch = 2 + service._http_bridge_sessions[predecessor.key] = predecessor + service._http_bridge_sessions[recovery.key] = recovery + if alias_kind == "turn_state": + predecessor.downstream_turn_state = alias + predecessor.downstream_turn_state_aliases.add(alias) + alias_key = proxy_service._http_bridge_turn_state_alias_key(alias, None) + service._http_bridge_turn_state_index[alias_key] = predecessor.key + else: + predecessor.previous_response_ids.add(alias) + alias_key = proxy_service._http_bridge_previous_response_alias_key(alias, None) + service._http_bridge_previous_response_index[alias_key] = predecessor.key - http_bridge_streaming_module._apply_http_bridge_downstream_turn_state( - request_state, - downstream_turn_state="real-turn-state", - incoming_turn_state_header=incoming_turn_state_header, + write_started = asyncio.Event() + release_write = asyncio.Event() + + async def persist_alias(**_kwargs: Any) -> DurableBridgeAliasRegistration: + write_started.set() + await release_write.wait() + if durable_outcome == "exception": + raise RuntimeError("durable alias write failed") + if durable_outcome == "protected": + return DurableBridgeAliasRegistration.ALIAS_PROTECTED + return DurableBridgeAliasRegistration.REGISTERED + + service._durable_bridge = SimpleNamespace( + register_turn_state=persist_alias, + register_previous_response_id=persist_alias, ) - assert request_state.session_id == "real-turn-state" - assert request_state.hard_continuity_anchor is True + if alias_kind == "turn_state": + registration = asyncio.create_task(service._register_http_bridge_turn_state(recovery, alias)) + else: + registration = asyncio.create_task(service._register_http_bridge_previous_response_id(recovery, alias)) + try: + await asyncio.wait_for(write_started.wait(), 1.0) + if alias_kind == "turn_state": + assert service._http_bridge_turn_state_index[alias_key] == predecessor.key + assert alias in predecessor.downstream_turn_state_aliases + assert alias not in recovery.downstream_turn_state_aliases + else: + assert service._http_bridge_previous_response_index[alias_key] == predecessor.key + assert alias in predecessor.previous_response_ids + assert alias not in recovery.previous_response_ids -@pytest.mark.asyncio -async def test_http_bridge_activity_snapshot_counts_only_bridge_cleanup_tasks(): - service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) - bridge_task = asyncio.create_task(asyncio.sleep(60), name="proxy-http_bridge_session_close-test") - api_key_task = asyncio.create_task(asyncio.sleep(60), name="proxy-stream-api-key-settle-test") - service._background_cleanup_tasks.update({bridge_task, api_key_task}) + release_write.set() + await asyncio.wait_for(registration, 1.0) - try: - snapshot = service.http_bridge_activity_snapshot_nowait() + expected_owner = recovery if durable_outcome == "registered" else predecessor + if alias_kind == "turn_state": + assert service._http_bridge_turn_state_index[alias_key] == expected_owner.key + assert (alias in recovery.downstream_turn_state_aliases) is (durable_outcome == "registered") + assert (alias in predecessor.downstream_turn_state_aliases) is (durable_outcome != "registered") + else: + assert service._http_bridge_previous_response_index[alias_key] == expected_owner.key + assert (alias in recovery.previous_response_ids) is (durable_outcome == "registered") + assert (alias in predecessor.previous_response_ids) is (durable_outcome != "registered") finally: - bridge_task.cancel() - api_key_task.cancel() - await asyncio.gather(bridge_task, api_key_task, return_exceptions=True) - - assert snapshot["http_bridge_background_cleanup_tasks"] == 1 + release_write.set() + if not registration.done(): + registration.cancel() + await asyncio.gather(registration, return_exceptions=True) @pytest.mark.asyncio -async def test_http_bridge_activity_snapshot_cleans_completed_stale_inflight_session( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) - key = proxy_service._HTTPBridgeSessionKey("session_header", "stale-inflight-drain-status", None) - inflight_future: asyncio.Future[proxy_service._HTTPBridgeSession] = asyncio.get_running_loop().create_future() - setattr(inflight_future, "_codex_lb_started_at", -1000.0) - inflight_future.set_result(_make_bridge_session()) - service._http_bridge_inflight_sessions[key] = inflight_future +@pytest.mark.parametrize("alias_kind", ["turn_state", "previous_response"]) +async def test_durable_verified_replay_alias_writes_do_not_serialize_unrelated_sessions(alias_kind: str) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + alias = "http_turn_serial_rebind" if alias_kind == "turn_state" else "resp_serial_rebind" + predecessor = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "serial-predecessor", None) + ) + recoveries = [ + _make_bridge_session(key=_make_account_neutral_replay_session_key(f"serial-recovery-{index}")) + for index in range(2) + ] + for index, recovery in enumerate(recoveries): + recovery.durable_session_id = f"durable-serial-recovery-{index}" + recovery.durable_owner_epoch = 2 + service._http_bridge_sessions[recovery.key] = recovery + service._http_bridge_sessions[predecessor.key] = predecessor + if alias_kind == "turn_state": + predecessor.downstream_turn_state = alias + predecessor.downstream_turn_state_aliases.add(alias) + alias_key = proxy_service._http_bridge_turn_state_alias_key(alias, None) + service._http_bridge_turn_state_index[alias_key] = predecessor.key + else: + predecessor.previous_response_ids.add(alias) + alias_key = proxy_service._http_bridge_previous_response_alias_key(alias, None) + service._http_bridge_previous_response_index[alias_key] = predecessor.key - monkeypatch.setattr(proxy_service, "_proxy_admission_wait_timeout_seconds", lambda settings=None: 0.001) + write_started = [asyncio.Event(), asyncio.Event()] + release_write = [asyncio.Event(), asyncio.Event()] + durable_write_order: list[str] = [] - with caplog.at_level(logging.WARNING, logger="app.modules.proxy.service"): - snapshot = service.http_bridge_activity_snapshot_nowait() + async def persist_alias(**kwargs: Any) -> DurableBridgeAliasRegistration: + write_index = len(durable_write_order) + durable_write_order.append(kwargs["session_id"]) + write_started[write_index].set() + await release_write[write_index].wait() + return ( + DurableBridgeAliasRegistration.REGISTERED + if write_index == 0 + else DurableBridgeAliasRegistration.ALIAS_PROTECTED + ) - assert key not in service._http_bridge_inflight_sessions - assert snapshot["http_bridge_inflight_session_creates"] == 0 - assert snapshot["http_bridge_stale_inflight_session_creates"] == 1 - assert snapshot["http_bridge_cleaned_inflight_session_creates"] == 1 - assert snapshot["http_bridge_active"] is False - assert snapshot["http_bridge_restart_blocking"] is False - assert "http_bridge_inflight_session_create_cleanup" in caplog.text + service._durable_bridge = SimpleNamespace( + register_turn_state=persist_alias, + register_previous_response_id=persist_alias, + ) + async def register(recovery: proxy_service._HTTPBridgeSession) -> bool: + if alias_kind == "turn_state": + return await service._register_http_bridge_turn_state(recovery, alias) + return await service._register_http_bridge_previous_response_id(recovery, alias) -@pytest.mark.asyncio -async def test_http_bridge_activity_snapshot_does_not_expire_live_inflight_session( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) - key = proxy_service._HTTPBridgeSessionKey("session_header", "live-stale-inflight-drain-status", None) - inflight_future: asyncio.Future[proxy_service._HTTPBridgeSession] = asyncio.get_running_loop().create_future() - setattr(inflight_future, "_codex_lb_started_at", -1000.0) - service._http_bridge_inflight_sessions[key] = inflight_future + registrations = [asyncio.create_task(register(recoveries[0]))] + try: + await asyncio.wait_for(write_started[0].wait(), 1.0) + registrations.append(asyncio.create_task(register(recoveries[1]))) + await asyncio.wait_for(write_started[1].wait(), 1.0) - monkeypatch.setattr(proxy_service, "_proxy_admission_wait_timeout_seconds", lambda settings=None: 0.001) + release_write[0].set() + first_registered = await asyncio.wait_for(asyncio.shield(registrations[0]), 1.0) + if alias_kind == "turn_state": + assert service._http_bridge_turn_state_index[alias_key] == recoveries[0].key + else: + assert service._http_bridge_previous_response_index[alias_key] == recoveries[0].key - snapshot = service.http_bridge_activity_snapshot_nowait() + release_write[1].set() + second_registered = await asyncio.wait_for(registrations[1], 1.0) - assert key in service._http_bridge_inflight_sessions - assert not inflight_future.done() - assert snapshot["http_bridge_inflight_session_creates"] == 1 - assert snapshot["http_bridge_stale_inflight_session_creates"] == 1 - assert snapshot["http_bridge_cleaned_inflight_session_creates"] == 0 - assert snapshot["http_bridge_active"] is True - assert snapshot["http_bridge_restart_blocking"] is True + assert durable_write_order == [recovery.durable_session_id for recovery in recoveries] + assert [first_registered, second_registered] == [True, False] + if alias_kind == "turn_state": + assert service._http_bridge_turn_state_index[alias_key] == recoveries[0].key + assert alias in recoveries[0].downstream_turn_state_aliases + assert alias not in recoveries[1].downstream_turn_state_aliases + else: + assert service._http_bridge_previous_response_index[alias_key] == recoveries[0].key + assert alias in recoveries[0].previous_response_ids + assert alias not in recoveries[1].previous_response_ids + finally: + for release in release_write: + release.set() + for registration in registrations: + if not registration.done(): + registration.cancel() + await asyncio.gather(*registrations, return_exceptions=True) @pytest.mark.asyncio -async def test_http_bridge_activity_snapshot_skips_inflight_cleanup_when_registry_locked( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) - key = proxy_service._HTTPBridgeSessionKey("session_header", "locked-stale-inflight-drain-status", None) - inflight_future: asyncio.Future[proxy_service._HTTPBridgeSession] = asyncio.get_running_loop().create_future() - setattr(inflight_future, "_codex_lb_started_at", -1000.0) - service._http_bridge_inflight_sessions[key] = inflight_future +async def test_durable_verified_replay_alias_writes_serialize_within_one_session() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("serial-one-session")) + recovery.durable_session_id = "durable-serial-one-session" + recovery.durable_owner_epoch = 2 + service._http_bridge_sessions[recovery.key] = recovery + writes_started = [asyncio.Event(), asyncio.Event()] + release_writes = [asyncio.Event(), asyncio.Event()] + write_count = 0 - monkeypatch.setattr(proxy_service, "_proxy_admission_wait_timeout_seconds", lambda settings=None: 0.001) + async def persist_alias(**_kwargs: Any) -> DurableBridgeAliasRegistration: + nonlocal write_count + write_index = write_count + write_count += 1 + writes_started[write_index].set() + await release_writes[write_index].wait() + return DurableBridgeAliasRegistration.REGISTERED - async with service._http_bridge_lock: - snapshot = service.http_bridge_activity_snapshot_nowait() - - assert key in service._http_bridge_inflight_sessions - assert not inflight_future.done() - assert snapshot["http_bridge_inflight_session_creates"] == 1 - assert snapshot["http_bridge_stale_inflight_session_creates"] == 1 - assert snapshot["http_bridge_cleaned_inflight_session_creates"] == 0 - assert snapshot["http_bridge_active"] is True - assert snapshot["http_bridge_restart_blocking"] is True - - -async def _wait_for_close_await(close_session: AsyncMock, session: proxy_service._HTTPBridgeSession) -> None: - for _ in range(10): - if any(call.args == (session,) for call in close_session.await_args_list): - return + service._durable_bridge = SimpleNamespace( + register_turn_state=persist_alias, + register_previous_response_id=persist_alias, + ) + turn_registration = asyncio.create_task( + service._register_http_bridge_turn_state(recovery, "http_turn_serial_one_session") + ) + response_registration: asyncio.Task[bool] | None = None + try: + await asyncio.wait_for(writes_started[0].wait(), 1.0) + response_registration = asyncio.create_task( + service._register_http_bridge_previous_response_id(recovery, "resp_serial_one_session") + ) await asyncio.sleep(0) - raise AssertionError("expected HTTP bridge session close to be awaited") - + assert writes_started[1].is_set() is False -def test_http_bridge_account_capacity_wait_treats_workspace_spend_cap_as_recoverable() -> None: - exc = ProxyResponseError( - 429, - openai_error( - "no_accounts", - ( - "You hit your spend cap set by the owner of your workspace. " - "Ask an owner to increase your spend cap to continue." - ), - ), - ) + release_writes[0].set() + assert await asyncio.wait_for(turn_registration, 1.0) is True + await asyncio.wait_for(writes_started[1].wait(), 1.0) + release_writes[1].set() + assert await asyncio.wait_for(response_registration, 1.0) is True + finally: + for release_write in release_writes: + release_write.set() + tasks = [turn_registration] + if response_registration is not None: + tasks.append(response_registration) + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) - assert http_bridge_streaming_module._http_bridge_account_capacity_wait_seconds(exc) == 30.0 +@pytest.mark.asyncio +@pytest.mark.parametrize("alias_kind", ["turn_state", "previous_response"]) +async def test_durable_active_recovery_alias_protection_prevents_superseding_replica_publication( + alias_kind: str, +) -> None: + services = [proxy_service.ProxyService(cast(Any, nullcontext())) for _ in range(2)] + recoveries = [ + _make_bridge_session(key=_make_account_neutral_replay_session_key(f"replica-recovery-{index}")) + for index in range(2) + ] + alias = "http_turn_replica_race" if alias_kind == "turn_state" else "resp_replica_race" + for index, (service, recovery) in enumerate(zip(services, recoveries, strict=True)): + recovery.account = cast( + Any, + SimpleNamespace(id=f"acc-replica-{index}", status=AccountStatus.ACTIVE, plan_type="plus"), + ) + recovery.durable_session_id = f"durable-replica-recovery-{index}" + recovery.durable_owner_epoch = 2 + service._http_bridge_sessions[recovery.key] = recovery -def test_http_bridge_account_capacity_wait_honors_upstream_rate_limit_retry_hint() -> None: - exc = ProxyResponseError( - 429, - openai_error( - "rate_limit_exceeded", - "Rate limit exceeded. Try again in 120s", - ), - ) + first_write_committed = asyncio.Event() + release_first_writer = asyncio.Event() + durable_owner: list[str] = [] - assert http_bridge_streaming_module._http_bridge_account_capacity_wait_seconds(exc) == 120.0 + async def persist_first(**kwargs: Any) -> DurableBridgeAliasRegistration: + durable_owner[:] = [kwargs["session_id"]] + first_write_committed.set() + await release_first_writer.wait() + return DurableBridgeAliasRegistration.REGISTERED + async def persist_second(**kwargs: Any) -> DurableBridgeAliasRegistration: + del kwargs + assert durable_owner == [recoveries[0].durable_session_id] + return DurableBridgeAliasRegistration.ALIAS_PROTECTED -def test_http_bridge_account_capacity_wait_ignores_local_no_accounts_retry_hint() -> None: - exc = ProxyResponseError( - 429, - openai_error( - "no_accounts", - "Rate limit exceeded. Try again in 120s", - ), + services[0]._durable_bridge = SimpleNamespace( + register_turn_state=persist_first, + register_previous_response_id=persist_first, ) - - assert http_bridge_streaming_module._http_bridge_account_capacity_wait_seconds(exc) is None - - -@pytest.mark.parametrize("error_code", ["account_stream_cap", "account_response_create_cap"]) -def test_http_bridge_account_capacity_wait_treats_local_account_caps_as_recoverable(error_code: str) -> None: - exc = ProxyResponseError( - 429, - openai_error( - error_code, - "Account stream capacity is exhausted; per-account limit is 8.", - ), + services[1]._durable_bridge = SimpleNamespace( + register_turn_state=persist_second, + register_previous_response_id=persist_second, ) - assert http_bridge_streaming_module._http_bridge_account_capacity_wait_seconds(exc) == 30.0 - + async def register(service: proxy_service.ProxyService, recovery: proxy_service._HTTPBridgeSession) -> bool: + if alias_kind == "turn_state": + return await service._register_http_bridge_turn_state(recovery, alias) + return await service._register_http_bridge_previous_response_id(recovery, alias) -def test_http_bridge_account_capacity_wait_treats_gate_timeout_as_recoverable() -> None: - exc = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( - "http_bridge_response_create_gate", - code="response_create_gate_timeout", - ) + first_registration = asyncio.create_task(register(services[0], recoveries[0])) + try: + await asyncio.wait_for(first_write_committed.wait(), 1.0) + second_registered = await asyncio.wait_for(register(services[1], recoveries[1]), 1.0) + release_first_writer.set() + first_registered = await asyncio.wait_for(first_registration, 1.0) - assert ( - http_bridge_streaming_module._http_bridge_account_capacity_wait_seconds(exc) - == http_bridge_streaming_module._RESPONSE_CREATE_GATE_RETRY_SLEEP_SECONDS - ) + assert first_registered is True + assert second_registered is False + assert durable_owner == [recoveries[0].durable_session_id] + if alias_kind == "turn_state": + alias_key = proxy_service._http_bridge_turn_state_alias_key(alias, None) + assert services[0]._http_bridge_turn_state_index[alias_key] == recoveries[0].key + assert alias in recoveries[0].downstream_turn_state_aliases + assert alias_key not in services[1]._http_bridge_turn_state_index + assert alias not in recoveries[1].downstream_turn_state_aliases + else: + alias_key = proxy_service._http_bridge_previous_response_alias_key(alias, None) + assert services[0]._http_bridge_previous_response_index[alias_key] == recoveries[0].key + assert alias in recoveries[0].previous_response_ids + assert alias_key not in services[1]._http_bridge_previous_response_index + assert alias not in recoveries[1].previous_response_ids + finally: + release_first_writer.set() + if not first_registration.done(): + first_registration.cancel() + await asyncio.gather(first_registration, return_exceptions=True) -def test_http_bridge_capacity_wait_plan_reserves_final_gate_attempt(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - http_bridge_streaming_module, - "_proxy_admission_wait_timeout_seconds", - lambda settings=None: 10.0, - ) - exc = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( - "http_bridge_response_create_gate", - code="response_create_gate_timeout", +@pytest.mark.asyncio +async def test_durable_protected_alias_rejection_preserves_sibling_session() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="shared-owner") + session.durable_session_id = "durable-shared-owner" + session.durable_owner_epoch = 3 + session.downstream_turn_state = "http_turn_recovered_elsewhere" + session.downstream_turn_state_aliases.update({"http_turn_recovered_elsewhere", "http_turn_sibling"}) + service._http_bridge_sessions[session.key] = session + recovered_alias_key = proxy_service._http_bridge_turn_state_alias_key("http_turn_recovered_elsewhere", None) + sibling_alias_key = proxy_service._http_bridge_turn_state_alias_key("http_turn_sibling", None) + service._http_bridge_turn_state_index[recovered_alias_key] = session.key + service._http_bridge_turn_state_index[sibling_alias_key] = session.key + service._durable_bridge = SimpleNamespace( + register_turn_state=AsyncMock(return_value=DurableBridgeAliasRegistration.ALIAS_PROTECTED) ) - now = time.monotonic() - plenty = http_bridge_streaming_module._http_bridge_capacity_wait_plan(exc, request_deadline=now + 120.0) - assert plenty is not None - assert plenty[0] == pytest.approx( - http_bridge_streaming_module._RESPONSE_CREATE_GATE_RETRY_SLEEP_SECONDS, - abs=0.5, - ) + await service._register_http_bridge_turn_state(session, "http_turn_recovered_elsewhere") - # With less budget left than the retry sleep, the plan reserves the tail - # for one final gate acquisition attempt instead of sleeping it away. - tail = http_bridge_streaming_module._http_bridge_capacity_wait_plan(exc, request_deadline=now + 8.0) - assert tail is not None - assert tail[0] == pytest.approx(0.0, abs=0.5) + assert session.closed is False + assert service._http_bridge_sessions[session.key] is session + assert "http_turn_recovered_elsewhere" not in session.downstream_turn_state_aliases + assert recovered_alias_key not in service._http_bridge_turn_state_index + assert "http_turn_sibling" in session.downstream_turn_state_aliases + assert service._http_bridge_turn_state_index[sibling_alias_key] == session.key -def test_http_bridge_account_capacity_wait_keeps_active_session_capacity_fail_fast() -> None: - exc = ProxyResponseError( - 429, - openai_error( - "capacity_exhausted_active_sessions", - "All accounts are serving active sessions", - error_type="rate_limit_error", - ), +@pytest.mark.asyncio +async def test_durable_response_fence_rejection_rolls_back_local_alias( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + session.durable_session_id = "durable-session" + session.durable_owner_epoch = 3 + service._http_bridge_sessions[session.key] = session + service._durable_bridge = SimpleNamespace( + register_previous_response_id=AsyncMock(return_value=DurableBridgeAliasRegistration.OWNER_FENCED) ) + monkeypatch.setattr(http_bridge_helpers_module, "get_settings", _make_app_settings) - assert http_bridge_streaming_module._http_bridge_account_capacity_wait_seconds(exc) is None + await service._register_http_bridge_previous_response_id(session, "resp-rejected") + + alias_key = proxy_service._http_bridge_previous_response_alias_key("resp-rejected", session.key.api_key_id) + assert "resp-rejected" not in session.previous_response_ids + assert alias_key not in service._http_bridge_previous_response_index @pytest.mark.asyncio -@pytest.mark.parametrize( - ("propagate_http_errors", "expected_event_types"), - [ - (False, ["codex.keepalive", "response.completed"]), - (True, ["response.completed"]), - ], -) -async def test_http_bridge_submit_waits_for_local_account_capacity( +async def test_durable_alias_fence_rejection_rolls_back_after_same_session_epoch_refresh( monkeypatch: pytest.MonkeyPatch, - propagate_http_errors: bool, - expected_event_types: list[str], ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="sid-submit-capacity") - request_state = proxy_service._WebSocketRequestState( - request_id="req-submit-capacity", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - transport="http", - event_queue=asyncio.Queue(), - ) - assert request_state.event_queue is not None - request_state.event_queue.put_nowait( - 'data: {"type":"response.completed","response":{"id":"resp_submit_capacity"}}\n\n' + session = _make_bridge_session() + session.durable_session_id = "durable-session" + session.durable_owner_epoch = 3 + service._http_bridge_sessions[session.key] = session + + async def reject_turn_state(**_kwargs: Any) -> DurableBridgeAliasRegistration: + session.durable_owner_epoch = 4 + return DurableBridgeAliasRegistration.OWNER_FENCED + + async def reject_previous_response(**_kwargs: Any) -> DurableBridgeAliasRegistration: + session.durable_owner_epoch = 5 + return DurableBridgeAliasRegistration.OWNER_FENCED + + service._durable_bridge = SimpleNamespace( + register_turn_state=reject_turn_state, + register_previous_response_id=reject_previous_response, ) - request_state.event_queue.put_nowait(None) - capacity_error = ProxyResponseError( - 429, - openai_error( - "account_response_create_cap", - "Account response-create concurrency limit reached", - error_type="rate_limit_error", - ), + monkeypatch.setattr(http_bridge_helpers_module, "get_settings", _make_app_settings) + + await service._register_http_bridge_turn_state(session, "turn-rejected-after-refresh") + await service._register_http_bridge_previous_response_id(session, "resp-rejected-after-refresh") + + turn_alias_key = proxy_service._http_bridge_turn_state_alias_key( + "turn-rejected-after-refresh", session.key.api_key_id ) - submit = AsyncMock(side_effect=[capacity_error, None]) - detach = AsyncMock() + response_alias_key = proxy_service._http_bridge_previous_response_alias_key( + "resp-rejected-after-refresh", session.key.api_key_id + ) + assert "turn-rejected-after-refresh" not in session.downstream_turn_state_aliases + assert session.downstream_turn_state is None + assert turn_alias_key not in service._http_bridge_turn_state_index + assert "resp-rejected-after-refresh" not in session.previous_response_ids + assert response_alias_key not in service._http_bridge_previous_response_index - settings = _make_app_settings() - monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) - monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 0.001) - monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 0.001) - monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - monkeypatch.setattr(service, "_detach_http_bridge_request", detach) - chunks = [ - chunk - async for chunk in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data='{"type":"response.create"}', - queue_limit=4, - propagate_http_errors=propagate_http_errors, - downstream_turn_state=None, - ) - ] +@pytest.mark.asyncio +async def test_stale_turn_state_rejection_preserves_newer_same_session_registration( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + session.durable_session_id = "durable-session" + session.durable_owner_epoch = 3 + service._http_bridge_sessions[session.key] = session + first_started = asyncio.Event() + release_first = asyncio.Event() - event_types = [cast(dict[str, object], proxy_service.parse_sse_data_json(chunk))["type"] for chunk in chunks] - assert event_types == expected_event_types - assert submit.await_count == 2 - detach.assert_awaited_once_with(session, request_state=request_state) + async def register_turn_state(*, owner_epoch: int, **_kwargs: Any) -> DurableBridgeAliasRegistration: + if owner_epoch == 3: + first_started.set() + await release_first.wait() + return DurableBridgeAliasRegistration.OWNER_FENCED + assert owner_epoch == 4 + return DurableBridgeAliasRegistration.REGISTERED + + service._durable_bridge = SimpleNamespace(register_turn_state=register_turn_state) + monkeypatch.setattr(http_bridge_helpers_module, "get_settings", _make_app_settings) + + stale_registration = asyncio.create_task(service._register_http_bridge_turn_state(session, "turn-race")) + try: + await asyncio.wait_for(first_started.wait(), 1.0) + session.durable_owner_epoch = 4 + await service._register_http_bridge_turn_state(session, "turn-race") + release_first.set() + await asyncio.wait_for(stale_registration, 1.0) + + alias_key = proxy_service._http_bridge_turn_state_alias_key("turn-race", session.key.api_key_id) + assert "turn-race" in session.downstream_turn_state_aliases + assert session.downstream_turn_state == "turn-race" + assert service._http_bridge_turn_state_index[alias_key] == session.key + finally: + release_first.set() + if not stale_registration.done(): + stale_registration.cancel() + await asyncio.gather(stale_registration, return_exceptions=True) @pytest.mark.asyncio -async def test_http_bridge_submit_waits_for_response_create_gate_contention( +async def test_stale_response_rejection_preserves_newer_same_session_registration( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="sid-submit-gate-contention") - request_state = proxy_service._WebSocketRequestState( - request_id="req-submit-gate-contention", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - transport="http", - event_queue=asyncio.Queue(), - ) - assert request_state.event_queue is not None - request_state.event_queue.put_nowait( - 'data: {"type":"response.completed","response":{"id":"resp_submit_gate_contention"}}\n\n' - ) - request_state.event_queue.put_nowait(None) - gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( - "http_bridge_response_create_gate", - code="response_create_gate_timeout", - ) - submit = AsyncMock(side_effect=[gate_timeout_error, None]) - detach = AsyncMock() + session = _make_bridge_session() + session.durable_session_id = "durable-session" + session.durable_owner_epoch = 3 + service._http_bridge_sessions[session.key] = session + first_started = asyncio.Event() + release_first = asyncio.Event() - settings = _make_app_settings() - monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) - monkeypatch.setattr(http_bridge_streaming_module, "_RESPONSE_CREATE_GATE_RETRY_SLEEP_SECONDS", 0.001) - monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 0.001) - monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - monkeypatch.setattr(service, "_detach_http_bridge_request", detach) + async def register_previous_response_id(*, owner_epoch: int, **_kwargs: Any) -> DurableBridgeAliasRegistration: + if owner_epoch == 3: + first_started.set() + await release_first.wait() + return DurableBridgeAliasRegistration.OWNER_FENCED + assert owner_epoch == 4 + return DurableBridgeAliasRegistration.REGISTERED - chunks = [ - chunk - async for chunk in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data='{"type":"response.create"}', - queue_limit=4, - propagate_http_errors=False, - downstream_turn_state=None, - ) - ] + service._durable_bridge = SimpleNamespace(register_previous_response_id=register_previous_response_id) + monkeypatch.setattr(http_bridge_helpers_module, "get_settings", _make_app_settings) - event_types = [cast(dict[str, object], proxy_service.parse_sse_data_json(chunk))["type"] for chunk in chunks] - assert event_types == ["codex.keepalive", "response.completed"] - keepalive = cast(dict[str, object], proxy_service.parse_sse_data_json(chunks[0])) - assert keepalive["status"] == "waiting_for_account_capacity" - assert submit.await_count == 2 - detach.assert_awaited_once_with(session, request_state=request_state) + stale_registration = asyncio.create_task(service._register_http_bridge_previous_response_id(session, "resp-race")) + try: + await asyncio.wait_for(first_started.wait(), 1.0) + session.durable_owner_epoch = 4 + await service._register_http_bridge_previous_response_id(session, "resp-race") + release_first.set() + await asyncio.wait_for(stale_registration, 1.0) + + alias_key = proxy_service._http_bridge_previous_response_alias_key("resp-race", session.key.api_key_id) + assert "resp-race" in session.previous_response_ids + assert service._http_bridge_previous_response_index[alias_key] == session.key + finally: + release_first.set() + if not stale_registration.done(): + stale_registration.cancel() + await asyncio.gather(stale_registration, return_exceptions=True) @pytest.mark.asyncio -async def test_http_bridge_stream_persists_original_deadline_on_request_state( +async def test_durable_alias_fence_rejection_preserves_new_local_generation( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="sid-deadline-persist") - request_state = proxy_service._WebSocketRequestState( - request_id="req-deadline-persist", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - # Retry/recovery states are re-prepared with a fresh started_at; - # the original deadline must still govern budget clamps. - started_at=time.monotonic(), - transport="http", - event_queue=asyncio.Queue(), + stale_session = _make_bridge_session() + stale_session.durable_session_id = "durable-stale" + stale_session.durable_owner_epoch = 3 + current_session = _make_bridge_session() + current_session.downstream_turn_state_aliases.add("turn-current") + current_session.previous_response_ids.add("resp-current") + service._http_bridge_sessions[current_session.key] = current_session + turn_alias_key = proxy_service._http_bridge_turn_state_alias_key("turn-current", current_session.key.api_key_id) + response_alias_key = proxy_service._http_bridge_previous_response_alias_key( + "resp-current", current_session.key.api_key_id ) - assert request_state.event_queue is not None - request_state.event_queue.put_nowait( - 'data: {"type":"response.completed","response":{"id":"resp_deadline_persist"}}\n\n' + service._http_bridge_turn_state_index[turn_alias_key] = current_session.key + service._http_bridge_previous_response_index[response_alias_key] = current_session.key + service._durable_bridge = SimpleNamespace( + register_turn_state=AsyncMock(return_value=DurableBridgeAliasRegistration.OWNER_FENCED), + register_previous_response_id=AsyncMock(return_value=DurableBridgeAliasRegistration.OWNER_FENCED), ) - request_state.event_queue.put_nowait(None) - submit = AsyncMock(return_value=None) - detach = AsyncMock() - settings = _make_app_settings() - monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) - monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - monkeypatch.setattr(service, "_detach_http_bridge_request", detach) + monkeypatch.setattr(http_bridge_helpers_module, "get_settings", _make_app_settings) - explicit_deadline = time.monotonic() + 42.0 - async for _ in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data='{"type":"response.create"}', - queue_limit=4, - propagate_http_errors=False, - downstream_turn_state=None, - request_deadline=explicit_deadline, - ): - pass + await service._register_http_bridge_turn_state(stale_session, "turn-current") + await service._register_http_bridge_previous_response_id(stale_session, "resp-current") - assert request_state.bridge_request_deadline == pytest.approx(explicit_deadline) + assert "turn-current" not in stale_session.downstream_turn_state_aliases + assert "resp-current" not in stale_session.previous_response_ids + assert service._http_bridge_turn_state_index[turn_alias_key] == current_session.key + assert service._http_bridge_previous_response_index[response_alias_key] == current_session.key + + +def test_codex_prewarm_eligibility_is_enabled_flag_alone() -> None: + assert proxy_service._http_bridge_prewarm_enabled( + _make_app_settings(http_responses_session_bridge_codex_prewarm_enabled=True) + ) + assert not proxy_service._http_bridge_prewarm_enabled(_make_app_settings()) @pytest.mark.asyncio -async def test_http_bridge_gate_contention_retry_fails_fast_when_queue_full( +async def test_maybe_prewarm_http_bridge_session_not_applicable_when_disabled( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="sid-gate-queue-full", queued_request_count=4) - request_state = proxy_service._WebSocketRequestState( - request_id="req-gate-queue-full", - model="gpt-5.4", + state = proxy_service._WebSocketRequestState( + request_id="req-prewarm-disabled", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), + started_at=1.0, + request_text=json.dumps({"input": "x" * 50000}), transport="http", - event_queue=asyncio.Queue(), ) - gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( - "http_bridge_response_create_gate", - code="response_create_gate_timeout", + session = _make_bridge_session() + session.codex_session = True + session.last_used_at = -180.0 + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings(), ) - submit = AsyncMock(side_effect=gate_timeout_error) - settings = _make_app_settings() - monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) - monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - with pytest.raises(ProxyResponseError) as exc_info: - async for _ in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data='{"type":"response.create"}', - queue_limit=4, - propagate_http_errors=True, - downstream_turn_state=None, - ): - pass + await service._maybe_prewarm_http_bridge_session( + session, + request_state=state, + text_data=state.request_text or "{}", + ) - # A sleeping gate waiter must occupy a queue slot; at the limit the - # retry fails fast instead of accumulating unbounded waiters. - assert exc_info.value.payload["error"]["code"] == "bridge_queue_full" - assert submit.await_count == 1 - assert session.queued_request_count == 4 + assert state.prewarm_status == "not_applicable" + assert session.prewarmed is False @pytest.mark.asyncio -async def test_http_bridge_gate_contention_retry_balances_queue_slot( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="sid-gate-slot-balance", queued_request_count=1) +async def test_http_bridge_activity_snapshot_counts_pending_and_inflight_sessions(): + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) request_state = proxy_service._WebSocketRequestState( - request_id="req-gate-slot-balance", - model="gpt-5.4", + request_id="req-drain-status", + model="gpt-5.5", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), + started_at=1.0, + response_id=None, + awaiting_response_created=True, + event_queue=None, transport="http", - event_queue=asyncio.Queue(), - ) - assert request_state.event_queue is not None - request_state.event_queue.put_nowait( - 'data: {"type":"response.completed","response":{"id":"resp_gate_slot_balance"}}\n\n' + skip_request_log=True, ) - request_state.event_queue.put_nowait(None) - gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( - "http_bridge_response_create_gate", - code="response_create_gate_timeout", + session = _make_bridge_session( + pending_requests=deque([request_state]), + queued_request_count=2, ) - submit = AsyncMock(side_effect=[gate_timeout_error, None]) - detach = AsyncMock() - settings = _make_app_settings() - monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) - monkeypatch.setattr(http_bridge_streaming_module, "_RESPONSE_CREATE_GATE_RETRY_SLEEP_SECONDS", 0.001) - monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 0.001) - monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - monkeypatch.setattr(service, "_detach_http_bridge_request", detach) + service._http_bridge_sessions[session.key] = session + service._http_bridge_inflight_sessions[ + proxy_service._HTTPBridgeSessionKey("session_header", "inflight-drain-status", None) + ] = asyncio.Future() - chunks = [ - chunk - async for chunk in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data='{"type":"response.create"}', - queue_limit=4, - propagate_http_errors=False, - downstream_turn_state=None, - ) - ] + snapshot = service.http_bridge_activity_snapshot_nowait() - assert any("response.completed" in chunk for chunk in chunks) - assert submit.await_count == 2 - # The temporary sleep-slot is released after each retry. - assert session.queued_request_count == 1 + assert snapshot == { + "http_bridge_live_sessions": 1, + "http_bridge_pending_or_queued_requests": 2, + "http_bridge_pending_unknown_sessions": 0, + "http_bridge_inflight_session_creates": 1, + "http_bridge_inflight_session_create_oldest_age_seconds": 0, + "http_bridge_stale_inflight_session_creates": 0, + "http_bridge_cleaned_inflight_session_creates": 0, + "http_bridge_background_cleanup_tasks": 0, + "http_bridge_active": True, + "http_bridge_restart_blocking": True, + } @pytest.mark.asyncio -async def test_http_bridge_gate_contention_does_not_retry_retired_session( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="sid-gate-retired") +async def test_http_bridge_activity_snapshot_counts_closed_admission_waiter_as_restart_blocking() -> None: + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + session = _make_bridge_session(queued_request_count=1) session.closed = True - request_state = proxy_service._WebSocketRequestState( - request_id="req-gate-retired", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - transport="http", - event_queue=asyncio.Queue(), - ) - gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( - "http_bridge_response_create_gate", - code="response_create_gate_timeout", - ) - submit = AsyncMock(side_effect=gate_timeout_error) - settings = _make_app_settings() - monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) - monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + session.admission_waiter_count = 1 + service._http_bridge_sessions[session.key] = session - with pytest.raises(ProxyResponseError) as exc_info: - async for _ in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data='{"type":"response.create"}', - queue_limit=4, - propagate_http_errors=True, - downstream_turn_state=None, - ): - pass + snapshot = service.http_bridge_activity_snapshot_nowait() - # A gate timeout that retired the session must fail startup cleanly - # instead of retrying the closed session mid-stream. - assert exc_info.value is gate_timeout_error - assert submit.await_count == 1 + assert snapshot["http_bridge_live_sessions"] == 0 + assert snapshot["http_bridge_pending_or_queued_requests"] == 1 + assert snapshot["http_bridge_active"] is True + assert snapshot["http_bridge_restart_blocking"] is True -@pytest.mark.parametrize( - ("unsafe_state", "unsafe_value"), - [ - ("response_id", "resp-already-created"), - ("response_event_count", 1), - ("last_downstream_sequence_number", 0), - ("downstream_visible", True), - ("awaiting_response_created", True), - ("response_create_gate_acquired", True), - ], -) -def test_http_bridge_retired_gate_replacement_requires_unsubmitted_waiter( - unsafe_state: str, - unsafe_value: object, -) -> None: - session = _make_bridge_session(key_value="sid-gate-replacement-guard") +@pytest.mark.asyncio +async def test_http_bridge_activity_snapshot_counts_pending_closed_detached_generation() -> None: + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + session = _make_bridge_session(queued_request_count=1) session.closed = True - request_state = proxy_service._WebSocketRequestState( - request_id="req-gate-replacement-guard", - model="gpt-5.4", + service._http_bridge_detached_sessions[id(session)] = session + + snapshot = service.http_bridge_activity_snapshot_nowait() + + assert snapshot["http_bridge_live_sessions"] == 0 + assert snapshot["http_bridge_pending_or_queued_requests"] == 1 + assert snapshot["http_bridge_active"] is True + assert snapshot["http_bridge_restart_blocking"] is True + + +@pytest.mark.asyncio +async def test_response_create_gate_timeout_retires_old_pending_without_upstream_event( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = _make_app_settings( + proxy_admission_wait_timeout_seconds=0.001, + http_responses_session_bridge_stuck_gate_retire_after_seconds=300.0, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + session = _make_bridge_session() + service._http_bridge_sessions[session.key] = session + await session.response_create_gate.acquire() + old_pending = proxy_service._WebSocketRequestState( + request_id="req-old-pending", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), + started_at=time.monotonic() - 301.0, transport="http", - request_text='{"type":"response.create"}', - event_queue=asyncio.Queue(), - ) - setattr(request_state, unsafe_state, unsafe_value) - gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( - "http_bridge_response_create_gate", - code="response_create_gate_timeout", - ) - - assert not http_bridge_streaming_module._http_bridge_can_replace_retired_gate_session( - gate_timeout_error, - session=session, - request_state=request_state, - request_was_enqueued=False, + response_create_gate_acquired=True, + awaiting_response_created=True, + # A downstream keepalive is not evidence that upstream created a + # response; this request is still safe to retire after the stale window. + downstream_visible=True, ) - - -def test_http_bridge_retired_gate_replacement_accepts_cleaned_hard_affinity_waiter() -> None: - session = _make_bridge_session(key_value="sid-gate-replacement-safe") - session.closed = True - request_state = proxy_service._WebSocketRequestState( - request_id="req-gate-replacement-safe", - model="gpt-5.4", + waiter = proxy_service._WebSocketRequestState( + request_id="req-visible-waiter", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), transport="http", - request_text='{"type":"response.create"}', - event_queue=asyncio.Queue(), - ) - gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( - "http_bridge_response_create_gate", - code="response_create_gate_timeout", + downstream_visible=True, ) + async with session.pending_lock: + session.pending_requests.append(old_pending) + session.queued_request_count = 1 - assert http_bridge_streaming_module._http_bridge_can_replace_retired_gate_session( - gate_timeout_error, - session=session, - request_state=request_state, - request_was_enqueued=False, - ) - assert not http_bridge_streaming_module._http_bridge_can_replace_retired_gate_session( - gate_timeout_error, - session=session, - request_state=request_state, - request_was_enqueued=True, - ) - session.key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "soft-gate-replacement", None) - assert not http_bridge_streaming_module._http_bridge_can_replace_retired_gate_session( - gate_timeout_error, - session=session, - request_state=request_state, - request_was_enqueued=False, - ) + retire_calls: list[str] = [] + + async def fake_retire( + retire_session: proxy_service._HTTPBridgeSession, + *, + detail: str, + retry_circuit_attempt_selection: proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, + ) -> None: + assert retry_circuit_attempt_selection.kind == "absent" + retire_calls.append(detail) + retire_session.closed = True + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", fake_retire) -def test_http_bridge_retired_gate_replacement_ignores_replay_count() -> None: - """A non-zero replay_count reflects the client's own reconnect attempts, - not upstream progress on this bridge attempt, so an otherwise fully - unsubmitted waiter must still be replaceable.""" - session = _make_bridge_session(key_value="sid-gate-replacement-replayed") - session.closed = True - request_state = proxy_service._WebSocketRequestState( - request_id="req-gate-replacement-replayed", - model="gpt-5.4", + try: + with pytest.raises(ProxyResponseError) as exc_info: + await service._acquire_request_state_response_create_admission( + waiter, + response_create_gate=session.response_create_gate, + account_id=session.account.id, + surface="http_bridge", + bridge_session=session, + ) + finally: + if session.response_create_gate.locked(): + session.response_create_gate.release() + + assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" + assert retire_calls == ["response_create_gate_timeout_stuck_pending"] + assert session.closed is True + assert waiter.response_create_gate is None + assert waiter.response_create_gate_acquired is False + + +def test_stale_gate_cleanup_keeps_draining_sibling_active() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + now = time.monotonic() + stale = proxy_service._WebSocketRequestState( + request_id="req-stale-gate-holder", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), + started_at=now - 301.0, transport="http", - request_text='{"type":"response.create"}', - event_queue=asyncio.Queue(), - replay_count=1, ) - gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( - "http_bridge_response_create_gate", - code="response_create_gate_timeout", + draining = proxy_service._WebSocketRequestState( + request_id="req-draining-terminal-sibling", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=now - 301.0, + transport="http", + draining_until_terminal=True, ) - assert http_bridge_streaming_module._http_bridge_can_replace_retired_gate_session( - gate_timeout_error, - session=session, - request_state=request_state, - request_was_enqueued=False, + stale_states, should_retire = service._classify_http_bridge_stale_gate_holders( + [stale, draining], + now=now, + threshold_seconds=300.0, + session_closed=False, ) + assert stale_states == [stale] + assert should_retire is False + @pytest.mark.asyncio -async def test_http_bridge_submit_gate_contention_still_reroutes_soft_sessions( +async def test_response_create_gate_timeout_retires_closed_anchored_pending_without_upstream_event( monkeypatch: pytest.MonkeyPatch, ) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="soft-submit-gate-contention") - session.key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "soft-submit-gate-contention", None) - request_state = proxy_service._WebSocketRequestState( - request_id="req-soft-submit-gate-contention", - model="gpt-5.4", + settings = _make_app_settings( + proxy_admission_wait_timeout_seconds=0.001, + http_responses_session_bridge_stuck_gate_retire_after_seconds=300.0, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + session = _make_bridge_session() + session.closed = True + service._http_bridge_sessions[session.key] = session + await session.response_create_gate.acquire() + old_pending = proxy_service._WebSocketRequestState( + request_id="req-closed-anchored-pending", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), + started_at=time.monotonic() - 301.0, transport="http", - bridge_soft_capacity_reroute_allowed=True, - ) - gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( - "http_bridge_response_create_gate", - code="response_create_gate_timeout", + previous_response_id="resp-anchored", + hard_continuity_anchor=True, + response_create_gate_acquired=True, + awaiting_response_created=True, ) - submit = AsyncMock(side_effect=gate_timeout_error) - monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - - with pytest.raises(ProxyResponseError) as exc_info: - async for _ in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data='{"type":"response.create"}', - queue_limit=4, - propagate_http_errors=True, - downstream_turn_state=None, - ): - pass - - assert exc_info.value is gate_timeout_error - assert submit.await_count == 1 - - -@pytest.mark.asyncio -async def test_http_bridge_submit_leaves_soft_capacity_for_session_reroute(monkeypatch: pytest.MonkeyPatch) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="soft-submit-capacity") - session.key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "soft-submit-capacity", None) - request_state = proxy_service._WebSocketRequestState( - request_id="req-soft-submit-capacity", - model="gpt-5.4", + waiter = proxy_service._WebSocketRequestState( + request_id="req-closed-anchored-waiter", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), transport="http", - bridge_soft_capacity_reroute_allowed=True, - ) - capacity_error = ProxyResponseError( - 429, - openai_error( - "account_response_create_cap", - "Account response-create concurrency limit reached", - error_type="rate_limit_error", - ), + downstream_visible=True, ) - submit = AsyncMock(side_effect=capacity_error) - monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + async with session.pending_lock: + session.pending_requests.append(old_pending) + session.queued_request_count = 1 - with pytest.raises(ProxyResponseError) as exc_info: - async for _ in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data='{"type":"response.create"}', - queue_limit=4, - propagate_http_errors=True, - downstream_turn_state=None, - ): - pass + retire_calls: list[str] = [] - assert exc_info.value is capacity_error - assert submit.await_count == 1 + async def fake_retire( + retire_session: proxy_service._HTTPBridgeSession, + *, + detail: str, + retry_circuit_attempt_selection: proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, + ) -> None: + assert retry_circuit_attempt_selection.kind == "absent" + retire_calls.append(detail) + retire_session.closed = True + + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", fake_retire) + + try: + with pytest.raises(ProxyResponseError) as exc_info: + await service._acquire_request_state_response_create_admission( + waiter, + response_create_gate=session.response_create_gate, + account_id=session.account.id, + surface="http_bridge", + bridge_session=session, + ) + finally: + if session.response_create_gate.locked(): + session.response_create_gate.release() + + assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" + assert retire_calls == ["response_create_gate_timeout_stuck_pending"] @pytest.mark.asyncio -async def test_http_bridge_submit_capacity_wait_uses_original_request_deadline( +async def test_response_create_gate_timeout_retires_old_precreated_request_after_rate_limit_telemetry( monkeypatch: pytest.MonkeyPatch, ) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="sid-submit-original-deadline") - request_state = proxy_service._WebSocketRequestState( - request_id="req-submit-original-deadline", - model="gpt-5.4", + settings = _make_app_settings( + proxy_admission_wait_timeout_seconds=0.001, + http_responses_session_bridge_stuck_gate_retire_after_seconds=300.0, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + session = _make_bridge_session() + service._http_bridge_sessions[session.key] = session + await session.response_create_gate.acquire() + old_pending = proxy_service._WebSocketRequestState( + request_id="req-old-pending-after-telemetry", + model="gpt-5.6-sol", service_tier=None, - reasoning_effort=None, + reasoning_effort="high", api_key_reservation=None, - started_at=99.5, + started_at=time.monotonic() - 301.0, transport="http", + response_create_gate=session.response_create_gate, + response_create_gate_acquired=True, + awaiting_response_created=True, + downstream_visible=False, event_queue=asyncio.Queue(), ) - capacity_error = ProxyResponseError( - 429, - openai_error( - "account_response_create_cap", - "Account response-create concurrency limit reached", - error_type="rate_limit_error", + waiter = proxy_service._WebSocketRequestState( + request_id="req-visible-waiter-after-telemetry", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + downstream_visible=True, + ) + async with session.pending_lock: + session.pending_requests.append(old_pending) + session.queued_request_count = 1 + + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "codex.rate_limits", + "plan_type": "pro", + "rate_limits": {"allowed": True, "limit_reached": False}, + }, + separators=(",", ":"), ), ) - submit = AsyncMock(side_effect=capacity_error) - clock = [100.0] - waited: list[float] = [] - async def fake_capacity_wait(**kwargs: object): - waited.append(cast(float, kwargs["sleep_seconds"])) - clock[0] += waited[-1] - if False: - yield "" - - monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 30.0) - monkeypatch.setattr(http_bridge_streaming_module, "_iter_account_capacity_wait_sse", fake_capacity_wait) - monkeypatch.setattr( - http_bridge_streaming_module, - "_service_time", - lambda: SimpleNamespace(monotonic=lambda: clock[0]), - ) - - with pytest.raises(ProxyResponseError) as exc_info: - async for _ in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data='{"type":"response.create"}', - queue_limit=4, - propagate_http_errors=True, - downstream_turn_state=None, - request_deadline=101.0, - ): - pass - - assert exc_info.value is capacity_error - assert waited == [1.0] - assert submit.await_count == 1 + assert old_pending.latency_first_upstream_event_ms is not None + assert old_pending.latency_response_created_ms is None + assert old_pending.awaiting_response_created is True + assert old_pending.response_id is None + assert old_pending.downstream_visible is False + assert session.response_create_gate.locked() is True + retire_calls: list[str] = [] -def _make_api_key( - *, - key_id: str, - assigned_account_ids: list[str], - account_assignment_scope_enabled: bool | None = None, -) -> proxy_service.ApiKeyData: - return proxy_service.ApiKeyData( - id=key_id, - name="bridge-key", - key_prefix="sk-bridge", - allowed_models=None, - enforced_model=None, - enforced_reasoning_effort=None, - enforced_service_tier=None, - expires_at=None, - is_active=True, - created_at=datetime(2025, 1, 1, tzinfo=timezone.utc), - last_used_at=None, - account_assignment_scope_enabled=( - bool(assigned_account_ids) if account_assignment_scope_enabled is None else account_assignment_scope_enabled - ), - assigned_account_ids=assigned_account_ids, - ) + async def fake_retire( + retire_session: proxy_service._HTTPBridgeSession, + *, + detail: str, + retry_circuit_attempt_selection: proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, + ) -> None: + assert retry_circuit_attempt_selection.kind == "absent" + retire_calls.append(detail) + retire_session.closed = True + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", fake_retire) -def test_http_bridge_request_budget_falls_back_to_proxy_budget() -> None: - settings = SimpleNamespace(proxy_request_budget_seconds=42.5) + try: + with pytest.raises(ProxyResponseError) as exc_info: + await service._acquire_request_state_response_create_admission( + waiter, + response_create_gate=session.response_create_gate, + account_id=session.account.id, + surface="http_bridge", + bridge_session=session, + ) + finally: + if session.response_create_gate.locked(): + session.response_create_gate.release() - assert http_bridge_streaming_module._http_bridge_request_budget_seconds(settings) == 42.5 + assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" + assert retire_calls == ["response_create_gate_timeout_stuck_pending"] + assert session.closed is True -def test_websocket_top_level_error_payload_uses_error_type_not_event_type() -> None: - payload: dict[str, proxy_service.JsonValue] = { - "type": "error", - "status": 400, - "error_type": "invalid_request_error", - "code": "previous_response_not_found", - "message": "Previous response with id 'resp_missing' not found.", - "param": "previous_response_id", - } +@pytest.mark.asyncio +@pytest.mark.parametrize( + ( + "awaiting_response_created", + "downstream_visible", + "latency_first_upstream_event_ms", + "latency_response_created_ms", + "response_event_count", + ), + [ + (False, True, 100, 100, 1), + ], +) +async def test_response_create_gate_timeout_does_not_retire_active_response_progress( + monkeypatch: pytest.MonkeyPatch, + awaiting_response_created: bool, + downstream_visible: bool, + latency_first_upstream_event_ms: int, + latency_response_created_ms: int | None, + response_event_count: int, +) -> None: + settings = _make_app_settings( + proxy_admission_wait_timeout_seconds=0.001, + http_responses_session_bridge_stuck_gate_retire_after_seconds=300.0, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + session = _make_bridge_session() + service._http_bridge_sessions[session.key] = session + await session.response_create_gate.acquire() + active_stream = proxy_service._WebSocketRequestState( + request_id="req-active-visible", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic() - 301.0, + transport="http", + response_create_gate_acquired=True, + awaiting_response_created=awaiting_response_created, + downstream_visible=downstream_visible, + latency_first_upstream_event_ms=latency_first_upstream_event_ms, + latency_response_created_ms=latency_response_created_ms, + response_event_count=response_event_count, + ) + stale_pending = proxy_service._WebSocketRequestState( + request_id="req-stale-unanchored", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic() - 301.0, + transport="http", + response_create_gate_acquired=True, + awaiting_response_created=True, + ) + waiter = proxy_service._WebSocketRequestState( + request_id="req-visible-waiter", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + downstream_visible=True, + ) + async with session.pending_lock: + session.pending_requests.append(active_stream) + session.pending_requests.append(stale_pending) + session.queued_request_count = 2 - error = proxy_service._websocket_event_error_payload("error", payload) + retire_calls: list[str] = [] - assert error == { - "type": "invalid_request_error", - "code": "previous_response_not_found", - "message": "Previous response with id 'resp_missing' not found.", - "param": "previous_response_id", - } - assert proxy_service._websocket_event_error_type("error", payload) == "invalid_request_error" - assert proxy_service._websocket_event_error_code("error", payload) == "previous_response_not_found" + async def fake_retire( + retire_session: proxy_service._HTTPBridgeSession, + *, + detail: str, + ) -> None: + retire_calls.append(detail) + retire_session.closed = True + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", fake_retire) -def test_http_error_status_from_payload_accepts_official_status_code_alias() -> None: - payload: dict[str, proxy_service.JsonValue] = { - "type": "error", - "status_code": 400, - "error": {"message": "bad request"}, - } + try: + with pytest.raises(ProxyResponseError) as exc_info: + await service._acquire_request_state_response_create_admission( + waiter, + response_create_gate=session.response_create_gate, + account_id=session.account.id, + surface="http_bridge", + bridge_session=session, + ) + finally: + if session.response_create_gate.locked(): + session.response_create_gate.release() - assert proxy_service._http_error_status_from_payload(payload) == 400 + assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" + assert retire_calls == [] + assert session.closed is False -def test_durable_tool_call_manifest_requires_complete_added_and_done_lifecycle() -> None: - state = proxy_service._WebSocketRequestState( - request_id="req-manifest", - model="gpt-5.6-sol", +def test_http_bridge_pending_state_with_recent_events_but_no_created_is_not_stale() -> None: + # Reattached streams can deliver events whose response.created was lost + # (observed events=54, created=None in prod on 2026-07-20). Recent upstream + # activity must keep the stream alive while the create gate remains held. + request_state = proxy_service._WebSocketRequestState( + request_id="req-events-no-created", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, + started_at=time.monotonic() - 301.0, + transport="http", + response_create_gate_acquired=True, + awaiting_response_created=True, + latency_first_upstream_event_ms=25, + response_event_count=54, + last_upstream_activity_at=time.monotonic(), ) - def record(event_type: str, call_id: str) -> None: - http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( - state, - event_type=event_type, - payload={ - "type": event_type, - "item": { - "type": "function_call", - "call_id": call_id, - "name": "lookup", - "arguments": "{}", - }, - }, - ) - - record("response.output_item.added", "call_1") - record("response.output_item.added", "call_2") - record("response.output_item.done", "call_1") - assert ( - http_bridge_upstream_events_module._durable_pending_tool_call_manifest( - state, - {"type": "response.completed", "response": {"output": []}}, + http_bridge_helpers_module._http_bridge_pending_state_is_stale( + request_state, + now=time.monotonic(), + threshold_seconds=300.0, ) - is None + is False ) - malformed_state = proxy_service._WebSocketRequestState( - request_id="req-malformed-manifest", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - ) - http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( - malformed_state, - event_type="response.output_item.added", - payload={ - "type": "response.output_item.added", - "item": {"type": [], "call_id": "call_malformed"}, - }, + request_state.last_upstream_activity_at = time.monotonic() - 301.0 + assert ( + http_bridge_helpers_module._http_bridge_pending_state_is_stale( + request_state, + now=time.monotonic(), + threshold_seconds=300.0, + ) + is True ) - assert malformed_state.tool_call_manifest_invalid is True - missing_item_state = proxy_service._WebSocketRequestState( - request_id="req-missing-item-manifest", - model="gpt-5.6-sol", + +def test_http_bridge_pending_state_with_first_event_latency_only_is_stale() -> None: + request_state = proxy_service._WebSocketRequestState( + request_id="req-sparse-active", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, - ) - http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( - missing_item_state, - event_type="response.output_item.done", - payload={"type": "response.output_item.done"}, + started_at=time.monotonic() - 301.0, + transport="http", + response_create_gate_acquired=True, + awaiting_response_created=True, + latency_first_upstream_event_ms=10, + downstream_visible=False, ) - assert missing_item_state.tool_call_manifest_invalid is True + assert ( - http_bridge_upstream_events_module._durable_pending_tool_call_manifest( - malformed_state, - {"type": "response.completed", "response": {"output": [{"type": []}]}}, + http_bridge_helpers_module._http_bridge_pending_state_is_stale( + request_state, + now=time.monotonic(), + threshold_seconds=300.0, ) - is None + is True ) - record("response.output_item.done", "call_2") - assert http_bridge_upstream_events_module._durable_pending_tool_call_manifest( - state, - {"type": "response.completed", "response": {"output": []}}, - ) == {"call_1": "function_call", "call_2": "function_call"} - duplicate_state = proxy_service._WebSocketRequestState( - request_id="req-duplicate-manifest", - model="gpt-5.6-sol", +@pytest.mark.parametrize( + ("previous_response_id", "session_id", "hard_continuity_anchor"), + [ + ("resp-anchored", None, False), + (None, "turn-anchored", True), + ], +) +def test_http_bridge_pending_state_with_continuity_anchor_is_not_stale( + previous_response_id: str | None, + session_id: str | None, + hard_continuity_anchor: bool, +) -> None: + request_state = proxy_service._WebSocketRequestState( + request_id="req-anchored-pending", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, - ) - duplicate_event: dict[str, proxy_service.JsonValue] = { - "type": "response.output_item.added", - "item": { - "type": "function_call", - "call_id": "call_duplicate", - "name": "lookup", - "arguments": "{}", - }, - } - http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( - duplicate_state, - event_type="response.output_item.added", - payload=duplicate_event, + started_at=time.monotonic() - 301.0, + transport="http", + response_create_gate_acquired=True, + awaiting_response_created=True, + previous_response_id=previous_response_id, + session_id=session_id, + hard_continuity_anchor=hard_continuity_anchor, ) - http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( - duplicate_state, - event_type="response.output_item.added", - payload=duplicate_event, + + assert ( + http_bridge_helpers_module._http_bridge_pending_state_is_stale( + request_state, + now=time.monotonic(), + threshold_seconds=300.0, + ) + is False ) - assert duplicate_state.tool_call_manifest_invalid is True - duplicate_done_state = proxy_service._WebSocketRequestState( - request_id="req-duplicate-done-manifest", - model="gpt-5.6-sol", + +@pytest.mark.parametrize( + ("previous_response_id", "session_id", "hard_continuity_anchor"), + [ + ("resp-anchored-silent", None, False), + (None, "turn-anchored-silent", True), + ], +) +def test_http_bridge_pending_state_with_continuity_anchor_is_stale_after_extended_silence( + previous_response_id: str | None, + session_id: str | None, + hard_continuity_anchor: bool, +) -> None: + request_state = proxy_service._WebSocketRequestState( + request_id="req-anchored-silent-pending", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, + started_at=time.monotonic() - 601.0, + transport="http", + response_create_gate_acquired=True, + awaiting_response_created=True, + previous_response_id=previous_response_id, + session_id=session_id, + hard_continuity_anchor=hard_continuity_anchor, ) - duplicate_done_event = { - **duplicate_event, - "type": "response.output_item.done", - } - for event_type, event_payload in ( - ("response.output_item.added", duplicate_event), - ("response.output_item.done", duplicate_done_event), - ("response.output_item.done", duplicate_done_event), - ): - http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( - duplicate_done_state, - event_type=event_type, - payload=event_payload, + + assert ( + http_bridge_helpers_module._http_bridge_pending_state_is_stale( + request_state, + now=time.monotonic(), + threshold_seconds=300.0, ) - assert duplicate_done_state.tool_call_manifest_invalid is True + is True + ) -@pytest.mark.asyncio -async def test_http_bridge_backfills_transition_manifest_from_streamed_output_items( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize( + ("previous_response_id", "session_id", "hard_continuity_anchor"), + [ + ("resp-closed-anchored", None, False), + (None, "turn-closed-anchored", True), + ], +) +def test_http_bridge_closed_session_pending_anchor_is_stale( + previous_response_id: str | None, + session_id: str | None, + hard_continuity_anchor: bool, ) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - register_previous = AsyncMock(return_value=True) - monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", register_previous) - monkeypatch.setattr(service, "_finalize_websocket_request_state", AsyncMock()) request_state = proxy_service._WebSocketRequestState( - request_id="req-streamed-transition-manifest", - response_id="resp_streamed_transition_manifest", - model="gpt-5.6-sol", + request_id="req-closed-anchored-pending", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, + started_at=time.monotonic() - 301.0, transport="http", - skip_request_log=True, - ) - session = _make_bridge_session( - key_value="bridge-streamed-transition-manifest", - pending_requests=deque([request_state]), - queued_request_count=1, + response_create_gate_acquired=True, + awaiting_response_created=True, + previous_response_id=previous_response_id, + session_id=session_id, + hard_continuity_anchor=hard_continuity_anchor, ) - output_items: list[dict[str, proxy_service.JsonValue]] = [ - { - "type": "reasoning", - "id": "rs_streamed_transition_manifest", - "summary": [], - "status": "completed", - }, - { - "type": "custom_tool_call", - "id": "ctc_streamed_transition_manifest", - "call_id": "call_streamed_transition_manifest", - "name": "shell", - "input": "content-not-persisted", - "status": "completed", - }, - ] - for output_index, item in enumerate(output_items): - for event_type in ("response.output_item.added", "response.output_item.done"): - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": event_type, - "response_id": "resp_streamed_transition_manifest", - "output_index": output_index, - "item": item, - }, - separators=(",", ":"), - ), - ) - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "response.completed", - "response": { - "id": "resp_streamed_transition_manifest", - "object": "response", - "status": "completed", - "output": [], - }, - }, - separators=(",", ":"), - ), + assert ( + http_bridge_helpers_module._http_bridge_pending_state_is_stale( + request_state, + now=time.monotonic(), + threshold_seconds=300.0, + session_closed=True, + ) + is True ) - registration = register_previous.await_args - assert registration is not None - assert registration.kwargs["pending_tool_calls"] == {"call_streamed_transition_manifest": "custom_tool_call"} - manifest = registration.kwargs["response_transition_manifest"] - assert manifest is not None - assert manifest.item_kinds == ("reasoning", "custom_tool_call") - assert "content-not-persisted" not in str(manifest.canonical_payload()) - -def test_http_bridge_transition_manifest_collection_fails_closed_on_missing_index() -> None: - state = proxy_service._WebSocketRequestState( - request_id="req-invalid-streamed-transition-manifest", - model="gpt-5.6-sol", +def test_http_bridge_pending_state_with_plain_session_header_is_stale() -> None: + request_state = proxy_service._WebSocketRequestState( + request_id="req-session-header-pending", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, - ) - http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( - state, - event_type="response.output_item.done", - payload={ - "type": "response.output_item.done", - "item": {"type": "reasoning", "summary": []}, - }, + started_at=time.monotonic() - 301.0, + transport="http", + session_id="session-header-only", + response_create_gate_acquired=True, + awaiting_response_created=True, ) - assert state.response_output_items_invalid is True assert ( - http_bridge_upstream_events_module._response_transition_payload( - state, - { - "type": "response.completed", - "response": { - "id": "resp_invalid_streamed_transition_manifest", - "status": "completed", - "output": [], - }, - }, + http_bridge_helpers_module._http_bridge_pending_state_is_stale( + request_state, + now=time.monotonic(), + threshold_seconds=300.0, ) - is None + is True ) -def test_durable_tool_call_manifest_rejects_unobserved_terminal_call() -> None: - state = proxy_service._WebSocketRequestState( - request_id="req-terminal-manifest", - model="gpt-5.6-sol", +def test_http_bridge_synthesized_downstream_turn_state_is_not_hard_anchor() -> None: + request_state = proxy_service._WebSocketRequestState( + request_id="req-synth-turn-state", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, + started_at=time.monotonic(), + transport="http", ) - assert ( - http_bridge_upstream_events_module._durable_pending_tool_call_manifest( - state, - { - "type": "response.completed", - "response": { - "output": [ - { - "type": "function_call", - "call_id": "call_unobserved", - "name": "lookup", - "arguments": "{}", - } - ] - }, - }, - ) - is None + http_bridge_streaming_module._apply_http_bridge_downstream_turn_state( + request_state, + downstream_turn_state="synthesized-turn-state", + incoming_turn_state_header=None, ) - state.added_tool_call_types = {"call_duplicate": "function_call"} - state.pending_tool_call_types = {"call_duplicate": "function_call"} - duplicate_terminal_call: dict[str, proxy_service.JsonValue] = { - "type": "function_call", - "call_id": "call_duplicate", - "name": "lookup", - "arguments": "{}", - } - assert ( - http_bridge_upstream_events_module._durable_pending_tool_call_manifest( - state, - { - "type": "response.completed", - "response": {"output": [duplicate_terminal_call, duplicate_terminal_call]}, - }, - ) - is None - ) + assert request_state.session_id == "synthesized-turn-state" + assert request_state.hard_continuity_anchor is False -def test_live_tool_call_manifest_rejects_added_only_but_allows_done_only() -> None: - completed_payload: dict[str, proxy_service.JsonValue] = { - "type": "response.completed", - "response": {"output": []}, - } - added_only = proxy_service._WebSocketRequestState( - request_id="req-live-added-only", - model="gpt-5.6-sol", +@pytest.mark.parametrize( + ("incoming_turn_state_header", "previous_response_id"), + [ + ("client-turn-state", None), + (None, "resp-continuation"), + ], +) +def test_http_bridge_real_continuity_sets_hard_anchor( + incoming_turn_state_header: str | None, + previous_response_id: str | None, +) -> None: + request_state = proxy_service._WebSocketRequestState( + request_id="req-real-anchor", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, - ) - added_only.added_tool_call_types = {"call_added": "function_call"} - assert ( - http_bridge_upstream_events_module._live_pending_tool_call_manifest_is_invalid( - added_only, - completed_payload, - ) - is True + started_at=time.monotonic(), + transport="http", + previous_response_id=previous_response_id, ) - done_only = proxy_service._WebSocketRequestState( - request_id="req-live-done-only", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - ) - done_only.pending_tool_call_types = {"call_done": "custom_tool_call"} - assert ( - http_bridge_upstream_events_module._live_pending_tool_call_manifest_is_invalid( - done_only, - completed_payload, - ) - is False + http_bridge_streaming_module._apply_http_bridge_downstream_turn_state( + request_state, + downstream_turn_state="real-turn-state", + incoming_turn_state_header=incoming_turn_state_header, ) + assert request_state.session_id == "real-turn-state" + assert request_state.hard_continuity_anchor is True -def test_durable_tool_call_manifest_rejects_mixed_client_settled_call_types() -> None: - state = proxy_service._WebSocketRequestState( - request_id="req-mixed-client-settled-manifest", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - ) - function_item: dict[str, proxy_service.JsonValue] = { - "type": "function_call", - "call_id": "call_function", - "name": "lookup", - "arguments": "{}", - } - for event_type in ("response.output_item.added", "response.output_item.done"): - http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( - state, - event_type=event_type, - payload={"type": event_type, "item": function_item}, - ) - assert ( - http_bridge_upstream_events_module._durable_pending_tool_call_manifest( - state, - { - "type": "response.completed", - "response": { - "output": [ - function_item, - { - "type": "computer_call", - "call_id": "call_computer", - "action": {"type": "screenshot"}, - }, - ] - }, - }, - ) - is None - ) +@pytest.mark.asyncio +async def test_http_bridge_activity_snapshot_counts_only_bridge_cleanup_tasks(): + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + bridge_task = asyncio.create_task(asyncio.sleep(60), name="proxy-http_bridge_session_close-test") + api_key_task = asyncio.create_task(asyncio.sleep(60), name="proxy-stream-api-key-settle-test") + service._background_cleanup_tasks.update({bridge_task, api_key_task}) - lifecycle_state = proxy_service._WebSocketRequestState( - request_id="req-unsupported-lifecycle-manifest", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - ) - http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( - lifecycle_state, - event_type="response.output_item.added", - payload={ - "type": "response.output_item.added", - "item": { - "type": "mcp_approval_request", - "id": "approval_1", - }, - }, - ) - assert lifecycle_state.tool_call_manifest_invalid is True + try: + snapshot = service.http_bridge_activity_snapshot_nowait() + finally: + bridge_task.cancel() + api_key_task.cancel() + await asyncio.gather(bridge_task, api_key_task, return_exceptions=True) + + assert snapshot["http_bridge_background_cleanup_tasks"] == 1 @pytest.mark.asyncio -async def test_http_bridge_malformed_tool_lifecycle_persists_unknown_manifest( +async def test_http_bridge_activity_snapshot_cleans_completed_stale_inflight_session( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - register_previous = AsyncMock(return_value=True) - monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", register_previous) - monkeypatch.setattr(service, "_finalize_websocket_request_state", AsyncMock()) - request_state = proxy_service._WebSocketRequestState( - request_id="req-malformed-lifecycle", - response_id="resp_malformed_lifecycle", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - transport="http", - skip_request_log=True, - ) - session = _make_bridge_session( - key_value="bridge-malformed-lifecycle", - pending_requests=deque([request_state]), - queued_request_count=1, - ) - valid_item = { - "type": "function_call", - "call_id": "call_1", - "name": "lookup", - "arguments": "{}", - } + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + key = proxy_service._HTTPBridgeSessionKey("session_header", "stale-inflight-drain-status", None) + inflight_future: asyncio.Future[proxy_service._HTTPBridgeSession] = asyncio.get_running_loop().create_future() + setattr(inflight_future, "_codex_lb_started_at", -1000.0) + inflight_future.set_result(_make_bridge_session()) + service._http_bridge_inflight_sessions[key] = inflight_future - for event in ( - { - "type": "response.output_item.added", - "response_id": "resp_malformed_lifecycle", - "item": valid_item, - }, - { - "type": "response.output_item.done", - "response_id": "resp_malformed_lifecycle", - "item": valid_item, - }, - { - "type": "response.output_item.added", - "response_id": "resp_malformed_lifecycle", - }, - { - "type": "response.completed", - "response": { - "id": "resp_malformed_lifecycle", - "object": "response", - "status": "completed", - "output": [], - }, - }, - ): - await service._process_http_bridge_upstream_text(session, json.dumps(event, separators=(",", ":"))) + monkeypatch.setattr(proxy_service, "_proxy_admission_wait_timeout_seconds", lambda settings=None: 0.001) - registration = register_previous.await_args - assert registration is not None - assert registration.kwargs["pending_tool_calls"] is None - assert session.last_pending_tool_call_manifest_invalid is True - assert session.last_pending_tool_calls == {"call_1": "function_call"} + with caplog.at_level(logging.WARNING, logger="app.modules.proxy.service"): + snapshot = service.http_bridge_activity_snapshot_nowait() + + assert key not in service._http_bridge_inflight_sessions + assert snapshot["http_bridge_inflight_session_creates"] == 0 + assert snapshot["http_bridge_stale_inflight_session_creates"] == 1 + assert snapshot["http_bridge_cleaned_inflight_session_creates"] == 1 + assert snapshot["http_bridge_active"] is False + assert snapshot["http_bridge_restart_blocking"] is False + assert "http_bridge_inflight_session_create_cleanup" in caplog.text - valid_request_state = proxy_service._WebSocketRequestState( - request_id="req-valid-lifecycle-after-invalid", - response_id="resp_valid_lifecycle_after_invalid", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=2.0, - transport="http", - skip_request_log=True, - ) - session.pending_requests.append(valid_request_state) - session.queued_request_count = 1 - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "response.completed", - "response": { - "id": "resp_valid_lifecycle_after_invalid", - "object": "response", - "status": "completed", - "output": [], - }, - }, - separators=(",", ":"), - ), - ) - assert session.last_pending_tool_call_manifest_invalid is False - assert session.last_pending_tool_calls == {} +@pytest.mark.asyncio +async def test_http_bridge_activity_snapshot_does_not_expire_live_inflight_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + key = proxy_service._HTTPBridgeSessionKey("session_header", "live-stale-inflight-drain-status", None) + inflight_future: asyncio.Future[proxy_service._HTTPBridgeSession] = asyncio.get_running_loop().create_future() + setattr(inflight_future, "_codex_lb_started_at", -1000.0) + service._http_bridge_inflight_sessions[key] = inflight_future + + monkeypatch.setattr(proxy_service, "_proxy_admission_wait_timeout_seconds", lambda settings=None: 0.001) + + snapshot = service.http_bridge_activity_snapshot_nowait() + + assert key in service._http_bridge_inflight_sessions + assert not inflight_future.done() + assert snapshot["http_bridge_inflight_session_creates"] == 1 + assert snapshot["http_bridge_stale_inflight_session_creates"] == 1 + assert snapshot["http_bridge_cleaned_inflight_session_creates"] == 0 + assert snapshot["http_bridge_active"] is True + assert snapshot["http_bridge_restart_blocking"] is True -@pytest.mark.parametrize( - ("upstream_code", "expected_retry_error_code"), - [ - ("invalid_request_error", "server_is_overloaded"), - ("rate_limit_exceeded", "rate_limit_exceeded"), - ], -) @pytest.mark.asyncio -async def test_http_bridge_model_capacity_waits_before_precreated_retry( +async def test_http_bridge_activity_snapshot_skips_inflight_cleanup_when_registry_locked( monkeypatch: pytest.MonkeyPatch, - upstream_code: str, - expected_retry_error_code: str, ) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - handle_stream_error = AsyncMock() - retry_precreated = AsyncMock(return_value=True) - monkeypatch.setattr(service, "_handle_stream_error", handle_stream_error) - monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - monkeypatch.setattr( - http_bridge_upstream_events_module, - "_ACCOUNT_SELECTION_RECOVERY_DEFAULT_SLEEP_SECONDS", - 0.001, + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + key = proxy_service._HTTPBridgeSessionKey("session_header", "locked-stale-inflight-drain-status", None) + inflight_future: asyncio.Future[proxy_service._HTTPBridgeSession] = asyncio.get_running_loop().create_future() + setattr(inflight_future, "_codex_lb_started_at", -1000.0) + service._http_bridge_inflight_sessions[key] = inflight_future + + monkeypatch.setattr(proxy_service, "_proxy_admission_wait_timeout_seconds", lambda settings=None: 0.001) + + async with service._http_bridge_lock: + snapshot = service.http_bridge_activity_snapshot_nowait() + + assert key in service._http_bridge_inflight_sessions + assert not inflight_future.done() + assert snapshot["http_bridge_inflight_session_creates"] == 1 + assert snapshot["http_bridge_stale_inflight_session_creates"] == 1 + assert snapshot["http_bridge_cleaned_inflight_session_creates"] == 0 + assert snapshot["http_bridge_active"] is True + assert snapshot["http_bridge_restart_blocking"] is True + + +async def _wait_for_close_await(close_session: AsyncMock, session: proxy_service._HTTPBridgeSession) -> None: + for _ in range(10): + if any(call.args == (session,) for call in close_session.await_args_list): + return + await asyncio.sleep(0) + raise AssertionError("expected HTTP bridge session close to be awaited") + + +def test_http_bridge_account_capacity_wait_treats_workspace_spend_cap_as_recoverable() -> None: + exc = ProxyResponseError( + 429, + openai_error( + "no_accounts", + ( + "You hit your spend cap set by the owner of your workspace. " + "Ask an owner to increase your spend cap to continue." + ), + ), ) - monkeypatch.setattr( - http_bridge_upstream_events_module, - "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", - 0.001, + + assert http_bridge_streaming_module._http_bridge_account_capacity_wait_seconds(exc) == 30.0 + + +def test_http_bridge_account_capacity_wait_honors_upstream_rate_limit_retry_hint() -> None: + exc = ProxyResponseError( + 429, + openai_error( + "rate_limit_exceeded", + "Rate limit exceeded. Try again in 120s", + ), ) - request_state = proxy_service._WebSocketRequestState( - request_id="req-model-capacity-wait", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - bridge_request_deadline=time.monotonic() + 60.0, - awaiting_response_created=True, - event_queue=asyncio.Queue(), - transport="http", - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + assert http_bridge_streaming_module._http_bridge_account_capacity_wait_seconds(exc) == 120.0 + + +def test_http_bridge_account_capacity_wait_ignores_local_no_accounts_retry_hint() -> None: + exc = ProxyResponseError( + 429, + openai_error( + "no_accounts", + "Rate limit exceeded. Try again in 120s", + ), ) - session = _make_bridge_session( - key_value="bridge-model-capacity-wait", - pending_requests=deque([request_state]), - queued_request_count=1, + + assert http_bridge_streaming_module._http_bridge_account_capacity_wait_seconds(exc) is None + + +@pytest.mark.parametrize("error_code", ["account_stream_cap", "account_response_create_cap"]) +def test_http_bridge_account_capacity_wait_treats_local_account_caps_as_recoverable(error_code: str) -> None: + exc = ProxyResponseError( + 429, + openai_error( + error_code, + "Account stream capacity is exhausted; per-account limit is 8.", + ), ) - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": upstream_code, - "message": "Selected model is at capacity. Please try a different model.", - }, - }, - separators=(",", ":"), + assert http_bridge_streaming_module._http_bridge_account_capacity_wait_seconds(exc) == 30.0 + + +def test_http_bridge_account_capacity_wait_treats_gate_timeout_as_recoverable() -> None: + exc = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( + "http_bridge_response_create_gate", + code="response_create_gate_timeout", + ) + + assert ( + http_bridge_streaming_module._http_bridge_account_capacity_wait_seconds(exc) + == http_bridge_streaming_module._RESPONSE_CREATE_GATE_RETRY_SLEEP_SECONDS + ) + + +def test_http_bridge_capacity_wait_plan_reserves_final_gate_attempt(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + http_bridge_streaming_module, + "_proxy_admission_wait_timeout_seconds", + lambda settings=None: 10.0, + ) + exc = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( + "http_bridge_response_create_gate", + code="response_create_gate_timeout", + ) + now = time.monotonic() + + plenty = http_bridge_streaming_module._http_bridge_capacity_wait_plan(exc, request_deadline=now + 120.0) + assert plenty is not None + assert plenty[0] == pytest.approx( + http_bridge_streaming_module._RESPONSE_CREATE_GATE_RETRY_SLEEP_SECONDS, + abs=0.5, + ) + + # With less budget left than the retry sleep, the plan reserves the tail + # for one final gate acquisition attempt instead of sleeping it away. + tail = http_bridge_streaming_module._http_bridge_capacity_wait_plan(exc, request_deadline=now + 8.0) + assert tail is not None + assert tail[0] == pytest.approx(0.0, abs=0.5) + + +def test_http_bridge_account_capacity_wait_keeps_active_session_capacity_fail_fast() -> None: + exc = ProxyResponseError( + 429, + openai_error( + "capacity_exhausted_active_sessions", + "All accounts are serving active sessions", + error_type="rate_limit_error", ), ) - assert request_state.event_queue is not None - keepalive_block = await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) - assert keepalive_block is not None - keepalive = proxy_service.parse_sse_data_json(keepalive_block) - assert isinstance(keepalive, dict) - assert keepalive["status"] == "waiting_for_account_capacity" - assert keepalive["request_id"] == "req-model-capacity-wait" - assert keepalive["retry_after_seconds"] == 0 - reason = keepalive["reason"] - assert isinstance(reason, str) - assert "Selected model is at capacity" in reason - handle_stream_error.assert_awaited_once() - handle_call = handle_stream_error.await_args - assert handle_call is not None - assert handle_call.args[2] == expected_retry_error_code - retry_precreated.assert_awaited_once_with(session, request_state=request_state) - assert request_state in session.pending_requests - assert session.queued_request_count == 1 - assert request_state.account_capacity_waiting is False + assert http_bridge_streaming_module._http_bridge_account_capacity_wait_seconds(exc) is None @pytest.mark.asyncio -async def test_http_bridge_model_capacity_waits_before_retrying_safe_injected_anchor( +@pytest.mark.parametrize( + ("propagate_http_errors", "expected_event_types"), + [ + (False, ["codex.keepalive", "response.completed"]), + (True, ["response.completed"]), + ], +) +async def test_http_bridge_submit_waits_for_local_account_capacity( monkeypatch: pytest.MonkeyPatch, + propagate_http_errors: bool, + expected_event_types: list[str], ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - capacity_message = "Selected model is at capacity. Please try a different model." - fresh_request_text = '{"type":"response.create","model":"gpt-5.6-sol","input":"full resend"}' + session = _make_bridge_session(key_value="sid-submit-capacity") request_state = proxy_service._WebSocketRequestState( - request_id="req-model-capacity-injected-anchor", - model="gpt-5.6-sol", + request_id="req-submit-capacity", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - bridge_request_deadline=time.monotonic() + 60.0, - awaiting_response_created=True, - event_queue=asyncio.Queue(), transport="http", - previous_response_id="resp-proxy-injected", - preferred_account_id="acc-owner", - proxy_injected_previous_response_id=True, - fresh_upstream_request_text=fresh_request_text, - fresh_upstream_request_is_retry_safe=True, - request_text=( - '{"type":"response.create","model":"gpt-5.6-sol",' - '"previous_response_id":"resp-proxy-injected","input":"trimmed"}' - ), - ) - session = _make_bridge_session( - key_value="bridge-model-capacity-injected-anchor", - pending_requests=deque([request_state]), - queued_request_count=1, + event_queue=asyncio.Queue(), ) - call_order: list[str] = [] - - async def wait_before_retry(*args: object, **kwargs: object) -> bool: - assert args == (request_state,) - assert kwargs == { - "emit_keepalives": True, - "error_message": capacity_message, - "cancel_when_detached": True, - } - assert request_state.previous_response_id == "resp-proxy-injected" - call_order.append("wait") - return True - - async def retry_precreated( - retry_session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState | None = None, - ) -> bool: - assert retry_session is session - assert request_state is not None - assert call_order == ["wait"] - call_order.append("retry") - return True - - monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) - monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - monkeypatch.setattr( - http_bridge_upstream_events_module, - "_wait_before_http_bridge_model_capacity_retry", - wait_before_retry, + assert request_state.event_queue is not None + request_state.event_queue.put_nowait( + 'data: {"type":"response.completed","response":{"id":"resp_submit_capacity"}}\n\n' ) - - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "error", - "status": 429, - "error": { - "type": "rate_limit_error", - "code": "rate_limit_exceeded", - "message": capacity_message, - }, - }, - separators=(",", ":"), + request_state.event_queue.put_nowait(None) + capacity_error = ProxyResponseError( + 429, + openai_error( + "account_response_create_cap", + "Account response-create concurrency limit reached", + error_type="rate_limit_error", ), ) + submit = AsyncMock(side_effect=[capacity_error, None]) + detach = AsyncMock() - assert call_order == ["wait", "retry"] - assert list(session.pending_requests) == [request_state] - assert session.queued_request_count == 1 + settings = _make_app_settings() + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 0.001) + monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 0.001) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(service, "_detach_http_bridge_request", detach) + + chunks = [ + chunk + async for chunk in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=4, + propagate_http_errors=propagate_http_errors, + downstream_turn_state=None, + ) + ] + + event_types = [cast(dict[str, object], proxy_service.parse_sse_data_json(chunk))["type"] for chunk in chunks] + assert event_types == expected_event_types + assert submit.await_count == 2 + detach.assert_awaited_once_with(session, request_state=request_state) @pytest.mark.asyncio -async def test_http_bridge_model_capacity_with_younger_request_releases_failed_queue_slot( +async def test_http_bridge_submit_waits_for_response_create_gate_contention( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - failed_request = proxy_service._WebSocketRequestState( - request_id="req-model-capacity-failed", - model="gpt-5.6-sol", + session = _make_bridge_session(key_value="sid-submit-gate-contention") + request_state = proxy_service._WebSocketRequestState( + request_id="req-submit-gate-contention", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - bridge_request_deadline=time.monotonic() + 60.0, - awaiting_response_created=True, - event_queue=asyncio.Queue(), transport="http", - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"first"}', + event_queue=asyncio.Queue(), ) - younger_request = proxy_service._WebSocketRequestState( - request_id="req-model-capacity-younger", - model="gpt-5.6-sol", + assert request_state.event_queue is not None + request_state.event_queue.put_nowait( + 'data: {"type":"response.completed","response":{"id":"resp_submit_gate_contention"}}\n\n' + ) + request_state.event_queue.put_nowait(None) + gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( + "http_bridge_response_create_gate", + code="response_create_gate_timeout", + ) + submit = AsyncMock(side_effect=[gate_timeout_error, None]) + detach = AsyncMock() + + settings = _make_app_settings() + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(http_bridge_streaming_module, "_RESPONSE_CREATE_GATE_RETRY_SLEEP_SECONDS", 0.001) + monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 0.001) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(service, "_detach_http_bridge_request", detach) + + chunks = [ + chunk + async for chunk in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=4, + propagate_http_errors=False, + downstream_turn_state=None, + ) + ] + + event_types = [cast(dict[str, object], proxy_service.parse_sse_data_json(chunk))["type"] for chunk in chunks] + assert event_types == ["codex.keepalive", "response.completed"] + keepalive = cast(dict[str, object], proxy_service.parse_sse_data_json(chunks[0])) + assert keepalive["status"] == "waiting_for_account_capacity" + assert submit.await_count == 2 + detach.assert_awaited_once_with(session, request_state=request_state) + + +@pytest.mark.asyncio +async def test_http_bridge_stream_persists_original_deadline_on_request_state( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-deadline-persist") + request_state = proxy_service._WebSocketRequestState( + request_id="req-deadline-persist", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, + # Retry/recovery states are re-prepared with a fresh started_at; + # the original deadline must still govern budget clamps. started_at=time.monotonic(), - bridge_request_deadline=time.monotonic() + 60.0, - awaiting_response_created=False, - response_id="resp-model-capacity-younger", - event_queue=asyncio.Queue(), transport="http", - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"second"}', - ) - session = _make_bridge_session( - key_value="bridge-model-capacity-with-younger-request", - pending_requests=deque([younger_request, failed_request]), - queued_request_count=2, + event_queue=asyncio.Queue(), ) - wait_before_retry = AsyncMock(return_value=True) - retry_precreated = AsyncMock(return_value=True) - monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) - monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - monkeypatch.setattr( - http_bridge_upstream_events_module, - "_wait_before_http_bridge_model_capacity_retry", - wait_before_retry, + assert request_state.event_queue is not None + request_state.event_queue.put_nowait( + 'data: {"type":"response.completed","response":{"id":"resp_deadline_persist"}}\n\n' ) + request_state.event_queue.put_nowait(None) + submit = AsyncMock(return_value=None) + detach = AsyncMock() + settings = _make_app_settings() + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(service, "_detach_http_bridge_request", detach) - await service._process_http_bridge_upstream_text( + explicit_deadline = time.monotonic() + 42.0 + async for _ in service._stream_http_bridge_session_events( session, - json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "invalid_request_error", - "message": "Selected model is at capacity. Please try a different model.", - }, - }, - separators=(",", ":"), - ), - ) + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=4, + propagate_http_errors=False, + downstream_turn_state=None, + request_deadline=explicit_deadline, + ): + pass - wait_before_retry.assert_not_awaited() - retry_precreated.assert_not_awaited() - assert list(session.pending_requests) == [younger_request] - assert session.queued_request_count == 1 - assert failed_request.event_queue is not None - assert await failed_request.event_queue.get() is not None - assert await failed_request.event_queue.get() is None + assert request_state.bridge_request_deadline == pytest.approx(explicit_deadline) @pytest.mark.asyncio -async def test_http_bridge_model_capacity_does_not_requeue_after_detach_during_health_update( +async def test_http_bridge_gate_contention_retry_fails_fast_when_queue_full( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-gate-queue-full", queued_request_count=4) request_state = proxy_service._WebSocketRequestState( - request_id="req-model-capacity-detached-during-health", - model="gpt-5.6-sol", + request_id="req-gate-queue-full", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - bridge_request_deadline=time.monotonic() + 60.0, - awaiting_response_created=True, - event_queue=asyncio.Queue(), transport="http", - propagate_http_errors=True, - capacity_startup_wait_event=asyncio.Event(), - capacity_startup_ready_event=asyncio.Event(), - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + event_queue=asyncio.Queue(), ) - session = _make_bridge_session( - key_value="bridge-model-capacity-detached-during-health", - pending_requests=deque([request_state]), - queued_request_count=1, + gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( + "http_bridge_response_create_gate", + code="response_create_gate_timeout", ) + submit = AsyncMock(side_effect=gate_timeout_error) + settings = _make_app_settings() + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - async def detach_during_health_update(*args: object, **kwargs: object) -> None: - del args, kwargs - assert request_state.capacity_startup_wait_event is not None - assert request_state.capacity_startup_wait_event.is_set() is True - assert request_state.capacity_startup_ready_event is not None - assert request_state.capacity_startup_ready_event.is_set() is False - assert request_state in session.pending_requests - assert await service._detach_http_bridge_request(session, request_state=request_state) is True - - wait_before_retry = AsyncMock(return_value=True) - retry_precreated = AsyncMock(return_value=True) - monkeypatch.setattr(service, "_handle_stream_error", detach_during_health_update) - monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - monkeypatch.setattr( - http_bridge_upstream_events_module, - "_wait_before_http_bridge_model_capacity_retry", - wait_before_retry, - ) - - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "invalid_request_error", - "message": "Selected model is at capacity. Please try a different model.", - }, - }, - separators=(",", ":"), - ), - ) + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=4, + propagate_http_errors=True, + downstream_turn_state=None, + ): + pass - wait_before_retry.assert_not_awaited() - retry_precreated.assert_not_awaited() - assert request_state not in session.pending_requests - assert session.queued_request_count == 0 + # A sleeping gate waiter must occupy a queue slot; at the limit the + # retry fails fast instead of accumulating unbounded waiters. + assert exc_info.value.payload["error"]["code"] == "bridge_queue_full" + assert submit.await_count == 1 + assert session.queued_request_count == 4 @pytest.mark.asyncio -async def test_http_bridge_model_capacity_wait_suppresses_keepalive_when_errors_propagate( +async def test_http_bridge_gate_contention_retry_balances_queue_slot( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - retry_precreated = AsyncMock(return_value=True) - monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) - monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - monkeypatch.setattr( - http_bridge_upstream_events_module, - "_ACCOUNT_SELECTION_RECOVERY_DEFAULT_SLEEP_SECONDS", - 0.001, - ) - monkeypatch.setattr( - http_bridge_upstream_events_module, - "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", - 0.001, - ) - + session = _make_bridge_session(key_value="sid-gate-slot-balance", queued_request_count=1) request_state = proxy_service._WebSocketRequestState( - request_id="req-model-capacity-propagate", - model="gpt-5.6-sol", + request_id="req-gate-slot-balance", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - bridge_request_deadline=time.monotonic() + 60.0, - awaiting_response_created=True, - event_queue=asyncio.Queue(), transport="http", - propagate_http_errors=True, - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + event_queue=asyncio.Queue(), ) - session = _make_bridge_session( - key_value="bridge-model-capacity-propagate", - pending_requests=deque([request_state]), - queued_request_count=1, + assert request_state.event_queue is not None + request_state.event_queue.put_nowait( + 'data: {"type":"response.completed","response":{"id":"resp_gate_slot_balance"}}\n\n' ) - - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "invalid_request_error", - "message": "Selected model is at capacity. Please try a different model.", - }, - }, - separators=(",", ":"), - ), + request_state.event_queue.put_nowait(None) + gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( + "http_bridge_response_create_gate", + code="response_create_gate_timeout", ) + submit = AsyncMock(side_effect=[gate_timeout_error, None]) + detach = AsyncMock() + settings = _make_app_settings() + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(http_bridge_streaming_module, "_RESPONSE_CREATE_GATE_RETRY_SLEEP_SECONDS", 0.001) + monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 0.001) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(service, "_detach_http_bridge_request", detach) - assert request_state.event_queue is not None - assert request_state.event_queue.empty() - retry_precreated.assert_awaited_once_with(session, request_state=request_state) + chunks = [ + chunk + async for chunk in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=4, + propagate_http_errors=False, + downstream_turn_state=None, + ) + ] + + assert any("response.completed" in chunk for chunk in chunks) + assert submit.await_count == 2 + # The temporary sleep-slot is released after each retry. + assert session.queued_request_count == 1 @pytest.mark.asyncio -async def test_http_bridge_model_capacity_wait_hides_keepalive_for_non_sdk_propagated_streams( +async def test_http_bridge_gate_contention_does_not_retry_retired_session( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - retry_precreated = AsyncMock(return_value=True) - monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) - monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - monkeypatch.setattr( - http_bridge_upstream_events_module, - "_ACCOUNT_SELECTION_RECOVERY_DEFAULT_SLEEP_SECONDS", - 0.001, - ) - monkeypatch.setattr( - http_bridge_upstream_events_module, - "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", - 0.001, - ) - + session = _make_bridge_session(key_value="sid-gate-retired") + session.closed = True request_state = proxy_service._WebSocketRequestState( - request_id="req-model-capacity-propagate-non-sdk", - model="gpt-5.6-sol", + request_id="req-gate-retired", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - bridge_request_deadline=time.monotonic() + 60.0, - awaiting_response_created=True, - event_queue=asyncio.Queue(), transport="http", - propagate_http_errors=True, - enforce_openai_sdk_contract=False, - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + event_queue=asyncio.Queue(), ) - session = _make_bridge_session( - key_value="bridge-model-capacity-propagate-non-sdk", - pending_requests=deque([request_state]), - queued_request_count=1, + gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( + "http_bridge_response_create_gate", + code="response_create_gate_timeout", ) + submit = AsyncMock(side_effect=gate_timeout_error) + settings = _make_app_settings() + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "invalid_request_error", - "message": "Selected model is at capacity. Please try a different model.", - }, - }, - separators=(",", ":"), - ), - ) + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=4, + propagate_http_errors=True, + downstream_turn_state=None, + ): + pass - assert request_state.event_queue is not None - assert request_state.event_queue.empty() - retry_precreated.assert_awaited_once_with(session, request_state=request_state) + # A gate timeout that retired the session must fail startup cleanly + # instead of retrying the closed session mid-stream. + assert exc_info.value is gate_timeout_error + assert submit.await_count == 1 -@pytest.mark.asyncio -async def test_http_bridge_model_capacity_wait_does_not_retry_after_deadline( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize( + ("unsafe_state", "unsafe_value"), + [ + ("response_id", "resp-already-created"), + ("response_event_count", 1), + ("last_downstream_sequence_number", 0), + ("downstream_visible", True), + ("awaiting_response_created", True), + ("response_create_gate_acquired", True), + ], +) +def test_http_bridge_retired_gate_replacement_requires_unsubmitted_waiter( + unsafe_state: str, + unsafe_value: object, ) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - retry_precreated = AsyncMock(return_value=True) - monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) - monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - + session = _make_bridge_session(key_value="sid-gate-replacement-guard") + session.closed = True request_state = proxy_service._WebSocketRequestState( - request_id="req-model-capacity-deadline", - model="gpt-5.6-sol", + request_id="req-gate-replacement-guard", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - bridge_request_deadline=time.monotonic() - 0.001, - awaiting_response_created=True, - event_queue=asyncio.Queue(), transport="http", - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + request_text='{"type":"response.create"}', + event_queue=asyncio.Queue(), ) - session = _make_bridge_session( - key_value="bridge-model-capacity-deadline", - pending_requests=deque([request_state]), - queued_request_count=1, + setattr(request_state, unsafe_state, unsafe_value) + gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( + "http_bridge_response_create_gate", + code="response_create_gate_timeout", ) - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "invalid_request_error", - "message": "Selected model is at capacity. Please try a different model.", - }, - }, - separators=(",", ":"), - ), + assert not http_bridge_streaming_module._http_bridge_can_replace_retired_gate_session( + gate_timeout_error, + session=session, + request_state=request_state, + request_was_enqueued=False, ) - retry_precreated.assert_not_awaited() - assert request_state not in session.pending_requests - assert session.queued_request_count == 0 - assert request_state.event_queue is not None - terminal_block = await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) - assert terminal_block is not None - terminal = proxy_service.parse_sse_data_json(terminal_block) - assert isinstance(terminal, dict) - assert terminal["type"] == "response.failed" - assert await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) is None - - -@pytest.mark.asyncio -async def test_http_bridge_model_capacity_wait_skips_sleep_for_non_replayable_request( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - wait_before_retry = AsyncMock(return_value=True) - retry_precreated = AsyncMock(return_value=False) - monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) - monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - monkeypatch.setattr( - http_bridge_upstream_events_module, - "_wait_before_http_bridge_model_capacity_retry", - wait_before_retry, - ) +def test_http_bridge_retired_gate_replacement_accepts_cleaned_hard_affinity_waiter() -> None: + session = _make_bridge_session(key_value="sid-gate-replacement-safe") + session.closed = True request_state = proxy_service._WebSocketRequestState( - request_id="req-model-capacity-not-replayable", - model="gpt-5.6-sol", + request_id="req-gate-replacement-safe", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - bridge_request_deadline=time.monotonic() + 60.0, - awaiting_response_created=True, - event_queue=asyncio.Queue(), transport="http", - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', - replay_count=1, + request_text='{"type":"response.create"}', + event_queue=asyncio.Queue(), ) - session = _make_bridge_session( - key_value="bridge-model-capacity-not-replayable", - pending_requests=deque([request_state]), - queued_request_count=1, + gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( + "http_bridge_response_create_gate", + code="response_create_gate_timeout", ) - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "invalid_request_error", - "message": "Selected model is at capacity. Please try a different model.", - }, - }, - separators=(",", ":"), - ), + assert http_bridge_streaming_module._http_bridge_can_replace_retired_gate_session( + gate_timeout_error, + session=session, + request_state=request_state, + request_was_enqueued=False, ) - - wait_before_retry.assert_not_awaited() - retry_precreated.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_http_bridge_model_capacity_wait_does_not_delay_anchored_errors( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - retry_precreated = AsyncMock(return_value=True) - monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) - monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - monkeypatch.setattr( - http_bridge_upstream_events_module, - "_ACCOUNT_SELECTION_RECOVERY_DEFAULT_SLEEP_SECONDS", - 60.0, + assert not http_bridge_streaming_module._http_bridge_can_replace_retired_gate_session( + gate_timeout_error, + session=session, + request_state=request_state, + request_was_enqueued=True, + ) + session.key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "soft-gate-replacement", None) + assert not http_bridge_streaming_module._http_bridge_can_replace_retired_gate_session( + gate_timeout_error, + session=session, + request_state=request_state, + request_was_enqueued=False, ) + +def test_http_bridge_retired_gate_replacement_ignores_replay_count() -> None: + """A non-zero replay_count reflects the client's own reconnect attempts, + not upstream progress on this bridge attempt, so an otherwise fully + unsubmitted waiter must still be replaceable.""" + session = _make_bridge_session(key_value="sid-gate-replacement-replayed") + session.closed = True request_state = proxy_service._WebSocketRequestState( - request_id="req-model-capacity-anchored", - model="gpt-5.6-sol", + request_id="req-gate-replacement-replayed", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - bridge_request_deadline=time.monotonic() + 60.0, - awaiting_response_created=True, - event_queue=asyncio.Queue(), transport="http", - previous_response_id="resp_anchor", - request_text='{"type":"response.create","model":"gpt-5.6-sol","previous_response_id":"resp_anchor"}', + request_text='{"type":"response.create"}', + event_queue=asyncio.Queue(), + replay_count=1, ) - session = _make_bridge_session( - key_value="bridge-model-capacity-anchored", - pending_requests=deque([request_state]), - queued_request_count=1, + gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( + "http_bridge_response_create_gate", + code="response_create_gate_timeout", ) - started = time.monotonic() - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "invalid_request_error", - "message": "Selected model is at capacity. Please try a different model.", - }, - }, - separators=(",", ":"), - ), + assert http_bridge_streaming_module._http_bridge_can_replace_retired_gate_session( + gate_timeout_error, + session=session, + request_state=request_state, + request_was_enqueued=False, ) - assert time.monotonic() - started < 1.0 - retry_precreated.assert_not_awaited() - assert request_state.event_queue is not None - terminal_block = await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) - assert terminal_block is not None - terminal = proxy_service.parse_sse_data_json(terminal_block) - assert isinstance(terminal, dict) - assert terminal["type"] == "response.failed" - @pytest.mark.asyncio -async def test_http_bridge_precreated_completed_terminal_falls_back_to_unresolved_request( +async def test_http_bridge_submit_gate_contention_still_reroutes_soft_sessions( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - monkeypatch.setattr( - http_bridge_retry_circuit_module, - "_HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD", - 1, - ) - finalize = AsyncMock() - register_previous = AsyncMock() - monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize) - monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", register_previous) - + session = _make_bridge_session(key_value="soft-submit-gate-contention") + session.key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "soft-submit-gate-contention", None) request_state = proxy_service._WebSocketRequestState( - request_id="req-precreated-completed", - model="gpt-5.2", + request_id="req-soft-submit-gate-contention", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - awaiting_response_created=True, - event_queue=asyncio.Queue(), transport="http", + bridge_soft_capacity_reroute_allowed=True, ) - session = _make_bridge_session( - key_value="bridge-precreated-completed", - pending_requests=deque([request_state]), - queued_request_count=1, - ) - await service._record_http_bridge_retry_circuit_failure(session, detail="stream_incomplete") - retry_circuits = cast(Any, service)._http_bridge_retry_circuits - assert session.key in retry_circuits - - await service._process_http_bridge_upstream_text( - session, - json.dumps({"type": "response.output_text.delta", "delta": "legacy text"}), - ) - await service._process_http_bridge_upstream_text( - session, - json.dumps({"type": "response.output_text.done", "text": "legacy text"}), - ) - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "response.completed", - "response": { - "id": "resp_precreated_completed", - "object": "response", - "status": "completed", - "output": [], - }, - } - ), + gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( + "http_bridge_response_create_gate", + code="response_create_gate_timeout", ) + submit = AsyncMock(side_effect=gate_timeout_error) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - assert request_state.event_queue is not None - blocks: list[str | None] = [] - while True: - block = await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) - blocks.append(block) - if block is None: - break + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=4, + propagate_http_errors=True, + downstream_turn_state=None, + ): + pass - payloads: list[dict[str, Any]] = [] - for block in blocks: - if block is None: - continue - payload = proxy_service.parse_sse_data_json(block) - assert isinstance(payload, dict) - payloads.append(payload) - assert [payload["type"] for payload in payloads] == [ - "response.output_text.delta", - "response.output_text.done", - "response.completed", - ] - assert request_state.response_id == "resp_precreated_completed" - assert session.last_completed_response_id == "resp_precreated_completed" - assert session.last_completed_response_account_id == session.account.id - assert session.queued_request_count == 0 - assert not session.pending_requests - assert session.key not in retry_circuits - register_previous.assert_awaited_once() - finalize.assert_awaited_once() + assert exc_info.value is gate_timeout_error + assert submit.await_count == 1 @pytest.mark.asyncio -async def test_recovery_completed_alias_persistence_failure_fails_response_and_retires_lane( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_http_bridge_submit_leaves_soft_capacity_for_session_reroute(monkeypatch: pytest.MonkeyPatch) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="soft-submit-capacity") + session.key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "soft-submit-capacity", None) request_state = proxy_service._WebSocketRequestState( - request_id="req-recovery-completed", - response_id="resp_recovery_completed", - model="gpt-5.6-sol", + request_id="req-soft-submit-capacity", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - event_queue=asyncio.Queue(), transport="http", - skip_request_log=True, + bridge_soft_capacity_reroute_allowed=True, ) - session = _make_bridge_session( - key=_make_account_neutral_replay_session_key("completed-alias-failure"), - pending_requests=deque([request_state]), - queued_request_count=1, - ) - register_previous = AsyncMock(return_value=False) - finalize = AsyncMock() - close_session = AsyncMock() - monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", register_previous) - monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize) - monkeypatch.setattr(service, "_close_http_bridge_session", close_session) - - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "response.completed", - "response": { - "id": "resp_recovery_completed", - "object": "response", - "status": "completed", - "output": [], - }, - } + capacity_error = ProxyResponseError( + 429, + openai_error( + "account_response_create_cap", + "Account response-create concurrency limit reached", + error_type="rate_limit_error", ), ) + submit = AsyncMock(side_effect=capacity_error) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - assert request_state.event_queue is not None - event_block = await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) - assert isinstance(event_block, str) - failed = proxy_service.parse_sse_data_json(event_block) - assert failed is not None - assert failed["type"] == "response.failed" - failed_response = failed.get("response") - assert isinstance(failed_response, dict) - failed_error = failed_response.get("error") - assert isinstance(failed_error, dict) - assert failed_error["code"] == "bridge_continuity_persistence_failed" - assert await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) is None - assert session.last_completed_response_id is None - assert session.upstream_control.reconnect_requested is True - assert session.upstream_control.retire_after_drain is True - finalize.assert_awaited_once() - finalize_call = finalize.await_args - assert finalize_call is not None - assert finalize_call.kwargs["event_type"] == "response.failed" + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=4, + propagate_http_errors=True, + downstream_turn_state=None, + ): + pass - assert await service._retire_http_bridge_after_drain_if_ready(session) is True - close_session.assert_awaited_once_with(session) + assert exc_info.value is capacity_error + assert submit.await_count == 1 @pytest.mark.asyncio -async def test_ordinary_completed_alias_rejection_preserves_successful_response( +async def test_http_bridge_submit_capacity_wait_uses_original_request_deadline( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-submit-original-deadline") request_state = proxy_service._WebSocketRequestState( - request_id="req-ordinary-completed", - response_id="resp_ordinary_completed", - model="gpt-5.6-sol", + request_id="req-submit-original-deadline", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), - event_queue=asyncio.Queue(), + started_at=99.5, transport="http", - skip_request_log=True, + event_queue=asyncio.Queue(), ) - session = _make_bridge_session( - key_value="ordinary-completed-alias-rejection", - pending_requests=deque([request_state]), - queued_request_count=1, + capacity_error = ProxyResponseError( + 429, + openai_error( + "account_response_create_cap", + "Account response-create concurrency limit reached", + error_type="rate_limit_error", + ), ) - register_previous = AsyncMock(return_value=False) - finalize = AsyncMock() - monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", register_previous) - monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize) + submit = AsyncMock(side_effect=capacity_error) + clock = [100.0] + waited: list[float] = [] - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "response.completed", - "response": { - "id": "resp_ordinary_completed", - "object": "response", - "status": "completed", - "output": [], - }, - } - ), + async def fake_capacity_wait(**kwargs: object): + waited.append(cast(float, kwargs["sleep_seconds"])) + clock[0] += waited[-1] + if False: + yield "" + + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 30.0) + monkeypatch.setattr(http_bridge_streaming_module, "_iter_account_capacity_wait_sse", fake_capacity_wait) + monkeypatch.setattr( + http_bridge_streaming_module, + "_service_time", + lambda: SimpleNamespace(monotonic=lambda: clock[0]), ) - assert request_state.event_queue is not None - event_block = await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) - assert isinstance(event_block, str) - completed = proxy_service.parse_sse_data_json(event_block) - assert completed is not None - assert completed["type"] == "response.completed" - assert await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) is None - assert session.last_completed_response_id == "resp_ordinary_completed" - assert session.last_completed_response_account_id == session.account.id - assert session.upstream_control.reconnect_requested is False - assert session.upstream_control.retire_after_drain is False - finalize.assert_awaited_once() - finalize_call = finalize.await_args - assert finalize_call is not None - assert finalize_call.kwargs["event_type"] == "response.completed" + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=4, + propagate_http_errors=True, + downstream_turn_state=None, + request_deadline=101.0, + ): + pass + + assert exc_info.value is capacity_error + assert waited == [1.0] + assert submit.await_count == 1 @pytest.mark.asyncio -async def test_http_bridge_upstream_text_archives_with_request_archive_id( +async def test_http_bridge_submit_capacity_retry_uses_advanced_request_body( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - archived: list[tuple[str | None, str]] = [] - + session = _make_bridge_session(key_value="sid-submit-advanced-body") request_state = proxy_service._WebSocketRequestState( - request_id="req-bridge-archive", - model="gpt-5.2", + request_id="req-submit-advanced-body", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), - archive_request_id="archive-bridge-archive", - awaiting_response_created=True, - event_queue=asyncio.Queue(), + started_at=99.5, transport="http", - skip_request_log=True, + event_queue=asyncio.Queue(), + previous_response_id="resp-parent", + proxy_injected_previous_response_id=True, + request_text='{"type":"response.create","previous_response_id":"resp-parent","input":"same"}', ) - session = _make_bridge_session( - key_value="bridge-archive", - pending_requests=deque([request_state]), - queued_request_count=1, + stale_text_data = '{"type":"response.create","input":"same"}' + capacity_error = ProxyResponseError( + 429, + openai_error( + "account_response_create_cap", + "Account response-create concurrency limit reached", + error_type="rate_limit_error", + ), ) + submitted_texts: list[str] = [] + clock = [100.0] - def archive_received(message: UpstreamWebSocketMessage) -> None: - archived.append((get_request_id(), message.text or "")) + async def submit(_session: Any, *, request_state: Any, text_data: str, **_kwargs: Any) -> None: + submitted_texts.append(text_data) + if len(submitted_texts) == 1: + raise capacity_error + await request_state.event_queue.put(None) - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace(close=AsyncMock(), archive_received=archive_received), + async def fake_capacity_wait(**kwargs: object): + clock[0] += cast(float, kwargs["sleep_seconds"]) + if False: + yield "" + + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 0.001) + monkeypatch.setattr(http_bridge_streaming_module, "_iter_account_capacity_wait_sse", fake_capacity_wait) + monkeypatch.setattr( + http_bridge_streaming_module, + "_service_time", + lambda: SimpleNamespace(monotonic=lambda: clock[0]), ) - monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", AsyncMock()) - upstream_text = json.dumps( - { - "type": "response.created", - "response": {"id": "resp_bridge_archive", "status": "in_progress"}, - }, - separators=(",", ":"), + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data=stale_text_data, + queue_limit=4, + propagate_http_errors=True, + downstream_turn_state=None, + request_deadline=101.0, + ): + pass + + assert submitted_texts == [stale_text_data, request_state.request_text] + + +def _make_api_key( + *, + key_id: str, + assigned_account_ids: list[str], + account_assignment_scope_enabled: bool | None = None, +) -> proxy_service.ApiKeyData: + return proxy_service.ApiKeyData( + id=key_id, + name="bridge-key", + key_prefix="sk-bridge", + allowed_models=None, + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=datetime(2025, 1, 1, tzinfo=timezone.utc), + last_used_at=None, + account_assignment_scope_enabled=( + bool(assigned_account_ids) if account_assignment_scope_enabled is None else account_assignment_scope_enabled + ), + assigned_account_ids=assigned_account_ids, ) - await service._process_http_bridge_upstream_text(session, upstream_text) - assert archived == [("archive-bridge-archive", upstream_text)] - assert request_state.response_id == "resp_bridge_archive" +def test_http_bridge_request_budget_falls_back_to_proxy_budget() -> None: + settings = SimpleNamespace(proxy_request_budget_seconds=42.5) + assert http_bridge_streaming_module._http_bridge_request_budget_seconds(settings) == 42.5 -@pytest.mark.asyncio -async def test_http_bridge_upstream_non_text_archives_with_request_archive_id( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - archived: list[tuple[str | None, str, int | None]] = [] - request_state = proxy_service._WebSocketRequestState( - request_id="req-bridge-close-archive", - model="gpt-5.2", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - archive_request_id="archive-bridge-close", - awaiting_response_created=True, - event_queue=asyncio.Queue(), - transport="http", - skip_request_log=True, - ) - close_message = UpstreamWebSocketMessage(kind="close", close_code=1000) - session = _make_bridge_session( - key_value="bridge-close-archive", - pending_requests=deque([request_state]), - queued_request_count=1, - ) +def test_websocket_top_level_error_payload_uses_error_type_not_event_type() -> None: + payload: dict[str, proxy_service.JsonValue] = { + "type": "error", + "status": 400, + "error_type": "invalid_request_error", + "code": "previous_response_not_found", + "message": "Previous response with id 'resp_missing' not found.", + "param": "previous_response_id", + } - def archive_received(message: UpstreamWebSocketMessage) -> None: - archived.append((get_request_id(), message.kind, message.close_code)) + error = proxy_service._websocket_event_error_payload("error", payload) - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace( - receive=AsyncMock(return_value=close_message), - close=AsyncMock(), - archive_received=archive_received, - ), - ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) - monkeypatch.setattr(service, "_fail_pending_websocket_requests", AsyncMock()) - monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", AsyncMock()) + assert error == { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": "Previous response with id 'resp_missing' not found.", + "param": "previous_response_id", + } + assert proxy_service._websocket_event_error_type("error", payload) == "invalid_request_error" + assert proxy_service._websocket_event_error_code("error", payload) == "previous_response_not_found" - await service._relay_http_bridge_upstream_messages(session) - assert archived == [("archive-bridge-close", "close", 1000)] - assert session.last_upstream_close_code == 1000 +def test_http_error_status_from_payload_accepts_official_status_code_alias() -> None: + payload: dict[str, proxy_service.JsonValue] = { + "type": "error", + "status_code": 400, + "error": {"message": "bad request"}, + } + assert proxy_service._http_error_status_from_payload(payload) == 400 -@pytest.mark.asyncio -async def test_http_bridge_relay_publishes_live_rate_limit_events( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from app.core.usage import live_hub - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="bridge-live-rate-limits") - rate_limit_text = ( - '{"type":"codex.rate_limits","rate_limits":{"primary":' - '{"used_percent":72,"window_minutes":300,"reset_at":1700000300}}}' - ) - messages = [ - UpstreamWebSocketMessage(kind="text", text=rate_limit_text), - UpstreamWebSocketMessage(kind="close", close_code=1000), - ] - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace( - receive=AsyncMock(side_effect=messages), - close=AsyncMock(), - archive_received=lambda message: None, - ), +def test_durable_tool_call_manifest_requires_complete_added_and_done_lifecycle() -> None: + state = proxy_service._WebSocketRequestState( + request_id="req-manifest", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service, "_process_http_bridge_upstream_text", AsyncMock()) - monkeypatch.setattr(service, "_retire_http_bridge_after_drain_if_ready", AsyncMock(return_value=False)) - monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) - monkeypatch.setattr(service, "_fail_http_bridge_reader_and_maybe_retire", AsyncMock()) - monkeypatch.setattr(service, "_fail_pending_websocket_requests", AsyncMock()) - captured: list[tuple[Any, str | None]] = [] - live_hub.register_live_usage_publisher( - lambda snapshot, *, account_id=None, chatgpt_account_id=None: captured.append((snapshot, account_id)) - ) - try: - await service._relay_http_bridge_upstream_messages(session) - finally: - live_hub.register_live_usage_publisher(None) + def record(event_type: str, call_id: str) -> None: + http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( + state, + event_type=event_type, + payload={ + "type": event_type, + "item": { + "type": "function_call", + "call_id": call_id, + "name": "lookup", + "arguments": "{}", + }, + }, + ) - assert len(captured) == 1 - snapshot, account_id = captured[0] - assert account_id == session.account.id - assert snapshot.primary is not None - assert snapshot.primary.used_percent == pytest.approx(72.0) + record("response.output_item.added", "call_1") + record("response.output_item.added", "call_2") + record("response.output_item.done", "call_1") + assert ( + http_bridge_upstream_events_module._durable_pending_tool_call_manifest( + state, + {"type": "response.completed", "response": {"output": []}}, + ) + is None + ) -def test_pop_terminal_websocket_request_state_precreated_completed_does_not_guess_with_ambiguous_pending() -> None: - draining = proxy_service._WebSocketRequestState( - request_id="req-draining", - model="gpt-5.2", + malformed_state = proxy_service._WebSocketRequestState( + request_id="req-malformed-manifest", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, - draining_until_terminal=True, + started_at=1.0, ) - visible = proxy_service._WebSocketRequestState( - request_id="req-visible", - model="gpt-5.2", + http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( + malformed_state, + event_type="response.output_item.added", + payload={ + "type": "response.output_item.added", + "item": {"type": [], "call_id": "call_malformed"}, + }, + ) + assert malformed_state.tool_call_manifest_invalid is True + + missing_item_state = proxy_service._WebSocketRequestState( + request_id="req-missing-item-manifest", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, + started_at=1.0, ) - pending = deque([draining, visible]) - - popped = proxy_service._pop_terminal_websocket_request_state( - pending, - response_id="resp_ambiguous_precreated_completed", - fallback_request_state=None, - allow_precreated_terminal_fallback=True, + http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( + missing_item_state, + event_type="response.output_item.done", + payload={"type": "response.output_item.done"}, + ) + assert missing_item_state.tool_call_manifest_invalid is True + assert ( + http_bridge_upstream_events_module._durable_pending_tool_call_manifest( + malformed_state, + {"type": "response.completed", "response": {"output": [{"type": []}]}}, + ) + is None ) - assert popped is None - assert list(pending) == [draining, visible] - assert draining.response_id is None - assert visible.response_id is None - - -def test_trim_http_bridge_previous_response_input_items_preserves_context_assistant_message() -> None: - items: list[proxy_service.JsonValue] = [ - {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "local context"}]}, - {"role": "user", "content": [{"type": "input_text", "text": "next"}]}, - ] - - assert proxy_service._trim_http_bridge_previous_response_input_items(items) == items - + record("response.output_item.done", "call_2") + assert http_bridge_upstream_events_module._durable_pending_tool_call_manifest( + state, + {"type": "response.completed", "response": {"output": []}}, + ) == {"call_1": "function_call", "call_2": "function_call"} -def test_trim_http_bridge_previous_response_input_items_trims_marked_replay_outputs() -> None: - items: list[proxy_service.JsonValue] = [ - {"id": "rs_replay", "type": "reasoning", "summary": []}, - { - "id": "msg_replay", - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "prior"}], - }, - { - "id": "fc_replay", + duplicate_state = proxy_service._WebSocketRequestState( + request_id="req-duplicate-manifest", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + ) + duplicate_event: dict[str, proxy_service.JsonValue] = { + "type": "response.output_item.added", + "item": { "type": "function_call", - "call_id": "call_1", + "call_id": "call_duplicate", "name": "lookup", "arguments": "{}", }, - {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, - {"role": "user", "content": [{"type": "input_text", "text": "next"}]}, - ] + } + http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( + duplicate_state, + event_type="response.output_item.added", + payload=duplicate_event, + ) + http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( + duplicate_state, + event_type="response.output_item.added", + payload=duplicate_event, + ) + assert duplicate_state.tool_call_manifest_invalid is True - assert proxy_service._trim_http_bridge_previous_response_input_items(items) == items[3:] + duplicate_done_state = proxy_service._WebSocketRequestState( + request_id="req-duplicate-done-manifest", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + ) + duplicate_done_event = { + **duplicate_event, + "type": "response.output_item.done", + } + for event_type, event_payload in ( + ("response.output_item.added", duplicate_event), + ("response.output_item.done", duplicate_done_event), + ("response.output_item.done", duplicate_done_event), + ): + http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( + duplicate_done_state, + event_type=event_type, + payload=event_payload, + ) + assert duplicate_done_state.tool_call_manifest_invalid is True -def test_trim_http_bridge_previous_response_input_items_trims_marked_apply_patch_replay_outputs() -> None: - items: list[proxy_service.JsonValue] = [ - { - "id": "apc_replay", - "type": "apply_patch_call", - "status": "completed", - "call_id": "call_patch_1", - }, - {"type": "apply_patch_call_output", "call_id": "call_patch_1", "status": "completed", "output": "patched"}, - {"role": "user", "content": [{"type": "input_text", "text": "next"}]}, - ] - - assert proxy_service._trim_http_bridge_previous_response_input_items(items) == items[1:] - - -def test_trim_http_bridge_previous_response_input_items_preserves_unmarked_call_context() -> None: - items: list[proxy_service.JsonValue] = [ - {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "local context"}]}, - {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, - {"role": "user", "content": [{"type": "input_text", "text": "next"}]}, - ] - - assert proxy_service._trim_http_bridge_previous_response_input_items(items) == items - - -@pytest.mark.asyncio -async def test_http_bridge_stream_masks_single_top_level_previous_response_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - monkeypatch.setattr(service, "_finalize_websocket_request_state", AsyncMock()) - monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) - - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-single-prev", None), - headers={"session_id": "sid-single-prev"}, - affinity=proxy_service._AffinityPolicy( - key="sid-single-prev", - kind=proxy_service.StickySessionKind.CODEX_SESSION, - ), - request_model="gpt-5.1", - account=cast(Any, SimpleNamespace(id="acc-single-prev", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=1, - last_used_at=1.0, - idle_ttl_seconds=120.0, - ) - request_state = proxy_service._WebSocketRequestState( - request_id="req-single-prev", - model="gpt-5.1", +def test_durable_tool_call_manifest_rejects_unobserved_terminal_call() -> None: + state = proxy_service._WebSocketRequestState( + request_id="req-terminal-manifest", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), - event_queue=asyncio.Queue(), - transport="http", - previous_response_id="resp_missing_single", + started_at=1.0, ) - upstream_text = json.dumps( - { - "type": "error", - "status": 400, - "error": { - "type": "invalid_request_error", - "code": "previous_response_not_found", - "message": "Previous response with id 'resp_missing_single' not found.", - "param": "previous_response_id", + + assert ( + http_bridge_upstream_events_module._durable_pending_tool_call_manifest( + state, + { + "type": "response.completed", + "response": { + "output": [ + { + "type": "function_call", + "call_id": "call_unobserved", + "name": "lookup", + "arguments": "{}", + } + ] + }, }, - }, - separators=(",", ":"), + ) + is None ) - async def fake_submit_http_bridge_request( - target_session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, - queue_limit: int, - ) -> None: - del text_data, queue_limit - target_session.pending_requests.append(request_state) - await service._process_http_bridge_upstream_text(target_session, upstream_text) - - monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) - - events = [ - event - async for event in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data="{}", - queue_limit=8, - propagate_http_errors=False, - downstream_turn_state=None, + state.added_tool_call_types = {"call_duplicate": "function_call"} + state.pending_tool_call_types = {"call_duplicate": "function_call"} + duplicate_terminal_call: dict[str, proxy_service.JsonValue] = { + "type": "function_call", + "call_id": "call_duplicate", + "name": "lookup", + "arguments": "{}", + } + assert ( + http_bridge_upstream_events_module._durable_pending_tool_call_manifest( + state, + { + "type": "response.completed", + "response": {"output": [duplicate_terminal_call, duplicate_terminal_call]}, + }, ) - ] - - assert session.upstream_control.reconnect_requested is False - assert request_state.error_http_status_override == 502 - assert len(events) == 1 - event_block = events[0] - assert "previous_response_not_found" not in event_block - payload = proxy_service.parse_sse_data_json(event_block) - assert isinstance(payload, dict) - assert payload["type"] == "response.failed" - response = payload["response"] - assert isinstance(response, dict) - error = response["error"] - assert isinstance(error, dict) - assert error["code"] == "stream_incomplete" + is None + ) -@pytest.mark.asyncio -async def test_http_bridge_startup_cooldown_releases_api_key_reservation( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="sid-startup-reservation") - reservation = cast(Any, object()) - request_state = proxy_service._WebSocketRequestState( - request_id="req-startup-reservation", - model="gpt-5.1", +def test_durable_tool_call_manifest_rejects_mixed_client_settled_call_types() -> None: + state = proxy_service._WebSocketRequestState( + request_id="req-mixed-client-settled-manifest", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, - api_key_reservation=reservation, - started_at=time.monotonic(), - event_queue=asyncio.Queue(), - transport="http", - previous_response_id="resp-anchor", + api_key_reservation=None, + started_at=1.0, ) - retry_snapshot = AsyncMock( - return_value=http_bridge_retry_circuit_module._HTTPBridgeRetryCircuitDecision( - allowed=False, - retry_after_seconds=30.0, - last_detail="stream_incomplete", - consecutive_failures=2, + function_item: dict[str, proxy_service.JsonValue] = { + "type": "function_call", + "call_id": "call_function", + "name": "lookup", + "arguments": "{}", + } + for event_type in ("response.output_item.added", "response.output_item.done"): + http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( + state, + event_type=event_type, + payload={"type": event_type, "item": function_item}, ) - ) - release = AsyncMock() - monkeypatch.setattr(service, "_http_bridge_retry_circuit_snapshot", retry_snapshot) - monkeypatch.setattr(service, "_release_websocket_request_state_reservation", release) - events = [ - event - async for event in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data='{"type":"response.create"}', - queue_limit=8, - propagate_http_errors=False, - downstream_turn_state=None, + assert ( + http_bridge_upstream_events_module._durable_pending_tool_call_manifest( + state, + { + "type": "response.completed", + "response": { + "output": [ + function_item, + { + "type": "computer_call", + "call_id": "call_computer", + "action": {"type": "screenshot"}, + }, + ] + }, + }, ) - ] + is None + ) - assert len(events) == 1 - assert '"code":"stream_idle_timeout"' in events[0] - release.assert_awaited_once_with(request_state) - assert request_state.api_key_reservation is None + lifecycle_state = proxy_service._WebSocketRequestState( + request_id="req-unsupported-lifecycle-manifest", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + ) + http_bridge_upstream_events_module._record_http_bridge_tool_call_lifecycle( + lifecycle_state, + event_type="response.output_item.added", + payload={ + "type": "response.output_item.added", + "item": { + "type": "mcp_approval_request", + "id": "approval_1", + }, + }, + ) + assert lifecycle_state.tool_call_manifest_invalid is True @pytest.mark.asyncio -async def test_http_bridge_post_submit_cooldown_race_detaches_request( +async def test_http_bridge_malformed_tool_lifecycle_persists_unknown_manifest( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="sid-post-submit-cooldown") + register_previous = AsyncMock(return_value=True) + monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", register_previous) + monkeypatch.setattr(service, "_finalize_websocket_request_state", AsyncMock()) request_state = proxy_service._WebSocketRequestState( - request_id="req-post-submit-cooldown", - model="gpt-5.1", + request_id="req-malformed-lifecycle", + response_id="resp_malformed_lifecycle", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), - event_queue=asyncio.Queue(), + started_at=1.0, transport="http", - previous_response_id="resp-anchor", + skip_request_log=True, ) - - async def submit(target_session: Any, *, request_state: Any, **kwargs: Any) -> None: - del kwargs - target_session.pending_requests.append(request_state) - - retry_snapshot = AsyncMock( - side_effect=[ - http_bridge_retry_circuit_module._HTTPBridgeRetryCircuitDecision(allowed=True), - http_bridge_retry_circuit_module._HTTPBridgeRetryCircuitDecision( - allowed=False, - retry_after_seconds=30.0, - last_detail="stream_incomplete", - consecutive_failures=2, - ), - ] + session = _make_bridge_session( + key_value="bridge-malformed-lifecycle", + pending_requests=deque([request_state]), + queued_request_count=1, ) - detach = AsyncMock() - monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - monkeypatch.setattr(service, "_http_bridge_retry_circuit_snapshot", retry_snapshot) - monkeypatch.setattr(service, "_detach_http_bridge_request", detach) + valid_item = { + "type": "function_call", + "call_id": "call_1", + "name": "lookup", + "arguments": "{}", + } - events = [ - event - async for event in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data='{"type":"response.create"}', - queue_limit=8, - propagate_http_errors=False, - downstream_turn_state=None, - ) - ] + for event in ( + { + "type": "response.output_item.added", + "response_id": "resp_malformed_lifecycle", + "item": valid_item, + }, + { + "type": "response.output_item.done", + "response_id": "resp_malformed_lifecycle", + "item": valid_item, + }, + { + "type": "response.output_item.added", + "response_id": "resp_malformed_lifecycle", + }, + { + "type": "response.completed", + "response": { + "id": "resp_malformed_lifecycle", + "object": "response", + "status": "completed", + "output": [], + }, + }, + ): + await service._process_http_bridge_upstream_text(session, json.dumps(event, separators=(",", ":"))) - assert len(events) == 1 - assert '"code":"stream_idle_timeout"' in events[0] - assert retry_snapshot.await_count == 2 - detach.assert_awaited_once_with(session, request_state=request_state) + registration = register_previous.await_args + assert registration is not None + assert registration.kwargs["pending_tool_calls"] is None +@pytest.mark.parametrize( + ("upstream_code", "expected_retry_error_code"), + [ + ("invalid_request_error", "server_is_overloaded"), + ("rate_limit_exceeded", "rate_limit_exceeded"), + ], +) @pytest.mark.asyncio -async def test_http_bridge_keepalive_counts_as_first_yield_before_late_response_failed( +async def test_http_bridge_model_capacity_waits_before_precreated_retry( monkeypatch: pytest.MonkeyPatch, + upstream_code: str, + expected_retry_error_code: str, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + handle_stream_error = AsyncMock() + retry_precreated = AsyncMock(return_value=True) + monkeypatch.setattr(service, "_handle_stream_error", handle_stream_error) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: SimpleNamespace(sse_keepalive_interval_seconds=0.001), + http_bridge_upstream_events_module, + "_ACCOUNT_SELECTION_RECOVERY_DEFAULT_SLEEP_SECONDS", + 0.001, ) - monkeypatch.setattr(proxy_service, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.001) - - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-keepalive-first", None), - headers={"session_id": "sid-keepalive-first"}, - affinity=proxy_service._AffinityPolicy( - key="sid-keepalive-first", - kind=proxy_service.StickySessionKind.CODEX_SESSION, - ), - request_model="gpt-5.1", - account=cast(Any, SimpleNamespace(id="acc-keepalive-first", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=1, - last_used_at=1.0, - idle_ttl_seconds=120.0, + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", + 0.001, ) + request_state = proxy_service._WebSocketRequestState( - request_id="req-keepalive-first", - model="gpt-5.1", + request_id="req-model-capacity-wait", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), + bridge_request_deadline=time.monotonic() + 60.0, + awaiting_response_created=True, event_queue=asyncio.Queue(), transport="http", - response_id="resp_keepalive_first", + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', ) - - async def fake_submit_http_bridge_request( - target_session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, - queue_limit: int, - ) -> None: - del text_data, queue_limit - target_session.pending_requests.append(request_state) - - monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) - - stream = service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data="{}", - queue_limit=8, - propagate_http_errors=True, - downstream_turn_state=None, + session = _make_bridge_session( + key_value="bridge-model-capacity-wait", + pending_requests=deque([request_state]), + queued_request_count=1, ) - keepalive = await asyncio.wait_for(anext(stream), timeout=1.0) - assert "response.in_progress" in keepalive - - event_queue = request_state.event_queue - assert event_queue is not None - request_state.error_http_status_override = 502 - await event_queue.put( - proxy_service.format_sse_event( - proxy_service.response_failed_event( - "upstream_unavailable", - "upstream failed after keepalive", - response_id="resp_keepalive_first", - ) - ) + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": upstream_code, + "message": "Selected model is at capacity. Please try a different model.", + }, + }, + separators=(",", ":"), + ), ) - failed = await asyncio.wait_for(anext(stream), timeout=1.0) - assert "response.failed" in failed - assert "upstream_unavailable" in failed - await event_queue.put(None) - with pytest.raises(StopAsyncIteration): - await asyncio.wait_for(anext(stream), timeout=1.0) + assert request_state.event_queue is not None + keepalive_block = await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) + assert keepalive_block is not None + keepalive = proxy_service.parse_sse_data_json(keepalive_block) + assert isinstance(keepalive, dict) + assert keepalive["status"] == "waiting_for_account_capacity" + assert keepalive["request_id"] == "req-model-capacity-wait" + assert keepalive["retry_after_seconds"] == 0 + reason = keepalive["reason"] + assert isinstance(reason, str) + assert "Selected model is at capacity" in reason + handle_stream_error.assert_awaited_once() + handle_call = handle_stream_error.await_args + assert handle_call is not None + assert handle_call.args[2] == expected_retry_error_code + retry_precreated.assert_awaited_once_with(session, request_state=request_state) + assert request_state in session.pending_requests + assert session.queued_request_count == 1 + assert request_state.account_capacity_waiting is False @pytest.mark.asyncio -async def test_http_bridge_account_capacity_wait_sends_keepalive_instead_of_idle_timeout( +async def test_http_bridge_model_capacity_waits_before_retrying_safe_injected_anchor( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) - monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: SimpleNamespace( - sse_keepalive_interval_seconds=0.001, - stream_idle_timeout_seconds=0.001, - ), - ) - monkeypatch.setattr(proxy_service, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.001) - - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-capacity-wait", None), - headers={"session_id": "sid-capacity-wait"}, - affinity=proxy_service._AffinityPolicy( - key="sid-capacity-wait", - kind=proxy_service.StickySessionKind.CODEX_SESSION, - ), - request_model="gpt-5.1", - account=cast(Any, SimpleNamespace(id="acc-capacity-wait", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=1, - last_used_at=1.0, - idle_ttl_seconds=120.0, - ) + capacity_message = "Selected model is at capacity. Please try a different model." + fresh_request_text = '{"type":"response.create","model":"gpt-5.6-sol","input":"full resend"}' request_state = proxy_service._WebSocketRequestState( - request_id="req-capacity-wait", - model="gpt-5.1", + request_id="req-model-capacity-injected-anchor", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), + bridge_request_deadline=time.monotonic() + 60.0, + awaiting_response_created=True, event_queue=asyncio.Queue(), transport="http", + previous_response_id="resp-proxy-injected", + preferred_account_id="acc-owner", + proxy_injected_previous_response_id=True, + fresh_upstream_request_text=fresh_request_text, + fresh_upstream_request_is_retry_safe=True, + request_text=( + '{"type":"response.create","model":"gpt-5.6-sol",' + '"previous_response_id":"resp-proxy-injected","input":"trimmed"}' + ), ) - request_state.account_capacity_waiting = True - request_state.account_capacity_wait_reason = "Rate limit exceeded. Try again in 120s" - request_state.account_capacity_wait_started_at = time.monotonic() - 3.0 - request_state.account_capacity_wait_retry_after_seconds = 120.0 - - async def fake_submit_http_bridge_request( - target_session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, - queue_limit: int, - ) -> None: - del text_data, queue_limit - target_session.pending_requests.append(request_state) + session = _make_bridge_session( + key_value="bridge-model-capacity-injected-anchor", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + call_order: list[str] = [] - monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) + async def wait_before_retry(*args: object, **kwargs: object) -> bool: + assert args == (request_state,) + assert kwargs == { + "emit_keepalives": True, + "error_message": capacity_message, + "cancel_when_detached": True, + } + assert request_state.previous_response_id == "resp-proxy-injected" + call_order.append("wait") + return True - stream = service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data="{}", - queue_limit=8, - propagate_http_errors=True, - downstream_turn_state=None, - ) + async def retry_precreated( + retry_session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState | None = None, + ) -> bool: + assert retry_session is session + assert request_state is not None + assert call_order == ["wait"] + call_order.append("retry") + return True - keepalive = await asyncio.wait_for(anext(stream), timeout=1.0) - payload = proxy_service.parse_sse_data_json(keepalive) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_wait_before_http_bridge_model_capacity_retry", + wait_before_retry, + ) - assert payload is not None - assert payload["type"] == "codex.keepalive" - assert payload["status"] == "waiting_for_account_capacity" - assert payload["request_id"] == "req-capacity-wait" - assert "stream_idle_timeout" not in keepalive + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "error", + "status": 429, + "error": { + "type": "rate_limit_error", + "code": "rate_limit_exceeded", + "message": capacity_message, + }, + }, + separators=(",", ":"), + ), + ) - await stream.aclose() + assert call_order == ["wait", "retry"] + assert list(session.pending_requests) == [request_state] + assert session.queued_request_count == 1 @pytest.mark.asyncio -async def test_http_bridge_idle_recovery_transport_failure_yields_terminal_event( +async def test_http_bridge_model_capacity_with_younger_request_releases_failed_queue_slot( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - detach = AsyncMock() - monkeypatch.setattr(service, "_detach_http_bridge_request", detach) - monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: SimpleNamespace( - http_responses_stream_request_budget_seconds=60.0, - sse_keepalive_interval_seconds=0.001, - stream_idle_timeout_seconds=0.001, - ), + failed_request = proxy_service._WebSocketRequestState( + request_id="req-model-capacity-failed", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + bridge_request_deadline=time.monotonic() + 60.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + transport="http", + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"first"}', ) - monkeypatch.setattr(proxy_service, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.001) - monkeypatch.setattr(http_bridge_streaming_module, "_stream_keepalive_max_count", lambda: 1) - - session = _make_bridge_session(key_value="sid-idle-retry-transport") - request_state = proxy_service._WebSocketRequestState( - request_id="req-idle-retry-transport", - model="gpt-5.1", + younger_request = proxy_service._WebSocketRequestState( + request_id="req-model-capacity-younger", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), + bridge_request_deadline=time.monotonic() + 60.0, + awaiting_response_created=False, + response_id="resp-model-capacity-younger", event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.1","input":"hello"}', transport="http", + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"second"}', ) - - async def fake_submit_http_bridge_request( - target_session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, - queue_limit: int, - ) -> None: - del text_data, queue_limit - target_session.pending_requests.append(request_state) - - retry_error = UpstreamWebSocketTransportError( - "Codex upstream websocket send failed: OSError", - error_code="proxy_network_unavailable", + session = _make_bridge_session( + key_value="bridge-model-capacity-with-younger-request", + pending_requests=deque([younger_request, failed_request]), + queued_request_count=2, ) - retry_precreated = AsyncMock(side_effect=retry_error) - monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) + wait_before_retry = AsyncMock(return_value=True) + retry_precreated = AsyncMock(return_value=True) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_wait_before_http_bridge_model_capacity_retry", + wait_before_retry, + ) - chunks = [ - chunk - async for chunk in service._stream_http_bridge_session_events( - session, - request_state=request_state, - text_data="{}", - queue_limit=8, - propagate_http_errors=True, - downstream_turn_state=None, - ) - ] + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "invalid_request_error", + "message": "Selected model is at capacity. Please try a different model.", + }, + }, + separators=(",", ":"), + ), + ) - assert len(chunks) == 1 - payload = proxy_service.parse_sse_data_json(chunks[0]) - assert payload is not None - assert payload["type"] == "response.failed" - response = payload["response"] - assert isinstance(response, dict) - error = response["error"] - assert isinstance(error, dict) - assert error["code"] == "proxy_network_unavailable" - assert error["message"] == "Codex upstream websocket send failed: OSError" - retry_precreated.assert_awaited_once_with(session, restart_reader=True) - detach.assert_awaited_once_with(session, request_state=request_state) + wait_before_retry.assert_not_awaited() + retry_precreated.assert_not_awaited() + assert list(session.pending_requests) == [younger_request] + assert session.queued_request_count == 1 + assert failed_request.event_queue is not None + assert await failed_request.event_queue.get() is not None + assert await failed_request.event_queue.get() is None @pytest.mark.asyncio -async def test_http_bridge_capacity_wait_with_response_id_sends_explicit_keepalive( +async def test_http_bridge_model_capacity_does_not_requeue_after_detach_during_health_update( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) - monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: SimpleNamespace( - sse_keepalive_interval_seconds=0.001, - stream_idle_timeout_seconds=0.001, - ), - ) - monkeypatch.setattr(proxy_service, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.001) - - session = _make_bridge_session(key_value="sid-capacity-response") request_state = proxy_service._WebSocketRequestState( - request_id="req-capacity-response", - model="gpt-5.1", + request_id="req-model-capacity-detached-during-health", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), + bridge_request_deadline=time.monotonic() + 60.0, + awaiting_response_created=True, event_queue=asyncio.Queue(), - response_id="resp-capacity-response", transport="http", + propagate_http_errors=True, + capacity_startup_wait_event=asyncio.Event(), + capacity_startup_ready_event=asyncio.Event(), + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + ) + session = _make_bridge_session( + key_value="bridge-model-capacity-detached-during-health", + pending_requests=deque([request_state]), + queued_request_count=1, ) - request_state.account_capacity_waiting = True - request_state.account_capacity_wait_reason = "Rate limit exceeded. Try again in 120s" - request_state.account_capacity_wait_started_at = time.monotonic() - 3.0 - request_state.account_capacity_wait_retry_after_seconds = 120.0 - async def fake_submit_http_bridge_request( - target_session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, - queue_limit: int, - ) -> None: - del text_data, queue_limit - target_session.pending_requests.append(request_state) + async def detach_during_health_update(*args: object, **kwargs: object) -> None: + del args, kwargs + assert request_state.capacity_startup_wait_event is not None + assert request_state.capacity_startup_wait_event.is_set() is True + assert request_state.capacity_startup_ready_event is not None + assert request_state.capacity_startup_ready_event.is_set() is False + assert request_state in session.pending_requests + assert await service._detach_http_bridge_request(session, request_state=request_state) is True - monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) + wait_before_retry = AsyncMock(return_value=True) + retry_precreated = AsyncMock(return_value=True) + monkeypatch.setattr(service, "_handle_stream_error", detach_during_health_update) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_wait_before_http_bridge_model_capacity_retry", + wait_before_retry, + ) - stream = service._stream_http_bridge_session_events( + await service._process_http_bridge_upstream_text( session, - request_state=request_state, - text_data="{}", - queue_limit=8, - propagate_http_errors=True, - downstream_turn_state=None, + json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "invalid_request_error", + "message": "Selected model is at capacity. Please try a different model.", + }, + }, + separators=(",", ":"), + ), ) - keepalive = proxy_service.parse_sse_data_json(await asyncio.wait_for(anext(stream), timeout=1.0)) - in_progress = proxy_service.parse_sse_data_json(await asyncio.wait_for(anext(stream), timeout=1.0)) - - assert keepalive is not None - assert keepalive["type"] == "codex.keepalive" - assert keepalive["status"] == "waiting_for_account_capacity" - assert in_progress is not None - assert in_progress["type"] == "response.in_progress" - response = in_progress["response"] - assert isinstance(response, dict) - assert response["id"] == "resp-capacity-response" - - await stream.aclose() + wait_before_retry.assert_not_awaited() + retry_precreated.assert_not_awaited() + assert request_state not in session.pending_requests + assert session.queued_request_count == 0 @pytest.mark.asyncio -async def test_get_or_create_http_bridge_session_reuses_live_local_session_without_ring_lookup( +async def test_http_bridge_model_capacity_wait_suppresses_keepalive_when_errors_propagate( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = proxy_service._HTTPBridgeSessionKey("prompt_cache_key", "bridge-key", None) - existing = proxy_service._HTTPBridgeSession( - key=key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-key"), - request_model="gpt-5.4-mini", - account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE, plan_type="plus")), - upstream=cast(UpstreamWebSocket, SimpleNamespace()), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, - ) - service._http_bridge_sessions[key] = existing + retry_precreated = AsyncMock(return_value=True) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) monkeypatch.setattr( - service, - "_prune_http_bridge_sessions_locked", - Mock(return_value=[]), + http_bridge_upstream_events_module, + "_ACCOUNT_SELECTION_RECOVERY_DEFAULT_SLEEP_SECONDS", + 0.001, ) monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: _make_app_settings(), + http_bridge_upstream_events_module, + "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", + 0.001, ) - async def _unexpected_owner_lookup(*args: object, **kwargs: object) -> str: - raise AssertionError("live local session reuse must not hit the ring") - - monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", _unexpected_owner_lookup) - monkeypatch.setattr(proxy_service, "_active_http_bridge_instance_ring", _unexpected_owner_lookup) + request_state = proxy_service._WebSocketRequestState( + request_id="req-model-capacity-propagate", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + bridge_request_deadline=time.monotonic() + 60.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + transport="http", + propagate_http_errors=True, + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + ) + session = _make_bridge_session( + key_value="bridge-model-capacity-propagate", + pending_requests=deque([request_state]), + queued_request_count=1, + ) - reused = await service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-key"), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "invalid_request_error", + "message": "Selected model is at capacity. Please try a different model.", + }, + }, + separators=(",", ":"), + ), ) - assert reused is existing - assert reused.request_model == "gpt-5.4" - assert reused.last_used_at > 1.0 + assert request_state.event_queue is not None + assert request_state.event_queue.empty() + retry_precreated.assert_awaited_once_with(session, request_state=request_state) @pytest.mark.asyncio -async def test_get_or_create_http_bridge_session_preserves_closed_admission_handoff( +async def test_http_bridge_model_capacity_wait_hides_keepalive_for_non_sdk_propagated_streams( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = proxy_service._HTTPBridgeSessionKey("session_header", "bridge-handoff", None) - existing = _make_bridge_session(key_value="bridge-handoff") - existing.key = key - existing.request_model = "gpt-5.4" - existing.closed = True - existing.admission_waiter_count = 1 - service._http_bridge_sessions[key] = existing - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - create = AsyncMock() - monkeypatch.setattr(service, "_create_http_bridge_session", create) + retry_precreated = AsyncMock(return_value=True) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_ACCOUNT_SELECTION_RECOVERY_DEFAULT_SLEEP_SECONDS", + 0.001, + ) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", + 0.001, + ) - resolved = await service._get_or_create_http_bridge_session( - key, - headers={"x-codex-session-id": "bridge-handoff"}, - affinity=proxy_service._AffinityPolicy( - key="bridge-handoff", - kind=proxy_service.StickySessionKind.CODEX_SESSION, + request_state = proxy_service._WebSocketRequestState( + request_id="req-model-capacity-propagate-non-sdk", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + bridge_request_deadline=time.monotonic() + 60.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + transport="http", + propagate_http_errors=True, + enforce_openai_sdk_contract=False, + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + ) + session = _make_bridge_session( + key_value="bridge-model-capacity-propagate-non-sdk", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "invalid_request_error", + "message": "Selected model is at capacity. Please try a different model.", + }, + }, + separators=(",", ":"), ), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, ) - assert resolved is existing - assert service._http_bridge_sessions[key] is existing - assert existing.request_model == "gpt-5.4" - create.assert_not_awaited() + assert request_state.event_queue is not None + assert request_state.event_queue.empty() + retry_precreated.assert_awaited_once_with(session, request_state=request_state) @pytest.mark.asyncio -async def test_get_or_create_http_bridge_session_rejects_anchored_incompatible_closed_admission_handoff( +async def test_http_bridge_model_capacity_wait_does_not_retry_after_deadline( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = proxy_service._HTTPBridgeSessionKey("session_header", "bridge-handoff", None) - existing = _make_bridge_session(key_value="bridge-handoff") - existing.key = key - existing.closed = True - existing.admission_waiter_count = 1 - service._http_bridge_sessions[key] = existing - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - create = AsyncMock() - monkeypatch.setattr(service, "_create_http_bridge_session", create) + retry_precreated = AsyncMock(return_value=True) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - with pytest.raises(proxy_service.ProxyResponseError) as exc_info: - await service._get_or_create_http_bridge_session( - key, - headers={"x-codex-session-id": "bridge-handoff"}, - affinity=proxy_service._AffinityPolicy(key="bridge-handoff"), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - preferred_account_id="different-account", - previous_response_id="resp-anchored", - ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-model-capacity-deadline", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + bridge_request_deadline=time.monotonic() - 0.001, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + transport="http", + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + ) + session = _make_bridge_session( + key_value="bridge-model-capacity-deadline", + pending_requests=deque([request_state]), + queued_request_count=1, + ) - assert exc_info.value.status_code == 503 - assert service._http_bridge_sessions[key] is existing - create.assert_not_awaited() + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "invalid_request_error", + "message": "Selected model is at capacity. Please try a different model.", + }, + }, + separators=(",", ":"), + ), + ) + + retry_precreated.assert_not_awaited() + assert request_state not in session.pending_requests + assert session.queued_request_count == 0 + assert request_state.event_queue is not None + terminal_block = await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) + assert terminal_block is not None + terminal = proxy_service.parse_sse_data_json(terminal_block) + assert isinstance(terminal, dict) + assert terminal["type"] == "response.failed" + assert await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) is None @pytest.mark.asyncio -async def test_get_or_create_http_bridge_session_recovers_unanchored_closed_admission_handoff( +async def test_http_bridge_model_capacity_wait_skips_sleep_for_non_replayable_request( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = proxy_service._HTTPBridgeSessionKey("session_header", "bridge-handoff", None) - existing = _make_bridge_session(key_value="bridge-handoff") - existing.key = key - existing.closed = True - existing.admission_waiter_count = 1 - service._http_bridge_sessions[key] = existing - replacement = _make_bridge_session(key_value="bridge-handoff") - replacement.key = key - settings = _make_app_settings() - monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + wait_before_retry = AsyncMock(return_value=True) + retry_precreated = AsyncMock(return_value=False) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) monkeypatch.setattr( - proxy_service, - "_http_bridge_owner_instance", - AsyncMock(return_value=settings.http_responses_session_bridge_instance_id), + http_bridge_upstream_events_module, + "_wait_before_http_bridge_model_capacity_retry", + wait_before_retry, ) - monkeypatch.setattr( - http_bridge_mixin_module, - "_http_bridge_owner_instance", - AsyncMock(return_value=settings.http_responses_session_bridge_instance_id), + + request_state = proxy_service._WebSocketRequestState( + request_id="req-model-capacity-not-replayable", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + bridge_request_deadline=time.monotonic() + 60.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + transport="http", + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + replay_count=1, + ) + session = _make_bridge_session( + key_value="bridge-model-capacity-not-replayable", + pending_requests=deque([request_state]), + queued_request_count=1, ) - monkeypatch.setattr(http_bridge_mixin_module, "_http_bridge_owner_check_required", lambda *args, **kwargs: False) - monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) - monkeypatch.setattr(service, "_schedule_http_bridge_session_closes", Mock()) - create = AsyncMock(return_value=replacement) - monkeypatch.setattr(service, "_create_http_bridge_session", create) - resolved = await service._get_or_create_http_bridge_session( - key, - headers={"x-codex-session-id": "bridge-handoff"}, - affinity=proxy_service._AffinityPolicy(key="bridge-handoff"), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - preferred_account_id="different-account", + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "invalid_request_error", + "message": "Selected model is at capacity. Please try a different model.", + }, + }, + separators=(",", ":"), + ), ) - assert resolved is replacement - assert service._http_bridge_sessions[key] is replacement - assert existing.closed is True - create.assert_awaited_once() + wait_before_retry.assert_not_awaited() + retry_precreated.assert_not_awaited() @pytest.mark.asyncio -async def test_get_or_create_http_bridge_session_replaces_routing_unavailable_account( +async def test_http_bridge_model_capacity_wait_does_not_delay_anchored_errors( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = proxy_service._HTTPBridgeSessionKey("request", "bridge-routing-unavailable", None) - stale_session = proxy_service._HTTPBridgeSession( - key=key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-routing-unavailable"), - request_model="gpt-5.4-mini", - account=cast(Any, SimpleNamespace(id="acc-unavailable", status=AccountStatus.ACTIVE, plan_type="plus")), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, + retry_precreated = AsyncMock(return_value=True) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_ACCOUNT_SELECTION_RECOVERY_DEFAULT_SLEEP_SECONDS", + 60.0, ) - replacement_session = proxy_service._HTTPBridgeSession( - key=key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-routing-unavailable"), - request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id="acc-fresh", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=2.0, - idle_ttl_seconds=120.0, + + request_state = proxy_service._WebSocketRequestState( + request_id="req-model-capacity-anchored", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + bridge_request_deadline=time.monotonic() + 60.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + transport="http", + previous_response_id="resp_anchor", + request_text='{"type":"response.create","model":"gpt-5.6-sol","previous_response_id":"resp_anchor"}', + ) + session = _make_bridge_session( + key_value="bridge-model-capacity-anchored", + pending_requests=deque([request_state]), + queued_request_count=1, ) - service._http_bridge_sessions[key] = stale_session - monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) - monkeypatch.setattr(service, "_create_http_bridge_session", AsyncMock(return_value=replacement_session)) - monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) - close_session = AsyncMock() - monkeypatch.setattr(service, "_close_http_bridge_session", close_session) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - mark_account_routing_unavailable("acc-unavailable") - try: - reused = await service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-routing-unavailable"), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - ) - finally: - clear_account_routing_unavailable("acc-unavailable") + started = time.monotonic() + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "invalid_request_error", + "message": "Selected model is at capacity. Please try a different model.", + }, + }, + separators=(",", ":"), + ), + ) - assert reused is replacement_session - assert service._http_bridge_sessions[key] is replacement_session - assert stale_session.closed is True - await _wait_for_close_await(close_session, stale_session) + assert time.monotonic() - started < 1.0 + retry_precreated.assert_not_awaited() + assert request_state.event_queue is not None + terminal_block = await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) + assert terminal_block is not None + terminal = proxy_service.parse_sse_data_json(terminal_block) + assert isinstance(terminal, dict) + assert terminal["type"] == "response.failed" @pytest.mark.asyncio -async def test_close_http_bridge_sessions_for_account_detaches_matching_sessions( +async def test_http_bridge_precreated_completed_terminal_falls_back_to_unresolved_request( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - matching = _make_bridge_session( - key=proxy_service._HTTPBridgeSessionKey("session_header", "bridge-matching", None), - key_value="bridge-matching", - ) - matching.account.id = "acc-close" - other = _make_bridge_session( - key=proxy_service._HTTPBridgeSessionKey("session_header", "bridge-other", None), - key_value="bridge-other", + monkeypatch.setattr( + http_bridge_retry_circuit_module, + "_HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD", + 1, ) - other.account.id = "acc-other" - service._http_bridge_sessions[matching.key] = matching - service._http_bridge_sessions[other.key] = other - close_session = AsyncMock() - monkeypatch.setattr(service, "_close_http_bridge_session_bounded", close_session) - - closed = await service.close_http_bridge_sessions_for_account("acc-close") - - assert closed == 1 - assert matching.key not in service._http_bridge_sessions - assert service._http_bridge_sessions[other.key] is other - assert matching.closed is True - close_session.assert_awaited_once_with(matching, reason="account_binding_changed") - + finalize = AsyncMock() + register_previous = AsyncMock() + monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize) + monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", register_previous) -def test_http_bridge_request_text_replaces_client_installation_id() -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - session.account.codex_installation_id = "account-installation" - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "", - "input": [{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}], - "client_metadata": { - "x-codex-installation-id": "client-installation", - "x-codex-turn-metadata": '{"installation_id":"client-installation","turn_id":"payload-turn"}', - }, - } - ) - request_state, text_data = service._prepare_http_bridge_request( - payload, - {}, - api_key=None, + request_state = proxy_service._WebSocketRequestState( + request_id="req-precreated-completed", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + event_queue=asyncio.Queue(), + transport="http", ) - request_state.fresh_upstream_request_text = json.dumps( - { - "type": "response.create", - "model": "gpt-5.4", - "input": [], - "client_metadata": {"x-codex-installation-id": "client-replay"}, - }, - separators=(",", ":"), + session = _make_bridge_session( + key_value="bridge-precreated-completed", + pending_requests=deque([request_state]), + queued_request_count=1, ) + await service._record_http_bridge_retry_circuit_failure(session, detail="stream_incomplete") + retry_circuits = cast(Any, service)._http_bridge_retry_circuits + assert session.key in retry_circuits - updated_text = service._http_bridge_text_with_account_installation_id(session, request_state, text_data) - - assert json.loads(updated_text)["client_metadata"] == { - "x-codex-installation-id": "account-installation", - "x-codex-turn-metadata": '{"installation_id":"account-installation","turn_id":"payload-turn"}', - } - assert request_state.fresh_upstream_request_text is not None - assert json.loads(request_state.fresh_upstream_request_text)["client_metadata"] == { - "x-codex-installation-id": "account-installation", - } + await service._process_http_bridge_upstream_text( + session, + json.dumps({"type": "response.output_text.delta", "delta": "legacy text"}), + ) + await service._process_http_bridge_upstream_text( + session, + json.dumps({"type": "response.output_text.done", "text": "legacy text"}), + ) + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_precreated_completed", + "object": "response", + "status": "completed", + "output": [], + }, + } + ), + ) + + assert request_state.event_queue is not None + blocks: list[str | None] = [] + while True: + block = await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) + blocks.append(block) + if block is None: + break + + payloads: list[dict[str, Any]] = [] + for block in blocks: + if block is None: + continue + payload = proxy_service.parse_sse_data_json(block) + assert isinstance(payload, dict) + payloads.append(payload) + assert [payload["type"] for payload in payloads] == [ + "response.output_text.delta", + "response.output_text.done", + "response.completed", + ] + assert request_state.response_id == "resp_precreated_completed" + assert session.last_completed_response_id == "resp_precreated_completed" + assert session.last_completed_response_account_id == session.account.id + assert session.queued_request_count == 0 + assert not session.pending_requests + assert session.key not in retry_circuits + register_previous.assert_awaited_once() + finalize.assert_awaited_once() -def test_http_bridge_request_text_rejects_installation_metadata_size_overflow( +@pytest.mark.asyncio +async def test_recovery_completed_alias_persistence_failure_fails_response_and_retires_lane( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - session.account.codex_installation_id = "account-installation" request_state = proxy_service._WebSocketRequestState( - request_id="req-http-installation-size", - model="gpt-5.4", + request_id="req-recovery-completed", + response_id="resp_recovery_completed", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), transport="http", - request_text='{"type":"response.create","input":"x"}', + skip_request_log=True, ) - stamped_text = service._http_bridge_text_with_account_installation_id( - session, - request_state, - request_state.request_text or "{}", + request_state.operation_id = "op-alias-persistence-failure" + session = _make_bridge_session( + key=_make_account_neutral_replay_session_key("completed-alias-failure"), + pending_requests=deque([request_state]), + queued_request_count=1, + ) + register_previous = AsyncMock(return_value=False) + finalize = AsyncMock() + operation_updates = AsyncMock() + persist_operation_event = AsyncMock() + close_session = AsyncMock() + monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", register_previous) + monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize) + monkeypatch.setattr(service, "_close_http_bridge_session", close_session) + monkeypatch.setattr(http_bridge_upstream_events_module, "_update_http_bridge_operation_state", operation_updates) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_persist_http_bridge_operation_event", + persist_operation_event, ) - max_bytes = len(stamped_text.encode("utf-8")) - 1 - request_state.request_text = '{"type":"response.create","input":"x"}' - assert len((request_state.request_text or "").encode("utf-8")) < max_bytes - - monkeypatch.setattr(proxy_service, "_UPSTREAM_RESPONSE_CREATE_WARN_BYTES", max_bytes + 1, raising=False) - monkeypatch.setattr(proxy_service, "_UPSTREAM_RESPONSE_CREATE_MAX_BYTES", max_bytes, raising=False) - - with pytest.raises(proxy_service.ProxyResponseError) as exc_info: - service._http_bridge_text_with_account_installation_id( - session, - request_state, - request_state.request_text or "{}", - ) - - assert exc_info.value.status_code == 400 - assert exc_info.value.payload["error"]["code"] == "payload_too_large" + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_recovery_completed", + "object": "response", + "status": "completed", + "output": [], + }, + } + ), + ) -def test_submit_http_bridge_request_uses_bridge_installation_metadata_helper() -> None: - source = inspect.getsource(proxy_service.ProxyService._submit_http_bridge_request_with_handoff) + assert request_state.event_queue is not None + event_block = await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) + assert isinstance(event_block, str) + failed = proxy_service.parse_sse_data_json(event_block) + assert failed is not None + assert failed["type"] == "response.failed" + failed_response = failed.get("response") + assert isinstance(failed_response, dict) + failed_error = failed_response.get("error") + assert isinstance(failed_error, dict) + assert failed_error["code"] == "bridge_continuity_persistence_failed" + assert await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) is None + assert session.last_completed_response_id is None + assert session.upstream_control.reconnect_requested is True + assert session.upstream_control.retire_after_drain is True + finalize.assert_awaited_once() + finalize_call = finalize.await_args + assert finalize_call is not None + assert finalize_call.kwargs["event_type"] == "response.failed" + operation_updates.assert_awaited_once() + assert operation_updates.await_args is not None + assert operation_updates.await_args.kwargs["state"] == "acknowledged" + persist_operation_event.assert_awaited_once() + assert persist_operation_event.await_args is not None + assert persist_operation_event.await_args.kwargs["terminal_state"] == "acknowledged" - assert "_response_create_text_with_account_installation_id(" not in source - assert source.count("_http_bridge_text_with_account_installation_id(") >= 3 + assert await service._retire_http_bridge_after_drain_if_ready(session) is True + close_session.assert_awaited_once_with(session) @pytest.mark.asyncio -async def test_get_or_create_http_bridge_session_skips_prune_when_pending_lock_is_wedged( +async def test_http_bridge_incomplete_event_terminalizes_operation( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = proxy_service._HTTPBridgeSessionKey("request", "bridge-wedged-idle", None) - existing_session = _make_bridge_session(key=key, key_value="bridge-wedged-idle") - existing_session.last_used_at = time.monotonic() - 300.0 - existing_session.idle_ttl_seconds = 1.0 - service._http_bridge_sessions[key] = existing_session - lock_acquired = asyncio.Event() - release_lock = asyncio.Event() - - async def hold_pending_lock() -> None: - async with existing_session.pending_lock: - lock_acquired.set() - await release_lock.wait() - - lock_holder = asyncio.create_task(hold_pending_lock()) - await asyncio.wait_for(lock_acquired.wait(), timeout=1.0) - - create_http_bridge_session = AsyncMock() - monkeypatch.setattr(service, "_create_http_bridge_session", create_http_bridge_session) - monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) - close_http_bridge_session = AsyncMock() - monkeypatch.setattr(service, "_close_http_bridge_session", close_http_bridge_session) - monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: _make_app_settings(), + request_state = proxy_service._WebSocketRequestState( + request_id="req-incomplete-terminal", + response_id="resp-incomplete-terminal", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + skip_request_log=True, + ) + request_state.operation_id = "op-incomplete-terminal" + session = _make_bridge_session( + key_value="incomplete-terminal", + pending_requests=deque([request_state]), + queued_request_count=1, ) + operation_updates = AsyncMock() + finalize = AsyncMock() + monkeypatch.setattr(http_bridge_upstream_events_module, "_update_http_bridge_operation_state", operation_updates) + monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize) - try: - resolved = await asyncio.wait_for( - service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-wedged-idle"), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - ), - timeout=1.0, - ) - finally: - release_lock.set() - await asyncio.wait_for(lock_holder, timeout=1.0) + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "response.incomplete", + "response": { + "id": "resp-incomplete-terminal", + "object": "response", + "status": "incomplete", + "output": [], + }, + } + ), + ) - assert resolved is existing_session - assert existing_session.closed is False - assert service._http_bridge_sessions[key] is existing_session - close_http_bridge_session.assert_not_awaited() - create_http_bridge_session.assert_not_awaited() + operation_updates.assert_awaited_once() + assert operation_updates.await_args is not None + assert operation_updates.await_args.kwargs["state"] == "incomplete" + finalize.assert_awaited_once() @pytest.mark.asyncio -async def test_prune_http_bridge_session_skips_wedged_session_with_visible_pending_request() -> None: +async def test_http_bridge_batched_terminal_state_precedes_spool_finalize( + monkeypatch: pytest.MonkeyPatch, +) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = proxy_service._HTTPBridgeSessionKey("request", "bridge-wedged-visible", None) request_state = proxy_service._WebSocketRequestState( - request_id="req-wedged-visible", - model="gpt-5.4", + request_id="req-batched-terminal-order", + response_id="resp-batched-terminal-order", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), event_queue=asyncio.Queue(), transport="http", + skip_request_log=True, ) + request_state.operation_id = "op-batched-terminal-order" session = _make_bridge_session( - key=key, - key_value="bridge-wedged-visible", + key_value="batched-terminal-order", pending_requests=deque([request_state]), queued_request_count=1, ) - session.last_used_at = time.monotonic() - 300.0 - session.idle_ttl_seconds = 1.0 - service._http_bridge_sessions[key] = session - lock_acquired = asyncio.Event() - release_lock = asyncio.Event() + session.durable_session_id = "durable-batched-terminal-order" + session.durable_owner_epoch = 1 + order: list[str] = [] - async def hold_pending_lock() -> None: - async with session.pending_lock: - lock_acquired.set() - await release_lock.wait() + async def update_state(*args: Any, **kwargs: Any) -> None: + del args, kwargs + order.append("state") - lock_holder = asyncio.create_task(hold_pending_lock()) - await asyncio.wait_for(lock_acquired.wait(), timeout=1.0) - try: - async with service._http_bridge_lock: - sessions_to_close = service._prune_http_bridge_sessions_locked() - finally: - release_lock.set() - await asyncio.wait_for(lock_holder, timeout=1.0) + async def append_terminal_event(*args: Any, **kwargs: Any) -> bool: + del args + assert kwargs["session_id"] == session.durable_session_id + order.append("terminal") + return True - assert sessions_to_close == [] - assert service._http_bridge_sessions[key] is session - assert session.closed is False + monkeypatch.setattr(http_bridge_upstream_events_module, "_update_http_bridge_operation_state", update_state) + service._http_bridge_operation_event_batcher = cast( + Any, + SimpleNamespace(append_terminal_event=append_terminal_event), + ) + await http_bridge_upstream_events_module._persist_http_bridge_operation_event( + service, + session, + request_state, + 'data: {"type":"response.completed"}\n\n', + terminal=True, + terminal_state="completed", + ) -@pytest.mark.asyncio -async def test_get_or_create_http_bridge_session_replaces_live_session_when_account_is_no_longer_assigned( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = proxy_service._HTTPBridgeSessionKey("request", "bridge-key", "key-1") - stale_session = proxy_service._HTTPBridgeSession( - key=key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-key"), - request_model="gpt-5.4-mini", - account=cast(Any, SimpleNamespace(id="acc-stale", status=AccountStatus.ACTIVE, plan_type="plus")), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, - ) - replacement_session = proxy_service._HTTPBridgeSession( - key=key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-key"), - request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id="acc-fresh", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=2.0, - idle_ttl_seconds=120.0, - ) - service._http_bridge_sessions[key] = stale_session - monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) - monkeypatch.setattr( - service, - "_create_http_bridge_session", - AsyncMock(return_value=replacement_session), - ) - monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) - monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: _make_app_settings(), - ) - monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) - monkeypatch.setattr( - proxy_service, - "_active_http_bridge_instance_ring", - AsyncMock(return_value=("instance-a", ["instance-a"])), - ) - close_session = AsyncMock() - monkeypatch.setattr(service, "_close_http_bridge_session", close_session) - - reused = await service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-key"), - api_key=_make_api_key(key_id="key-1", assigned_account_ids=["acc-fresh"]), - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - ) - - assert reused is replacement_session - assert service._http_bridge_sessions[key] is replacement_session - assert stale_session.closed is True - await _wait_for_close_await(close_session, stale_session) + assert order == ["terminal"] @pytest.mark.asyncio -async def test_get_or_create_http_bridge_session_replaces_prompt_cache_session_promoted_to_codex( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_terminal_append_failure_retains_last_persisted_response_id_after_retry_reset() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-key", "key-1") - stale_session = proxy_service._HTTPBridgeSession( - key=key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-key"), - request_model="gpt-5.4-mini", - account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE, plan_type="plus")), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, - codex_session=True, - downstream_turn_state="http_turn_legacy", - downstream_turn_state_aliases={"http_turn_legacy"}, - previous_response_ids=set(), - ) - replacement_session = proxy_service._HTTPBridgeSession( - key=key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-key"), - request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=2.0, - idle_ttl_seconds=120.0, + request_state = SimpleNamespace( + operation_id="op-terminal-retry-reset", + operation_attempt_generation=0, + operation_persisted_response_id=None, + request_id="req-terminal-retry-reset", + response_id="resp-before-retry", + replay_downstream_response_id=None, ) - service._http_bridge_sessions[key] = stale_session - monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) - monkeypatch.setattr( + session = _make_bridge_session(key_value="terminal-retry-reset") + session.durable_session_id = "durable-terminal-retry-reset" + session.durable_owner_epoch = 2 + service._durable_bridge = cast(Any, SimpleNamespace(update_operation=AsyncMock(return_value=True))) + + await http_bridge_upstream_events_module._update_http_bridge_operation_state( service, - "_create_http_bridge_session", - AsyncMock(return_value=replacement_session), - ) - monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) - monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: _make_app_settings(), + session, + request_state, + state="acknowledged", + response_id="resp-before-retry", ) - monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) - monkeypatch.setattr( - proxy_service, - "_active_http_bridge_instance_ring", - AsyncMock(return_value=("instance-a", ["instance-a"])), + assert request_state.operation_persisted_response_id == "resp-before-retry" + + request_state.response_id = None + settle_terminal_event = AsyncMock() + service._http_bridge_operation_event_batcher = cast( + Any, + SimpleNamespace( + append_terminal_event=AsyncMock( + return_value=TerminalOperationEventAppendResult(persisted=False, settlement_required=True) + ), + settle_terminal_event=settle_terminal_event, + ), ) - close_session = AsyncMock() - monkeypatch.setattr(service, "_close_http_bridge_session", close_session) - reused = await service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-key"), - api_key=_make_api_key(key_id="key-1", assigned_account_ids=["acc-1"], account_assignment_scope_enabled=True), - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, + await http_bridge_upstream_events_module._persist_http_bridge_operation_event( + service, + session, + request_state, + 'data: {"type":"response.failed"}\n\n', + terminal=True, + terminal_state="failed", ) - assert reused is replacement_session - assert service._http_bridge_sessions[key] is replacement_session - assert stale_session.closed is True - await _wait_for_close_await(close_session, stale_session) + assert settle_terminal_event.await_args is not None + assert settle_terminal_event.await_args.kwargs["expected_response_id"] == "resp-before-retry" @pytest.mark.asyncio -async def test_get_or_create_http_bridge_session_registers_turn_state_alias_without_rekeying_prompt_cache_session( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize( + ("upstream_response_id", "replay_response_id", "expected_response_id", "alternate_expected_response_id"), + [ + ( + "resp-terminal-append-fallback-order", + "resp-client-visible-replay", + "resp-terminal-append-fallback-order", + "resp-client-visible-replay", + ), + (None, "resp-persisted-before-replay", "resp-persisted-before-replay", None), + ], +) +async def test_terminal_append_failure_queues_before_stalled_fallback_settlement( + upstream_response_id: str | None, + replay_response_id: str, + expected_response_id: str, + alternate_expected_response_id: str | None, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - prompt_cache_key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-key", "key-1") - session = proxy_service._HTTPBridgeSession( - key=prompt_cache_key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-key"), - request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE, plan_type="plus")), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, - codex_session=False, - downstream_turn_state=None, - downstream_turn_state_aliases=set(), - previous_response_ids={"resp_prev_1"}, - ) - service._http_bridge_sessions[prompt_cache_key] = session - service._http_bridge_previous_response_index[ - proxy_service._http_bridge_previous_response_alias_key("resp_prev_1", "key-1") - ] = prompt_cache_key - monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) - monkeypatch.setattr( - proxy_service, - "_active_http_bridge_instance_ring", - AsyncMock(return_value=("instance-a", ["instance-a"])), + event_queue: asyncio.Queue[str | None] = asyncio.Queue() + request_state = proxy_service._WebSocketRequestState( + request_id="req-terminal-append-fallback-order", + response_id=upstream_response_id, + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=event_queue, + transport="http", + skip_request_log=True, ) - refresh_durable = AsyncMock() - monkeypatch.setattr(service, "_refresh_durable_http_bridge_session", refresh_durable) - - resolved = await service._get_or_create_http_bridge_session( - prompt_cache_key, - headers={"x-codex-turn-state": "http_turn_promoted"}, - affinity=proxy_service._AffinityPolicy(key="bridge-key"), - api_key=_make_api_key(key_id="key-1", assigned_account_ids=["acc-1"], account_assignment_scope_enabled=True), - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - previous_response_id="resp_prev_1", + request_state.operation_id = "op-terminal-append-fallback-order" + request_state.operation_attempt_generation = 2 + request_state.replay_downstream_response_id = replay_response_id + session = _make_bridge_session( + key_value="terminal-append-fallback-order", + pending_requests=deque([request_state]), + queued_request_count=1, ) - - assert resolved is session - assert session.key == prompt_cache_key - assert service._http_bridge_sessions[prompt_cache_key] is session - assert ( - service._http_bridge_previous_response_index[ - proxy_service._http_bridge_previous_response_alias_key("resp_prev_1", "key-1") - ] - == prompt_cache_key + session.durable_session_id = "durable-terminal-append-fallback-order" + session.durable_owner_epoch = 3 + append_started = asyncio.Event() + release_append = asyncio.Event() + settlement_started = asyncio.Event() + release_settlement = asyncio.Event() + settlement_finished = asyncio.Event() + append_kwargs: dict[str, Any] = {} + settlement_kwargs: dict[str, Any] = {} + + async def append_terminal_event(*args: Any, **kwargs: Any) -> TerminalOperationEventAppendResult: + del args + append_kwargs.update(kwargs) + append_started.set() + await release_append.wait() + return TerminalOperationEventAppendResult(persisted=False, settlement_required=True) + + async def settle_terminal_event(*args: Any, **kwargs: Any) -> None: + del args + settlement_kwargs.update(kwargs) + settlement_started.set() + await release_settlement.wait() + settlement_finished.set() + + service._http_bridge_operation_event_batcher = cast( + Any, + SimpleNamespace( + append_terminal_event=append_terminal_event, + settle_terminal_event=settle_terminal_event, + ), ) - assert ( - service._http_bridge_turn_state_index[ - proxy_service._http_bridge_turn_state_alias_key("http_turn_promoted", "key-1") - ] - == prompt_cache_key + event_block = 'data: {"type":"response.failed"}\n\n' + persist_task = asyncio.create_task( + http_bridge_upstream_events_module._persist_http_bridge_operation_event( + service, + session, + request_state, + event_block, + terminal=True, + terminal_state="failed", + terminal_event_queue=event_queue, + ) ) - refresh_durable.assert_awaited_once_with(session) + + await asyncio.wait_for(append_started.wait(), timeout=1.0) + persist_task.cancel() + assert persist_task.cancelling() + assert persist_task.done() is False + release_append.set() + await asyncio.wait_for(settlement_started.wait(), timeout=1.0) + assert append_kwargs["response_id"] == replay_response_id + assert append_kwargs["expected_recovery_dispatch_count"] == 2 + assert settlement_kwargs["expected_response_id"] == expected_response_id + assert settlement_kwargs["expected_recovery_dispatch_count"] == 2 + assert settlement_kwargs["alternate_expected_response_id"] == alternate_expected_response_id + assert settlement_kwargs["response_id"] == replay_response_id + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) == event_block + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) is None + assert event_queue.empty() + assert persist_task.done() is False + release_settlement.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(persist_task, timeout=1.0) + assert settlement_finished.is_set() @pytest.mark.asyncio -async def test_stream_via_http_bridge_turn_state_request_ignores_prompt_cache_owner_mismatch( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_terminal_append_failure_defers_cancellation_through_delivery_claim() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": "hello"} - ) + event_queue: asyncio.Queue[str | None] = asyncio.Queue() request_state = proxy_service._WebSocketRequestState( - request_id="req-hard-turn-state", - model="gpt-5.4", + request_id="req-terminal-delivery-claim-cancellation", + response_id="resp-terminal-delivery-claim-cancellation", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, - event_queue=asyncio.Queue(), + started_at=time.monotonic(), + event_queue=event_queue, transport="http", + skip_request_log=True, ) - event_queue = request_state.event_queue - assert event_queue is not None - await event_queue.put(None) + request_state.operation_id = "op-terminal-delivery-claim-cancellation" + session = _make_bridge_session( + key_value="terminal-delivery-claim-cancellation", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.durable_session_id = "durable-terminal-delivery-claim-cancellation" + session.durable_owner_epoch = 4 + append_started = asyncio.Event() + release_append = asyncio.Event() + settlement_started = asyncio.Event() + release_settlement = asyncio.Event() + settlement_finished = asyncio.Event() + completed_delivery_scope = proxy_support_module._HTTPBridgeCompletedDeliveryScope(active=True) - def fake_prepare( - _prepared_payload: proxy_service.ResponsesRequest, - _headers: dict[str, str] | Any, - *, - api_key: proxy_service.ApiKeyData | None, - api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, - request_id: str, - client_ip: str | None = None, - ) -> tuple[proxy_service._WebSocketRequestState, str]: - del api_key, api_key_reservation, request_id, client_ip - return request_state, '{"type":"response.create"}' + async def append_terminal_event(*args: Any, **kwargs: Any) -> TerminalOperationEventAppendResult: + del args, kwargs + append_started.set() + await release_append.wait() + return TerminalOperationEventAppendResult(persisted=False, settlement_required=True) - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_promoted", None), - headers={"x-codex-turn-state": "http_turn_promoted"}, - affinity=proxy_service._AffinityPolicy( - key="http_turn_promoted", - kind=proxy_service.StickySessionKind.CODEX_SESSION, + async def settle_terminal_event(*args: Any, **kwargs: Any) -> None: + del args, kwargs + settlement_started.set() + await release_settlement.wait() + settlement_finished.set() + + service._http_bridge_operation_event_batcher = cast( + Any, + SimpleNamespace( + append_terminal_event=append_terminal_event, + settle_terminal_event=settle_terminal_event, ), - request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, ) - captured_key: dict[str, object] = {} - captured_lookup: dict[str, object] = {} + event_block = 'data: {"type":"response.failed"}\n\n' + persist_task = asyncio.create_task( + http_bridge_upstream_events_module._persist_http_bridge_operation_event( + service, + session, + request_state, + event_block, + terminal=True, + terminal_state="failed", + terminal_event_queue=event_queue, + terminal_delivery_scope=completed_delivery_scope, + ) + ) - async def fake_get_or_create_http_bridge_session(*args: object, **kwargs: object): - captured_key["value"] = args[0] - captured_lookup["value"] = kwargs.get("durable_lookup") - return session + await asyncio.wait_for(append_started.wait(), timeout=1.0) + await session.pending_lock.acquire() + release_append.set() + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) == event_block + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) is None + persist_task.cancel() + assert persist_task.done() is False + session.pending_lock.release() + await asyncio.wait_for(settlement_started.wait(), timeout=1.0) + assert completed_delivery_scope.terminal_enqueued is True + assert persist_task.done() is False + release_settlement.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(persist_task, timeout=1.0) + assert settlement_finished.is_set() - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), - ), + +@pytest.mark.asyncio +async def test_terminal_append_success_queues_output_before_preserving_cancellation() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + event_queue: asyncio.Queue[str | None] = asyncio.Queue() + request_state = proxy_service._WebSocketRequestState( + request_id="req-terminal-append-success-cancellation", + response_id="resp-terminal-append-success-cancellation", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=event_queue, + transport="http", + skip_request_log=True, ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr( - service._durable_bridge, - "lookup_request_targets", - AsyncMock( - return_value=proxy_service.DurableBridgeLookup( - session_id="durable-prompt-cache", - canonical_kind="prompt_cache", - canonical_key="cache-derived", - api_key_scope="__anonymous__", - account_id="acc-1", - owner_instance_id="instance-remote", - owner_epoch=1, - lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_promoted", - latest_response_id=None, - ) - ), + request_state.operation_id = "op-terminal-append-success-cancellation" + session = _make_bridge_session( + key_value="terminal-append-success-cancellation", + pending_requests=deque([request_state]), + queued_request_count=1, ) - monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create_http_bridge_session) - monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) - monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + session.durable_session_id = "durable-terminal-append-success-cancellation" + session.durable_owner_epoch = 4 + append_started = asyncio.Event() + release_append = asyncio.Event() + completed_delivery_scope = proxy_support_module._HTTPBridgeCompletedDeliveryScope(active=True) - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( - payload, - headers={"x-codex-turn-state": "http_turn_promoted"}, - codex_session_affinity=True, - propagate_http_errors=False, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, + async def append_terminal_event(*args: Any, **kwargs: Any) -> TerminalOperationEventAppendResult: + del args, kwargs + append_started.set() + await release_append.wait() + return TerminalOperationEventAppendResult(persisted=True, settlement_required=False) + + service._http_bridge_operation_event_batcher = cast( + Any, + SimpleNamespace(append_terminal_event=append_terminal_event), + ) + event_block = 'data: {"type":"response.completed"}\n\n' + persist_task = asyncio.create_task( + http_bridge_upstream_events_module._persist_http_bridge_operation_event( + service, + session, + request_state, + event_block, + terminal=True, + terminal_state="completed", + terminal_event_queue=event_queue, + terminal_delivery_scope=completed_delivery_scope, ) - ] + ) - assert chunks == [] - assert request_state.affinity_policy.key == "http_turn_promoted" - assert request_state.affinity_policy.kind == proxy_service.StickySessionKind.CODEX_SESSION - key = cast(proxy_service._HTTPBridgeSessionKey, captured_key["value"]) - assert key.affinity_kind == "prompt_cache" - assert key.affinity_key == "cache-derived" - lookup = cast(proxy_service.DurableBridgeLookup, captured_lookup["value"]) - assert lookup.canonical_kind == "prompt_cache" - assert lookup.canonical_key == "cache-derived" - assert lookup.owner_instance_id == "instance-remote" - assert lookup.lease_expires_at is not None + await asyncio.wait_for(append_started.wait(), timeout=1.0) + persist_task.cancel() + release_append.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(persist_task, timeout=1.0) + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) == event_block + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) is None + assert event_queue.empty() + assert completed_delivery_scope.terminal_enqueued is True @pytest.mark.asyncio -async def test_stream_via_http_bridge_durable_outage_does_not_reuse_stale_recovery_alias( +@pytest.mark.parametrize( + ("cancel_during_settlement", "finalizer_fails"), + [(False, True), (True, False), (True, True)], +) +async def test_grouped_terminal_fanout_queues_all_siblings_before_stalled_settlement( monkeypatch: pytest.MonkeyPatch, + cancel_during_settlement: bool, + finalizer_fails: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("stale-local-recovery")) - recovery.account = cast( - Any, - SimpleNamespace(id="acc-stale-recovery", status=AccountStatus.ACTIVE, plan_type="plus"), + queues: list[asyncio.Queue[str | None]] = [asyncio.Queue(), asyncio.Queue()] + request_states: list[proxy_service._WebSocketRequestState] = [] + for index, event_queue in enumerate(queues): + request_state = proxy_service._WebSocketRequestState( + request_id=f"req-grouped-terminal-{index}", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=event_queue, + transport="http", + previous_response_id="resp-shared-grouped-terminal", + skip_request_log=True, + ) + request_state.operation_id = f"op-grouped-terminal-{index}" + request_states.append(request_state) + session = _make_bridge_session( + key_value="grouped-terminal-fanout", + pending_requests=deque(request_states), + queued_request_count=2, ) - stale_turn_state = "http_turn_stale_recovery_owner" - recovery.downstream_turn_state = stale_turn_state - recovery.downstream_turn_state_aliases.add(stale_turn_state) - service._http_bridge_sessions[recovery.key] = recovery - alias_key = proxy_service._http_bridge_turn_state_alias_key(stale_turn_state, None) - service._http_bridge_turn_state_index[alias_key] = recovery.key - get_or_create = AsyncMock() - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), - ), + session.durable_session_id = "durable-grouped-terminal-fanout" + session.durable_owner_epoch = 5 + all_appends_started = asyncio.Event() + release_first_append = asyncio.Event() + settlement_started = asyncio.Event() + release_settlement = asyncio.Event() + append_calls: list[str] = [] + + async def append_terminal_event(*args: Any, **kwargs: Any) -> TerminalOperationEventAppendResult: + del args + operation_id = kwargs["operation_id"] + append_calls.append(operation_id) + if len(append_calls) == 2: + all_appends_started.set() + if operation_id == "op-grouped-terminal-0": + await release_first_append.wait() + return TerminalOperationEventAppendResult(persisted=False, settlement_required=True) + + async def settle_terminal_event(*args: Any, **kwargs: Any) -> None: + del args, kwargs + settlement_started.set() + await release_settlement.wait() + + finalize_request = AsyncMock( + side_effect=[RuntimeError("first sibling finalization failed"), None] if finalizer_fails else None ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr( - service._durable_bridge, - "lookup_request_targets", - AsyncMock(side_effect=RuntimeError("durable metadata unavailable")), + monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize_request) + monkeypatch.setattr(service, "_maybe_release_idle_http_bridge_session_lease", AsyncMock()) + service._http_bridge_operation_event_batcher = cast( + Any, + SimpleNamespace( + append_terminal_event=append_terminal_event, + settle_terminal_event=settle_terminal_event, + ), ) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) - payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": "continue"} + process_task = asyncio.create_task( + service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": "Previous response with id 'resp-shared-grouped-terminal' not found.", + "param": "previous_response_id", + }, + }, + separators=(",", ":"), + ), + ) ) - with pytest.raises(ProxyResponseError) as exc_info: - async for _ in service._stream_via_http_bridge( - payload, - headers={"x-codex-turn-state": stale_turn_state}, - codex_session_affinity=True, - propagate_http_errors=True, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - ): - pass - - assert exc_info.value.status_code == 502 - assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" - assert exc_info.value.payload["error"]["message"] == "HTTP bridge owner metadata unavailable; retry later." - get_or_create.assert_not_awaited() - assert service._http_bridge_turn_state_index[alias_key] == recovery.key + await asyncio.wait_for(all_appends_started.wait(), timeout=1.0) + assert append_calls == ["op-grouped-terminal-0", "op-grouped-terminal-1"] + assert all(event_queue.empty() for event_queue in queues) + release_first_append.set() + await asyncio.wait_for(settlement_started.wait(), timeout=1.0) + for event_queue in queues: + terminal_event = await asyncio.wait_for(event_queue.get(), timeout=1.0) + assert terminal_event is not None + assert '"type":"response.failed"' in terminal_event + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) is None + assert event_queue.empty() + assert process_task.done() is False + if cancel_during_settlement: + process_task.cancel() + assert process_task.cancelling() + release_settlement.set() + expected_error = asyncio.CancelledError if cancel_during_settlement else RuntimeError + with pytest.raises(expected_error): + await asyncio.wait_for(process_task, timeout=1.0) + assert append_calls == ["op-grouped-terminal-0", "op-grouped-terminal-1"] + assert finalize_request.await_count == 2 @pytest.mark.asyncio -async def test_stream_via_http_bridge_keeps_sse_alive_while_session_creation_waits_for_capacity( +async def test_ordinary_completed_alias_rejection_preserves_successful_response( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - settings = SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - session = _make_bridge_session(key_value="sid-capacity-create") - get_or_create = AsyncMock( - side_effect=[ - ProxyResponseError( - 503, - openai_error("no_accounts", "Rate limit exceeded. Try again in 120s"), - ), - session, - ] - ) request_state = proxy_service._WebSocketRequestState( - request_id="req-capacity-create", - model="gpt-5.4", + request_id="req-ordinary-completed", + response_id="resp_ordinary_completed", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic() - 10.0, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), transport="http", + skip_request_log=True, ) - - def fake_prepare( - _prepared_payload: proxy_service.ResponsesRequest, - _headers: dict[str, str] | Any, - *, - api_key: proxy_service.ApiKeyData | None, - api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, - request_id: str, - client_ip: str | None = None, - ) -> tuple[proxy_service._WebSocketRequestState, str]: - del api_key, api_key_reservation, request_id, client_ip - return request_state, '{"type":"response.create"}' - - async def fake_stream_events( - _session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, - queue_limit: int, - propagate_http_errors: bool, - downstream_turn_state: str | None, - request_deadline: float | None = None, - ): - del request_state, text_data, queue_limit, propagate_http_errors, downstream_turn_state, request_deadline - yield ( - 'data: {"type":"response.completed","response":{"id":"resp_capacity_create_ok",' - '"usage":{"input_tokens":1,"output_tokens":2}}}\n\n' - ) - - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), - ) - monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: _make_app_settings( - proxy_request_budget_seconds=0.001, - http_responses_session_bridge_request_budget_seconds=120.0, - ), + session = _make_bridge_session( + key_value="ordinary-completed-alias-rejection", + pending_requests=deque([request_state]), + queued_request_count=1, ) - monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 0.001) - monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 0.001) - monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) - monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) + register_previous = AsyncMock(return_value=False) + finalize = AsyncMock() + monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", register_previous) + monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize) - payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": [], "stream": True} + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_ordinary_completed", + "object": "response", + "status": "completed", + "output": [], + }, + } + ), ) - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( - payload, - headers={"session_id": "sid-capacity-create"}, - codex_session_affinity=True, - propagate_http_errors=False, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - ) - ] - - keepalive = proxy_service.parse_sse_data_json(chunks[0]) - completed = proxy_service.parse_sse_data_json(chunks[-1]) - - assert keepalive is not None + assert request_state.event_queue is not None + event_block = await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) + assert isinstance(event_block, str) + completed = proxy_service.parse_sse_data_json(event_block) assert completed is not None - assert keepalive["type"] == "codex.keepalive" - assert keepalive["status"] == "waiting_for_account_capacity" assert completed["type"] == "response.completed" - assert get_or_create.await_count == 2 - expected_deadline = request_state.started_at + 120.0 - assert get_or_create.await_args_list[0].kwargs["request_deadline"] == pytest.approx(expected_deadline) - assert get_or_create.await_args_list[1].kwargs["request_deadline"] == pytest.approx(expected_deadline) + assert await asyncio.wait_for(request_state.event_queue.get(), timeout=1.0) is None + assert session.last_completed_response_id == "resp_ordinary_completed" + assert session.last_completed_response_account_id == session.account.id + assert session.upstream_control.reconnect_requested is False + assert session.upstream_control.retire_after_drain is False + finalize.assert_awaited_once() + finalize_call = finalize.await_args + assert finalize_call is not None + assert finalize_call.kwargs["event_type"] == "response.completed" @pytest.mark.asyncio -async def test_stream_via_http_bridge_stops_session_creation_retry_after_budget_wait( +async def test_http_bridge_upstream_text_archives_with_request_archive_id( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - settings = SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - get_or_create = AsyncMock( - side_effect=ProxyResponseError( - 503, - openai_error("no_accounts", "Rate limit exceeded. Try again in 120s"), - ) - ) - now = 100.0 + archived: list[tuple[str | None, str]] = [] + request_state = proxy_service._WebSocketRequestState( - request_id="req-capacity-create-budget", - model="gpt-5.4", + request_id="req-bridge-archive", + model="gpt-5.2", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=now, + started_at=time.monotonic(), + archive_request_id="archive-bridge-archive", + awaiting_response_created=True, + event_queue=asyncio.Queue(), transport="http", + skip_request_log=True, + ) + session = _make_bridge_session( + key_value="bridge-archive", + pending_requests=deque([request_state]), + queued_request_count=1, ) - def fake_prepare( - _prepared_payload: proxy_service.ResponsesRequest, - _headers: dict[str, str] | Any, - *, - api_key: proxy_service.ApiKeyData | None, - api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, - request_id: str, - client_ip: str | None = None, - ) -> tuple[proxy_service._WebSocketRequestState, str]: - del api_key, api_key_reservation, request_id, client_ip - return request_state, '{"type":"response.create"}' - - async def fake_sleep(seconds: float) -> None: - nonlocal now - now += seconds + def archive_received(message: UpstreamWebSocketMessage) -> None: + archived.append((get_request_id(), message.text or "")) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), - ) - monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: _make_app_settings( - proxy_request_budget_seconds=1.0, - http_responses_session_bridge_request_budget_seconds=1.0, - ), - ) - monkeypatch.setattr( - http_bridge_streaming_module, - "_service_time", - lambda: SimpleNamespace(monotonic=lambda: now), + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(close=AsyncMock(), archive_received=archive_received), ) - monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", fake_sleep) - monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 120.0) - monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 120.0) - monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", AsyncMock()) - payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": [], "stream": True} + upstream_text = json.dumps( + { + "type": "response.created", + "response": {"id": "resp_bridge_archive", "status": "in_progress"}, + }, + separators=(",", ":"), ) - chunks: list[str] = [] - - with pytest.raises(ProxyResponseError): - async for chunk in service._stream_via_http_bridge( - payload, - headers={"session_id": "sid-capacity-create-budget"}, - codex_session_affinity=True, - propagate_http_errors=False, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - ): - chunks.append(chunk) - - keepalive = proxy_service.parse_sse_data_json(chunks[0]) - - assert keepalive is not None - assert keepalive["type"] == "codex.keepalive" - assert keepalive["status"] == "waiting_for_account_capacity" - assert get_or_create.await_count == 1 + await service._process_http_bridge_upstream_text(session, upstream_text) -def test_http_bridge_session_key_infers_strength_from_affinity_kind() -> None: - assert proxy_service._HTTPBridgeSessionKey("turn_state_header", "turn", None).strength == "hard" - assert proxy_service._HTTPBridgeSessionKey("session_header", "session", None).strength == "hard" - assert proxy_service._HTTPBridgeSessionKey("prompt_cache", "cache", None).strength == "soft" - assert proxy_service._HTTPBridgeSessionKey("request", "request", None).strength == "soft" + assert archived == [("archive-bridge-archive", upstream_text)] + assert request_state.response_id == "resp_bridge_archive" -def test_http_bridge_session_header_key_is_scoped_by_explicit_prompt_cache_key() -> None: - headers = {"session_id": "process-session"} - first = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.6-sol", - "instructions": "", - "input": [], - "prompt_cache_key": "parent-thread", - } - ) - child = first.model_copy(update={"prompt_cache_key": "child-thread"}) +@pytest.mark.asyncio +async def test_http_bridge_upstream_non_text_archives_with_request_archive_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + archived: list[tuple[str | None, str, int | None]] = [] - first_key = proxy_service._make_http_bridge_session_key( - first, - headers=headers, - affinity=proxy_service._AffinityPolicy(), - api_key=None, - request_id="request-1", - explicit_prompt_cache_key="parent-thread", + request_state = proxy_service._WebSocketRequestState( + request_id="req-bridge-close-archive", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + archive_request_id="archive-bridge-close", + awaiting_response_created=True, + event_queue=asyncio.Queue(), + transport="http", + skip_request_log=True, ) - same_thread_key = proxy_service._make_http_bridge_session_key( - first, - headers=headers, - affinity=proxy_service._AffinityPolicy(), - api_key=None, - request_id="request-2", - explicit_prompt_cache_key="parent-thread", + close_message = UpstreamWebSocketMessage(kind="close", close_code=1000) + session = _make_bridge_session( + key_value="bridge-close-archive", + pending_requests=deque([request_state]), + queued_request_count=1, ) - child_key = proxy_service._make_http_bridge_session_key( - child, - headers=headers, - affinity=proxy_service._AffinityPolicy(), - api_key=None, - request_id="request-3", - explicit_prompt_cache_key="child-thread", + + def archive_received(message: UpstreamWebSocketMessage) -> None: + archived.append((get_request_id(), message.kind, message.close_code)) + + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock(return_value=close_message), + close=AsyncMock(), + archive_received=archive_received, + ), ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", AsyncMock()) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", AsyncMock()) - assert first_key.affinity_kind == "session_header" - assert first_key == same_thread_key - assert first_key != child_key + await service._relay_http_bridge_upstream_messages(session) + assert archived == [("archive-bridge-close", "close", 1000)] + assert session.last_upstream_close_code == 1000 -def test_http_bridge_session_header_key_without_prompt_cache_key_stays_legacy_compatible() -> None: - payload = proxy_service.ResponsesRequest.model_validate({"model": "gpt-5.6-sol", "instructions": "", "input": []}) - key = proxy_service._make_http_bridge_session_key( - payload, - headers={"session_id": "legacy-session"}, - affinity=proxy_service._AffinityPolicy(), - api_key=None, - request_id="request-1", +@pytest.mark.asyncio +async def test_http_bridge_relay_publishes_live_rate_limit_events( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from app.core.usage import live_hub + + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-live-rate-limits") + session.account.chatgpt_account_id = "workspace-bridge-live-rate-limits" + rate_limit_text = ( + '{"type":"codex.rate_limits","rate_limits":{"primary":' + '{"used_percent":72,"window_minutes":300,"reset_at":1700000300}}}' + ) + messages = [ + UpstreamWebSocketMessage(kind="text", text=rate_limit_text), + UpstreamWebSocketMessage(kind="close", close_code=1000), + ] + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock(side_effect=messages), + close=AsyncMock(), + archive_received=lambda message: None, + ), ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_process_http_bridge_upstream_text", AsyncMock()) + monkeypatch.setattr(service, "_retire_http_bridge_after_drain_if_ready", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_fail_http_bridge_reader_and_maybe_retire", AsyncMock()) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", AsyncMock()) - assert key == proxy_service._HTTPBridgeSessionKey("session_header", "legacy-session", None) + captured: list[tuple[Any, str | None, str | None]] = [] + live_hub.register_live_usage_publisher( + lambda snapshot, *, account_id=None, chatgpt_account_id=None: captured.append( + (snapshot, account_id, chatgpt_account_id) + ) + ) + try: + await service._relay_http_bridge_upstream_messages(session) + finally: + live_hub.register_live_usage_publisher(None) + assert len(captured) == 1 + snapshot, account_id, chatgpt_account_id = captured[0] + assert (account_id, chatgpt_account_id) == (session.account.id, session.account.chatgpt_account_id) + assert snapshot.primary is not None + assert snapshot.primary.used_percent == pytest.approx(72.0) -def test_http_bridge_owner_check_required_keeps_prompt_cache_soft() -> None: - key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "cache", None) - assert proxy_service._http_bridge_owner_check_required(key, gateway_safe_mode=False) is False - assert proxy_service._http_bridge_owner_check_required(key, gateway_safe_mode=True) is False +def test_pop_terminal_websocket_request_state_precreated_completed_does_not_guess_with_ambiguous_pending() -> None: + draining = proxy_service._WebSocketRequestState( + request_id="req-draining", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + draining_until_terminal=True, + ) + visible = proxy_service._WebSocketRequestState( + request_id="req-visible", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + ) + pending = deque([draining, visible]) + popped = proxy_service._pop_terminal_websocket_request_state( + pending, + response_id="resp_ambiguous_precreated_completed", + fallback_request_state=None, + allow_precreated_terminal_fallback=True, + ) -def test_http_bridge_owner_check_required_enables_sticky_thread_in_gateway_safe_mode() -> None: - key = proxy_service._HTTPBridgeSessionKey("sticky_thread", "thread-key", None) + assert popped is None + assert list(pending) == [draining, visible] + assert draining.response_id is None + assert visible.response_id is None + + +def test_trim_http_bridge_previous_response_input_items_preserves_context_assistant_message() -> None: + items: list[proxy_service.JsonValue] = [ + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "local context"}]}, + {"role": "user", "content": [{"type": "input_text", "text": "next"}]}, + ] + + assert proxy_service._trim_http_bridge_previous_response_input_items(items) == items + + +def test_trim_http_bridge_previous_response_input_items_trims_marked_replay_outputs() -> None: + items: list[proxy_service.JsonValue] = [ + {"id": "rs_replay", "type": "reasoning", "summary": []}, + { + "id": "msg_replay", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "prior"}], + }, + { + "id": "fc_replay", + "type": "function_call", + "call_id": "call_1", + "name": "lookup", + "arguments": "{}", + }, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + {"role": "user", "content": [{"type": "input_text", "text": "next"}]}, + ] + + assert proxy_service._trim_http_bridge_previous_response_input_items(items) == items[3:] + + +def test_trim_http_bridge_previous_response_input_items_trims_marked_apply_patch_replay_outputs() -> None: + items: list[proxy_service.JsonValue] = [ + { + "id": "apc_replay", + "type": "apply_patch_call", + "status": "completed", + "call_id": "call_patch_1", + }, + {"type": "apply_patch_call_output", "call_id": "call_patch_1", "status": "completed", "output": "patched"}, + {"role": "user", "content": [{"type": "input_text", "text": "next"}]}, + ] + + assert proxy_service._trim_http_bridge_previous_response_input_items(items) == items[1:] - assert proxy_service._http_bridge_owner_check_required(key, gateway_safe_mode=False) is False - assert proxy_service._http_bridge_owner_check_required(key, gateway_safe_mode=True) is True + +def test_trim_http_bridge_previous_response_input_items_preserves_unmarked_call_context() -> None: + items: list[proxy_service.JsonValue] = [ + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "local context"}]}, + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + {"role": "user", "content": [{"type": "input_text", "text": "next"}]}, + ] + + assert proxy_service._trim_http_bridge_previous_response_input_items(items) == items @pytest.mark.asyncio -async def test_stream_via_http_bridge_replaces_retired_hard_gate_before_submit( +async def test_http_bridge_stream_masks_single_top_level_previous_response_error( monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.6-sol", - "instructions": "hi", - "input": "continue", - "previous_response_id": "resp-before-retired-gate", - } + monkeypatch.setattr(service, "_finalize_websocket_request_state", AsyncMock()) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-single-prev", None), + headers={"session_id": "sid-single-prev"}, + affinity=proxy_service._AffinityPolicy( + key="sid-single-prev", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + request_model="gpt-5.1", + account=cast(Any, SimpleNamespace(id="acc-single-prev", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=1.0, + idle_ttl_seconds=120.0, ) - retired_session = _make_bridge_session(key_value="sid-retired-gate-replace") - replacement_session = _make_bridge_session(key_value="sid-retired-gate-replace") - get_or_create = AsyncMock(side_effect=[retired_session, replacement_session]) request_state = proxy_service._WebSocketRequestState( - request_id="req-retired-gate-replace", - model="gpt-5.6-sol", + request_id="req-single-prev", + model="gpt-5.1", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - transport="http", - request_text=( - '{"type":"response.create","model":"gpt-5.6-sol","previous_response_id":"resp-before-retired-gate"}' - ), - previous_response_id="resp-before-retired-gate", event_queue=asyncio.Queue(), + transport="http", + previous_response_id="resp_missing_single", + ) + upstream_text = json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": "Previous response with id 'resp_missing_single' not found.", + "param": "previous_response_id", + }, + }, + separators=(",", ":"), ) - gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( - "http_bridge_response_create_gate", - code="response_create_gate_timeout", - ) - - def fake_prepare( - _prepared_payload: proxy_service.ResponsesRequest, - _headers: dict[str, str] | Any, - **_kwargs: object, - ) -> tuple[proxy_service._WebSocketRequestState, str]: - return request_state, request_state.request_text or "{}" - async def fake_submit( - session: proxy_service._HTTPBridgeSession, + async def fake_submit_http_bridge_request( + target_session: proxy_service._HTTPBridgeSession, *, request_state: proxy_service._WebSocketRequestState, text_data: str, queue_limit: int, ) -> None: del text_data, queue_limit - if session is retired_session: - retired_session.closed = True - request_state.awaiting_response_created = False - request_state.response_create_gate = None - request_state.response_create_gate_acquired = False - raise gate_timeout_error - assert session is replacement_session - assert request_state.event_queue is not None - request_state.event_queue.put_nowait( - 'data: {"type":"response.completed","response":{"id":"resp-replaced-gate"}}\n\n' + target_session.pending_requests.append(request_state) + await service._process_http_bridge_upstream_text(target_session, upstream_text) + + monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) + + events = [ + event + async for event in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data="{}", + queue_limit=8, + propagate_http_errors=False, + downstream_turn_state=None, ) - request_state.event_queue.put_nowait(None) + ] - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), - ), + assert session.upstream_control.reconnect_requested is False + assert request_state.error_http_status_override == 502 + assert len(events) == 1 + event_block = events[0] + assert "previous_response_not_found" not in event_block + payload = proxy_service.parse_sse_data_json(event_block) + assert isinstance(payload, dict) + assert payload["type"] == "response.failed" + response = payload["response"] + assert isinstance(response, dict) + error = response["error"] + assert isinstance(error, dict) + assert error["code"] == "stream_incomplete" + + +@pytest.mark.asyncio +async def test_http_bridge_startup_cooldown_releases_api_key_reservation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-startup-reservation") + reservation = cast(Any, object()) + request_state = proxy_service._WebSocketRequestState( + request_id="req-startup-reservation", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=reservation, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + previous_response_id="resp-anchor", ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-bridge")) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) - submit = AsyncMock(side_effect=fake_submit) - detach = AsyncMock() - monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - monkeypatch.setattr(service, "_detach_http_bridge_request", detach) + cooldown = AsyncMock(return_value=30.0) + release = AsyncMock() + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", cooldown) + monkeypatch.setattr(service, "_release_websocket_request_state_reservation", release) - caplog.set_level(logging.INFO, logger="app.modules.proxy.service") - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( - payload, - headers={"session_id": "sid-retired-gate-replace"}, - codex_session_affinity=True, - propagate_http_errors=True, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, + events = [ + event + async for event in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, + propagate_http_errors=False, + downstream_turn_state=None, ) ] - assert chunks == ['data: {"type":"response.completed","response":{"id":"resp-replaced-gate"}}\n\n'] - assert get_or_create.await_count == 2 - initial_call, replacement_call = get_or_create.await_args_list - assert initial_call.args[0] == replacement_call.args[0] - assert replacement_call.kwargs["allow_forward_to_owner"] is False - assert replacement_call.kwargs["allow_previous_response_recovery_rebind"] is True - assert replacement_call.kwargs["preferred_account_id"] == retired_session.account.id - assert replacement_call.kwargs["fallback_on_preferred_account_unavailable"] is False - assert replacement_call.kwargs["request_deadline"] == initial_call.kwargs["request_deadline"] - assert submit.await_count == 2 - detach.assert_awaited_once_with(replacement_session, request_state=request_state) - assert "event=replace_retired_gate" in caplog.text + assert len(events) == 1 + assert '"code":"stream_idle_timeout"' in events[0] + release.assert_awaited_once_with(request_state) + assert request_state.api_key_reservation is None @pytest.mark.asyncio -async def test_stream_via_http_bridge_replaces_retired_hard_gate_excludes_stuck_account( +async def test_http_bridge_one_shot_hard_turn_waits_through_startup_cooldown( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Unlike a continuity (previous-response-owner) turn — which is - intentionally re-pinned to the same account via preferred_account_id — - a plain waiter's replacement session must exclude the account whose gate - session just proved stuck, or the load balancer could legally reselect - the exact same wedged account for the "replacement".""" service = proxy_service.ProxyService(cast(Any, nullcontext())) - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.6-sol", - "instructions": "hi", - "input": "continue", - } - ) - retired_session = _make_bridge_session(key_value="sid-retired-gate-exclude") - replacement_session = _make_bridge_session(key_value="sid-retired-gate-exclude") - get_or_create = AsyncMock(side_effect=[retired_session, replacement_session]) + session = _make_bridge_session(key_value="sid-hard-turn-cooldown-wait") request_state = proxy_service._WebSocketRequestState( - request_id="req-retired-gate-exclude", - model="gpt-5.6-sol", + request_id="req-hard-turn-cooldown-wait", + model="gpt-5.6", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - transport="http", - request_text='{"type":"response.create","model":"gpt-5.6-sol"}', event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-hard-anchor", + hard_continuity_anchor=True, ) - gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( - "http_bridge_response_create_gate", - code="response_create_gate_timeout", - ) - - def fake_prepare( - _prepared_payload: proxy_service.ResponsesRequest, - _headers: dict[str, str] | Any, - **_kwargs: object, - ) -> tuple[proxy_service._WebSocketRequestState, str]: - return request_state, request_state.request_text or "{}" + session.durable_session_id = "durable-hard-turn-cooldown-wait" + session.durable_owner_epoch = 7 + cooldown = AsyncMock(side_effect=[0.01, 0.0]) + submit = AsyncMock(side_effect=RuntimeError("submitted after cooldown")) + sleeps: list[float] = [] - async def fake_submit( - session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, - queue_limit: int, - ) -> None: - del text_data, queue_limit - if session is retired_session: - retired_session.closed = True - request_state.awaiting_response_created = False - request_state.response_create_gate = None - request_state.response_create_gate_acquired = False - raise gate_timeout_error - assert session is replacement_session - assert request_state.event_queue is not None - request_state.event_queue.put_nowait( - 'data: {"type":"response.completed","response":{"id":"resp-replaced-gate-exclude"}}\n\n' - ) - request_state.event_queue.put_nowait(None) + async def sleep(delay: float) -> None: + sleeps.append(delay) monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", ), ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) - submit = AsyncMock(side_effect=fake_submit) - detach = AsyncMock() + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", cooldown) monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - monkeypatch.setattr(service, "_detach_http_bridge_request", detach) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", sleep) - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( - payload, - headers={"session_id": "sid-retired-gate-exclude"}, - codex_session_affinity=True, + with pytest.raises(RuntimeError, match="submitted after cooldown"): + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, propagate_http_errors=True, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - ) - ] + downstream_turn_state="turn-state-hard-anchor", + ): + pass - assert chunks == ['data: {"type":"response.completed","response":{"id":"resp-replaced-gate-exclude"}}\n\n'] - assert get_or_create.await_count == 2 - _initial_call, replacement_call = get_or_create.await_args_list - assert replacement_call.kwargs["preferred_account_id"] is None - assert replacement_call.kwargs["exclude_account_ids"] == {retired_session.account.id} + assert sleeps == [pytest.approx(0.01)] + assert cooldown.await_count == 2 + submit.assert_awaited_once() @pytest.mark.asyncio -async def test_stream_via_http_bridge_replaces_retired_hard_gate_keeps_pinned_account_unexcluded( +async def test_http_bridge_previous_response_anchor_bypasses_hard_turn_cooldown_wait( monkeypatch: pytest.MonkeyPatch, ) -> None: - """A waiter whose replacement is already required to land on a specific - account (a resolved previous-response owner, or a file-pinned account — - simulated here directly via a pre-set preferred_account_id with no - previous_response_id) must keep that account, unexcluded, even though it - is the same account whose gate just proved stuck. Excluding a waiter's - own required account would make its required-account replacement - impossible and poison every later recovery call on the request.""" service = proxy_service.ProxyService(cast(Any, nullcontext())) - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.6-sol", - "instructions": "hi", - "input": "continue", - } - ) - retired_session = _make_bridge_session(key_value="sid-retired-gate-pinned") - replacement_session = _make_bridge_session(key_value="sid-retired-gate-pinned") - get_or_create = AsyncMock(side_effect=[retired_session, replacement_session]) + session = _make_bridge_session(key_value="sid-anchored-replay-cooldown-bypass") + session.durable_session_id = "durable-anchored-replay-cooldown-bypass" + session.durable_owner_epoch = 8 request_state = proxy_service._WebSocketRequestState( - request_id="req-retired-gate-pinned", - model="gpt-5.6-sol", + request_id="req-anchored-replay-cooldown-bypass", + model="gpt-5.6", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - transport="http", - request_text='{"type":"response.create","model":"gpt-5.6-sol"}', event_queue=asyncio.Queue(), - preferred_account_id=retired_session.account.id, - ) - gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( - "http_bridge_response_create_gate", - code="response_create_gate_timeout", + transport="http", + previous_response_id="resp-anchor-before-cooldown", + hard_continuity_anchor=True, ) - - def fake_prepare( - _prepared_payload: proxy_service.ResponsesRequest, - _headers: dict[str, str] | Any, - **_kwargs: object, - ) -> tuple[proxy_service._WebSocketRequestState, str]: - return request_state, request_state.request_text or "{}" - - async def fake_submit( - session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, - queue_limit: int, - ) -> None: - del text_data, queue_limit - if session is retired_session: - retired_session.closed = True - request_state.awaiting_response_created = False - request_state.response_create_gate = None - request_state.response_create_gate_acquired = False - raise gate_timeout_error - assert session is replacement_session - assert request_state.event_queue is not None - request_state.event_queue.put_nowait( - 'data: {"type":"response.completed","response":{"id":"resp-replaced-gate-pinned"}}\n\n' - ) - request_state.event_queue.put_nowait(None) + cooldown = AsyncMock(return_value=30.0) + submit = AsyncMock(side_effect=RuntimeError("submitted without cooldown wait")) + sleep = AsyncMock() monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", ), ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) - submit = AsyncMock(side_effect=fake_submit) - detach = AsyncMock() + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", cooldown) monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - monkeypatch.setattr(service, "_detach_http_bridge_request", detach) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", sleep) - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( - payload, - headers={"session_id": "sid-retired-gate-pinned"}, - codex_session_affinity=True, + with pytest.raises(RuntimeError, match="submitted without cooldown wait"): + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create","previous_response_id":"resp-anchor-before-cooldown"}', + queue_limit=8, propagate_http_errors=True, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - ) - ] + downstream_turn_state=None, + ): + pass - assert chunks == ['data: {"type":"response.completed","response":{"id":"resp-replaced-gate-pinned"}}\n\n'] - assert get_or_create.await_count == 2 - _initial_call, replacement_call = get_or_create.await_args_list - assert replacement_call.kwargs["preferred_account_id"] == retired_session.account.id - assert replacement_call.kwargs["exclude_account_ids"] is None + cooldown.assert_not_awaited() + sleep.assert_not_awaited() + submit.assert_awaited_once() @pytest.mark.asyncio -async def test_stream_via_http_bridge_soft_prompt_cache_queue_full_reroutes( +async def test_http_bridge_one_shot_hard_turn_without_durable_fence_fails_closed( monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "hi", - "input": "hello", - "prompt_cache_key": "soft-queue-full", - } - ) - saturated_session = _make_bridge_session(key_value="soft-queue-full", queued_request_count=8) - saturated_session.key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "soft-queue-full", None) - reroute_session = _make_bridge_session(key_value="soft-reroute") - capacity_unavailable = ProxyResponseError( - 503, - proxy_service.openai_error("no_accounts", "Rate limit exceeded. Try again in 120s"), + session = _make_bridge_session(key_value="sid-hard-turn-no-durable-fence") + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-no-durable-fence", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-without-durable-fence", + hard_continuity_anchor=True, ) - get_or_create = AsyncMock(side_effect=[saturated_session, capacity_unavailable, reroute_session]) - - async def fake_stream_events( - session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, - queue_limit: int, - propagate_http_errors: bool, - downstream_turn_state: str | None, - request_deadline: float | None = None, - ): - del request_state, text_data, queue_limit, propagate_http_errors, downstream_turn_state, request_deadline - if session is saturated_session: - raise ProxyResponseError( - 429, - proxy_service.openai_error( - "bridge_queue_full", - "HTTP responses session bridge queue is full", - error_type="rate_limit_error", - ), - ) - yield 'data: {"type":"response.completed"}\n\n' - + submit = AsyncMock() + sleep = AsyncMock() monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", ), ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) - monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) - monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 0.001) - monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 0.001) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=30.0)) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", sleep) - caplog.set_level(logging.INFO, logger="app.modules.proxy.service") - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( - payload, - headers={}, - codex_session_affinity=False, + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, propagate_http_errors=True, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - ) - ] + downstream_turn_state="turn-state-without-durable-fence", + ): + pass - assert chunks == ['data: {"type":"response.completed"}\n\n'] - assert get_or_create.await_count == 3 - reroute_key = get_or_create.await_args_list[1].args[0] - retry_reroute_key = get_or_create.await_args_list[2].args[0] - assert reroute_key.affinity_kind == "internal_soft_affinity_reroute" - assert reroute_key.strength == "soft" - assert retry_reroute_key.affinity_kind == "internal_soft_affinity_reroute" - assert retry_reroute_key.strength == "soft" - assert get_or_create.await_args_list[1].kwargs["previous_response_id"] is None - assert get_or_create.await_args_list[2].kwargs["previous_response_id"] is None - assert "internal_soft_affinity_reroute" in caplog.text + assert exc_info.value.status_code == 503 + assert exc_info.value.payload["error"]["code"] == "upstream_request_timeout" + submit.assert_not_awaited() + sleep.assert_not_awaited() @pytest.mark.asyncio -async def test_stream_via_http_bridge_file_pin_queue_full_does_not_reroute( +async def test_http_bridge_one_shot_hard_turn_requires_operation_ledger_for_cooldown_wait( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "hi", - "input": [{"type": "input_file", "file_id": "file_doc"}], - } + session = _make_bridge_session(key_value="sid-hard-turn-no-ledger") + session.durable_session_id = "durable-hard-turn-no-ledger" + session.durable_owner_epoch = 11 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-no-ledger", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-no-ledger", + hard_continuity_anchor=True, ) - saturated_session = _make_bridge_session(key_value="file-pin-queue-full", queued_request_count=8) - get_or_create = AsyncMock(return_value=saturated_session) + submit = AsyncMock() + sleep = AsyncMock() + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", + http_responses_session_bridge_operation_ledger_enabled=False, + ), + ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=30.0)) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", sleep) - async def fake_stream_events( - session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, - queue_limit: int, - propagate_http_errors: bool, - downstream_turn_state: str | None, - request_deadline: float | None = None, - ): - del ( + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( session, - request_state, - text_data, - queue_limit, - propagate_http_errors, - downstream_turn_state, - request_deadline, - ) - raise ProxyResponseError( - 429, - proxy_service.openai_error( - "bridge_queue_full", - "HTTP responses session bridge queue is full", - error_type="rate_limit_error", - ), - ) - yield "" - - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), - ), - ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) - monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) - - with pytest.raises(ProxyResponseError) as info: - async for _ in service._stream_via_http_bridge( - payload, - headers={}, - codex_session_affinity=False, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, propagate_http_errors=True, - openai_cache_affinity=False, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - rewritten_file_account_id="acc-file", + downstream_turn_state="turn-state-no-ledger", ): pass - assert info.value.status_code == 429 - assert get_or_create.await_count == 1 - create_call = get_or_create.await_args - assert create_call is not None - assert create_call.kwargs["preferred_account_id"] == "acc-file" - assert create_call.kwargs["fallback_on_preferred_account_unavailable"] is False + assert exc_info.value.status_code == 503 + assert exc_info.value.payload["error"]["code"] == "upstream_request_timeout" + submit.assert_not_awaited() + sleep.assert_not_awaited() @pytest.mark.asyncio -async def test_select_account_with_budget_prefers_durable_account_id_when_available( +async def test_http_bridge_one_shot_hard_turn_cooldown_wait_rejects_when_queue_is_full( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - select_account = AsyncMock( - return_value=proxy_service.AccountSelection( - account=cast(Any, SimpleNamespace(id="acc-preferred")), - error_message=None, - error_code=None, - ) + session = _make_bridge_session(key_value="sid-hard-turn-cooldown-queue-full", queued_request_count=8) + session.durable_session_id = "durable-hard-turn-cooldown-queue-full" + session.durable_owner_epoch = 12 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-cooldown-queue-full", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-cooldown-queue-full", + hard_continuity_anchor=True, ) - service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) + submit = AsyncMock() monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock(return_value=SimpleNamespace(sticky_reallocation_budget_threshold_pct=95.0)) + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", ), ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=30.0)) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) - selection = await service._select_account_with_budget( - time.monotonic() + 60.0, - request_id="req-1", - kind="http_bridge", - request_stage="reattach", - prefer_earlier_reset_window="primary", - preferred_account_id="acc-preferred", - ) + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state="turn-state-cooldown-queue-full", + ): + pass - assert selection.account is not None - assert selection.account.id == "acc-preferred" - assert select_account.await_count == 1 - first_call = select_account.await_args_list[0] - assert first_call.kwargs["account_ids"] is None - assert first_call.kwargs["required_account_id"] == "acc-preferred" + assert exc_info.value.status_code == 429 + assert exc_info.value.payload["error"]["code"] == "bridge_queue_full" + submit.assert_not_awaited() + assert session.queued_request_count == 8 @pytest.mark.asyncio -async def test_select_account_with_budget_skips_preferred_account_outside_assignment_scope( +async def test_http_bridge_one_shot_hard_turn_renews_durable_lease_while_waiting( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - select_account = AsyncMock( - return_value=proxy_service.AccountSelection( - account=cast(Any, SimpleNamespace(id="acc-allowed")), - error_message=None, - error_code=None, + session = _make_bridge_session(key_value="sid-hard-turn-renew-wait") + session.durable_session_id = "durable-hard-turn-renew-wait" + session.durable_owner_epoch = 13 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-renew-wait", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-renew-wait", + hard_continuity_anchor=True, + ) + renew_live_session = AsyncMock( + return_value=SimpleNamespace( + owner_instance_id="instance-hard-turn-renew", + owner_epoch=13, ) ) - service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) + service._durable_bridge = cast(Any, SimpleNamespace(renew_live_session=renew_live_session)) + cooldown = AsyncMock(side_effect=[25.0, 0.0]) + submit = AsyncMock(side_effect=RuntimeError("submitted after renewed cooldown")) + slept: list[float] = [] + + async def sleep(delay: float) -> None: + slept.append(delay) + monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock(return_value=SimpleNamespace(sticky_reallocation_budget_threshold_pct=95.0)) + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", + http_responses_session_bridge_instance_id="instance-hard-turn-renew", ), ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", cooldown) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", sleep) - selection = await service._select_account_with_budget( - time.monotonic() + 60.0, - request_id="req-2", - kind="http_bridge", - request_stage="reattach", - api_key=_make_api_key(key_id="key-1", assigned_account_ids=["acc-allowed"]), - prefer_earlier_reset_window="primary", - preferred_account_id="acc-preferred", - ) + with pytest.raises(RuntimeError, match="submitted after renewed cooldown"): + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state="turn-state-renew-wait", + ): + pass - assert selection.account is not None - assert selection.account.id == "acc-allowed" - assert select_account.await_count == 1 - first_call = select_account.await_args_list[0] - assert first_call.kwargs["account_ids"] == {"acc-allowed"} + assert slept == [pytest.approx(10.0), pytest.approx(10.0), pytest.approx(5.0)] + assert renew_live_session.await_count == 2 + submit.assert_awaited_once() + assert session.queued_request_count == 0 @pytest.mark.asyncio -async def test_select_account_with_budget_classifies_continuity_owner_outside_assignment_scope( +async def test_http_bridge_one_shot_hard_turn_fails_closed_when_lease_renewal_raises( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - select_account = AsyncMock( - return_value=proxy_service.AccountSelection( - account=None, - error_message="Required continuity owner is outside the effective account policy", - error_code="continuity_owner_policy_conflict", - ) + session = _make_bridge_session(key_value="sid-hard-turn-renew-failure") + session.durable_session_id = "durable-hard-turn-renew-failure" + session.durable_owner_epoch = 14 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-renew-failure", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-renew-failure", + hard_continuity_anchor=True, ) - service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) + renew_live_session = AsyncMock(side_effect=RuntimeError("durable store unavailable")) + service._durable_bridge = cast(Any, SimpleNamespace(renew_live_session=renew_live_session)) + submit = AsyncMock() + monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock(return_value=SimpleNamespace(sticky_reallocation_budget_threshold_pct=95.0)) + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", + http_responses_session_bridge_instance_id="instance-hard-turn-renew-failure", ), ) - - selection = await service._select_account_with_budget( - time.monotonic() + 60.0, - request_id="req-continuity-owner-scope", - kind="http_bridge", - request_stage="reattach", - api_key=_make_api_key(key_id="key-1", assigned_account_ids=["acc-allowed"]), - prefer_earlier_reset_window="primary", - preferred_account_id="acc-continuity-owner", - preferred_account_is_continuity_owner=True, - fallback_on_preferred_account_unavailable=False, + monkeypatch.setattr( + service, + "_http_bridge_precreated_retry_cooldown_seconds", + AsyncMock(return_value=25.0), ) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", AsyncMock()) - assert selection.error_code == "continuity_owner_policy_conflict" - select_account.assert_awaited_once() - selection_call = select_account.await_args - assert selection_call is not None - assert selection_call.kwargs["account_ids"] == {"acc-allowed"} - assert selection_call.kwargs["required_account_id"] == "acc-continuity-owner" - assert selection_call.kwargs["required_account_is_ownership_constraint"] is True - assert selection_call.kwargs["required_continuity_owner"] is True + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state="turn-state-renew-failure", + ): + pass + + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "bridge_continuity_persistence_failed" + renew_live_session.assert_awaited_once() + submit.assert_not_awaited() + assert session.closed is True + assert session.upstream_control.reconnect_requested is True + assert session.upstream_control.retire_after_drain is True + assert session.queued_request_count == 0 @pytest.mark.asyncio -async def test_create_http_bridge_session_passes_dashboard_reset_window_to_selection( +async def test_http_bridge_one_shot_hard_turn_does_not_submit_after_wait_budget( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - settings = SimpleNamespace( - prefer_earlier_reset_accounts=True, - prefer_earlier_reset_window="primary", - routing_strategy="usage_weighted", + session = _make_bridge_session(key_value="sid-hard-turn-wait-budget") + session.durable_session_id = "durable-hard-turn-wait-budget" + session.durable_owner_epoch = 9 + reservation = cast(Any, object()) + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-wait-budget", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=reservation, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-wait-budget", + hard_continuity_anchor=True, ) - selection_kwargs: list[dict[str, object]] = [] + clock = SimpleNamespace(now=100.0) + submit = AsyncMock() + release = AsyncMock() - async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: - selection_kwargs.append(kwargs) - return proxy_service.AccountSelection(account=None, error_message="No active accounts available") + async def sleep(delay: float) -> None: + clock.now += delay - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) monkeypatch.setattr( - proxy_service, "get_settings_cache", lambda: SimpleNamespace(get=AsyncMock(return_value=settings)) + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", + ), ) - monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) - - with pytest.raises(ProxyResponseError): - await service._create_http_bridge_session( - proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), - headers={}, - affinity=proxy_service._AffinityPolicy(key="sid-123"), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - ) - - assert selection_kwargs[0]["prefer_earlier_reset_accounts"] is True - assert selection_kwargs[0]["prefer_earlier_reset_window"] == "primary" + monkeypatch.setattr(http_bridge_streaming_module._service_time(), "monotonic", lambda: clock.now) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=30.0)) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(service, "_release_websocket_request_state_reservation", release) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", sleep) + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state="turn-state-wait-budget", + request_deadline=105.0, + ): + pass -def _pre_dispatch_proxy_error(message: str = "sanitized proxy connect failure") -> ProxyResponseError: - return ProxyResponseError( - 502, - openai_error("upstream_unavailable", message), - failure_phase="connect", - retryable_same_contract=True, - failure_detail="proxy_connect_pre_dispatch", - failure_exception_type="ClientProxyConnectionError", - ) + assert exc_info.value.status_code == 503 + assert exc_info.value.payload["error"]["code"] == "upstream_request_timeout" + submit.assert_not_awaited() + release.assert_awaited_once_with(request_state) + assert request_state.api_key_reservation is None -def _bridge_selection_settings() -> SimpleNamespace: - return SimpleNamespace( - prefer_earlier_reset_accounts=False, - prefer_earlier_reset_window="secondary", - routing_strategy="usage_weighted", +@pytest.mark.asyncio +async def test_http_bridge_replay_detach_releases_reservation_without_pending_ownership( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-replay-reservation") + reservation = cast(Any, object()) + request_state = proxy_service._WebSocketRequestState( + request_id="req-replay-reservation", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=reservation, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + operation_replay=True, ) + release = AsyncMock() + monkeypatch.setattr(service, "_release_websocket_request_state_reservation", release) + + assert await service._detach_http_bridge_request(session, request_state=request_state) is False + + release.assert_awaited_once_with(request_state) + assert request_state.api_key_reservation is None + assert request_state.operation_replay is False @pytest.mark.asyncio -async def test_create_http_bridge_session_defers_confirmed_proxy_backoff_until_reservation_release( +async def test_http_bridge_post_submit_cooldown_race_detaches_request( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - account_a = cast(Any, SimpleNamespace(id="acc-proxy-a", status=AccountStatus.ACTIVE, plan_type="plus")) - account_b = cast(Any, SimpleNamespace(id="acc-proxy-b", status=AccountStatus.ACTIVE, plan_type="plus")) - lease_a = proxy_service.AccountLease("lease-bridge-a", account_a.id, "stream", time.monotonic()) - lease_b = proxy_service.AccountLease("lease-bridge-b", account_b.id, "stream", time.monotonic()) - selections: list[set[str]] = [] - reallocate_flags: list[bool] = [] - released_leases: list[proxy_service.AccountLease] = [] - backed_off_accounts: list[object] = [] - settlement_order: list[str] = [] - reservation = proxy_service.ApiKeyUsageReservationData( - reservation_id="resv-http-bridge-proxy-failover", - key_id="key-http-bridge-proxy-failover", - model="gpt-5.4", + session = _make_bridge_session(key_value="sid-post-submit-cooldown") + request_state = proxy_service._WebSocketRequestState( + request_id="req-post-submit-cooldown", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + previous_response_id="resp-anchor", ) - lifecycle = proxy_support_module._DeferredAccountBackoffLifecycle(reservation=reservation) - upstream = cast(Any, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) - async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: - excluded = set(cast(set[str], kwargs["exclude_account_ids"])) - selections.append(excluded) - affinity_policy = cast(proxy_service._AffinityPolicy, kwargs["affinity_policy"]) - reallocate_flags.append(affinity_policy.reallocate_sticky) - if not excluded: - return proxy_service.AccountSelection(account=account_a, error_message=None, lease=lease_a) - return proxy_service.AccountSelection(account=account_b, error_message=None, lease=lease_b) + async def submit(target_session: Any, *, request_state: Any, **kwargs: Any) -> None: + del kwargs + target_session.pending_requests.append(request_state) - async def release_account_lease(lease: proxy_service.AccountLease | None) -> None: - if lease is not None: - released_leases.append(lease) + cooldown = AsyncMock(side_effect=[0.0, 30.0]) + detach = AsyncMock() + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", cooldown) + monkeypatch.setattr(service, "_detach_http_bridge_request", detach) - async def record_error_backoff(account: object) -> None: - backed_off_accounts.append(account) - # The dead route's stream lease must settle before the health write. - assert lease_a in released_leases - assert settlement_order == ["settle"] - settlement_order.append("backoff") + events = [ + event + async for event in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, + propagate_http_errors=False, + downstream_turn_state=None, + ) + ] - async def release_reservation(candidate: object) -> None: - assert candidate is reservation - settlement_order.append("settle") + assert len(events) == 1 + assert '"code":"stream_idle_timeout"' in events[0] + assert cooldown.await_count == 2 + detach.assert_awaited_once_with(session, request_state=request_state) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + +@pytest.mark.asyncio +async def test_http_bridge_keepalive_counts_as_first_yield_before_late_response_failed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=_bridge_selection_settings())), - ) - monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) - monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=[account_a, account_b])) - monkeypatch.setattr( - service, - "_open_upstream_websocket_with_budget", - AsyncMock(side_effect=[_pre_dispatch_proxy_error(), upstream]), + "get_settings", + lambda: SimpleNamespace(sse_keepalive_interval_seconds=0.001), ) - monkeypatch.setattr(service._load_balancer, "release_account_lease", release_account_lease) - monkeypatch.setattr(service._load_balancer, "record_error_backoff", record_error_backoff) - monkeypatch.setattr(service._load_balancer, "record_error", AsyncMock()) - monkeypatch.setattr(service, "_release_websocket_reservation", release_reservation) - monkeypatch.setattr(service, "_relay_http_bridge_upstream_messages", AsyncMock()) + monkeypatch.setattr(proxy_service, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.001) - session = await service._create_http_bridge_session( - proxy_service._HTTPBridgeSessionKey("session_header", "sid-proxy-failover", None), - headers={}, - affinity=proxy_service._AffinityPolicy(key="sid-proxy-failover"), - api_key=None, - request_model="gpt-5.4", + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-keepalive-first", None), + headers={"session_id": "sid-keepalive-first"}, + affinity=proxy_service._AffinityPolicy( + key="sid-keepalive-first", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + request_model="gpt-5.1", + account=cast(Any, SimpleNamespace(id="acc-keepalive-first", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=1.0, idle_ttl_seconds=120.0, - deferred_account_backoff_lifecycle=lifecycle, - defer_account_health_writes=True, + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-keepalive-first", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + response_id="resp_keepalive_first", ) - assert backed_off_accounts == [] - assert lifecycle.pending_backoffs == {account_a.id: account_a} - await service._release_websocket_reservation(reservation) - lifecycle.settlement_confirmed = True - await service._drain_deferred_account_error_backoffs(lifecycle.pending_backoffs) + async def fake_submit_http_bridge_request( + target_session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + ) -> None: + del text_data, queue_limit + target_session.pending_requests.append(request_state) - assert session.account is account_b - assert selections == [set(), {account_a.id}] - assert reallocate_flags == [False, True] - assert backed_off_accounts == [account_a] - assert settlement_order == ["settle", "backoff"] - assert lease_a in released_leases - assert lease_b not in released_leases + monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) + + stream = service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data="{}", + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state=None, + ) + + keepalive = await asyncio.wait_for(anext(stream), timeout=1.0) + assert "response.in_progress" in keepalive + + event_queue = request_state.event_queue + assert event_queue is not None + request_state.error_http_status_override = 502 + await event_queue.put( + proxy_service.format_sse_event( + proxy_service.response_failed_event( + "upstream_unavailable", + "upstream failed after keepalive", + response_id="resp_keepalive_first", + ) + ) + ) + failed = await asyncio.wait_for(anext(stream), timeout=1.0) + assert "response.failed" in failed + assert "upstream_unavailable" in failed + + await event_queue.put(None) + with pytest.raises(StopAsyncIteration): + await asyncio.wait_for(anext(stream), timeout=1.0) @pytest.mark.asyncio -async def test_create_http_bridge_session_confirmed_proxy_failure_keeps_hard_owner_pinned( +async def test_http_bridge_account_capacity_wait_sends_keepalive_instead_of_idle_timeout( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - account = cast(Any, SimpleNamespace(id="acc-proxy-owner", status=AccountStatus.ACTIVE, plan_type="plus")) - lease = proxy_service.AccountLease("lease-bridge-owner", account.id, "stream", time.monotonic()) - select_account = AsyncMock( - return_value=proxy_service.AccountSelection(account=account, error_message=None, lease=lease) - ) - release_account_lease = AsyncMock() - record_error_backoff = AsyncMock() - original_error = _pre_dispatch_proxy_error("owner proxy unavailable") - - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=_bridge_selection_settings())), + "get_settings", + lambda: SimpleNamespace( + sse_keepalive_interval_seconds=0.001, + stream_idle_timeout_seconds=0.001, + ), ) - monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) - monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) - monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(side_effect=original_error)) - monkeypatch.setattr(service._load_balancer, "release_account_lease", release_account_lease) - monkeypatch.setattr(service._load_balancer, "record_error_backoff", record_error_backoff) + monkeypatch.setattr(proxy_service, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.001) - with pytest.raises(ProxyResponseError) as exc_info: - await service._create_http_bridge_session( - proxy_service._HTTPBridgeSessionKey("turn_state_header", "turn-owner", None, strength="hard"), - headers={"x-codex-turn-state": "turn-owner"}, - affinity=proxy_service._AffinityPolicy(key="turn-owner"), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - preferred_account_id=account.id, - require_preferred_account=True, - fallback_on_preferred_account_unavailable=False, - ) - - # The hard-required owner fails closed on the original sanitized failure. - assert exc_info.value is original_error - select_account.assert_awaited_once() - record_error_backoff.assert_awaited_once_with(account) - assert release_account_lease.await_args_list[0].args == (lease,) - - -@pytest.mark.asyncio -async def test_create_http_bridge_session_preserves_proxy_failure_when_no_replacement( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - account = cast(Any, SimpleNamespace(id="acc-proxy-only", status=AccountStatus.ACTIVE, plan_type="plus")) - lease = proxy_service.AccountLease("lease-bridge-only", account.id, "stream", time.monotonic()) - selections: list[set[str]] = [] - original_error = _pre_dispatch_proxy_error("original bridge proxy failure") - - async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: - excluded = set(cast(set[str], kwargs["exclude_account_ids"])) - selections.append(excluded) - if not excluded: - return proxy_service.AccountSelection(account=account, error_message=None, lease=lease) - return proxy_service.AccountSelection( - account=None, - error_message="No active accounts available", - error_code="no_accounts", - ) - - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=_bridge_selection_settings())), + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-capacity-wait", None), + headers={"session_id": "sid-capacity-wait"}, + affinity=proxy_service._AffinityPolicy( + key="sid-capacity-wait", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + request_model="gpt-5.1", + account=cast(Any, SimpleNamespace(id="acc-capacity-wait", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=1.0, + idle_ttl_seconds=120.0, ) - monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) - monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) - monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(side_effect=original_error)) - monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) - monkeypatch.setattr(service._load_balancer, "record_error_backoff", AsyncMock()) - - with pytest.raises(ProxyResponseError) as exc_info: - await service._create_http_bridge_session( - proxy_service._HTTPBridgeSessionKey("session_header", "sid-proxy-no-replacement", None), - headers={}, - affinity=proxy_service._AffinityPolicy(key="sid-proxy-no-replacement"), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-capacity-wait", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + ) + request_state.account_capacity_waiting = True + request_state.account_capacity_wait_reason = "Rate limit exceeded. Try again in 120s" + request_state.account_capacity_wait_started_at = time.monotonic() - 3.0 + request_state.account_capacity_wait_retry_after_seconds = 120.0 - # The original sanitized failure is preserved instead of ``no_accounts``. - assert exc_info.value is original_error - assert selections == [set(), {account.id}] + async def fake_submit_http_bridge_request( + target_session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + ) -> None: + del text_data, queue_limit + target_session.pending_requests.append(request_state) + monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) -@pytest.mark.asyncio -async def test_create_http_bridge_session_idle_close_error_is_not_treated_as_dead_route( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - account = cast(Any, SimpleNamespace(id="acc-proxy-idle", status=AccountStatus.ACTIVE, plan_type="plus")) - lease = proxy_service.AccountLease("lease-bridge-idle", account.id, "stream", time.monotonic()) - record_error_backoff = AsyncMock() - idle_error = ProxyResponseError( - 502, - openai_error("upstream_unavailable", "Upstream websocket closed while idle"), - failure_phase="upstream", - failure_detail="stream_idle_timeout", + stream = service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data="{}", + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state=None, ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=_bridge_selection_settings())), - ) - monkeypatch.setattr( - service, - "_select_account_with_budget_compatible", - AsyncMock(return_value=proxy_service.AccountSelection(account=account, error_message=None, lease=lease)), - ) - monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) - monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(side_effect=idle_error)) - monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) - monkeypatch.setattr(service._load_balancer, "record_error_backoff", record_error_backoff) + keepalive = await asyncio.wait_for(anext(stream), timeout=1.0) + payload = proxy_service.parse_sse_data_json(keepalive) - with pytest.raises(ProxyResponseError) as exc_info: - await service._create_http_bridge_session( - proxy_service._HTTPBridgeSessionKey("session_header", "sid-proxy-idle", None), - headers={}, - affinity=proxy_service._AffinityPolicy(key="sid-proxy-idle"), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - ) + assert payload is not None + assert payload["type"] == "codex.keepalive" + assert payload["status"] == "waiting_for_account_capacity" + assert payload["request_id"] == "req-capacity-wait" + assert "stream_idle_timeout" not in keepalive - # An idle disconnect is not provable pre-dispatch evidence: no account - # exclusion, no transient-backoff health write. - assert exc_info.value is idle_error - record_error_backoff.assert_not_awaited() + await stream.aclose() @pytest.mark.asyncio -async def test_reconnect_http_bridge_session_passes_dashboard_reset_window_to_selection( +async def test_http_bridge_idle_recovery_transport_failure_yields_terminal_event( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - session.request_service_tier = "priority" - settings = SimpleNamespace( - prefer_earlier_reset_accounts=True, - prefer_earlier_reset_window="primary", - routing_strategy="usage_weighted", + detach = AsyncMock() + monkeypatch.setattr(service, "_detach_http_bridge_request", detach) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace( + http_responses_stream_request_budget_seconds=60.0, + sse_keepalive_interval_seconds=0.001, + stream_idle_timeout_seconds=0.001, + ), ) - selection_kwargs: list[dict[str, object]] = [] - - async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: - selection_kwargs.append(kwargs) - return proxy_service.AccountSelection(account=None, error_message="No active accounts available") + monkeypatch.setattr(proxy_service, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.001) + monkeypatch.setattr(http_bridge_streaming_module, "_stream_keepalive_max_count", lambda: 1) + session = _make_bridge_session(key_value="sid-idle-retry-transport") request_state = proxy_service._WebSocketRequestState( - request_id="req-reconnect", - model="gpt-5.4", + request_id="req-idle-retry-transport", + model="gpt-5.1", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - response_create_sent_at=1.0, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.1","input":"hello"}', + transport="http", ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr( - proxy_service, "get_settings_cache", lambda: SimpleNamespace(get=AsyncMock(return_value=settings)) + + async def fake_submit_http_bridge_request( + target_session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + ) -> None: + del text_data, queue_limit + target_session.pending_requests.append(request_state) + + retry_error = UpstreamWebSocketTransportError( + "Codex upstream websocket send failed: OSError", + error_code="proxy_network_unavailable", ) - monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) + retry_precreated = AsyncMock(side_effect=retry_error) + monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - with pytest.raises(ProxyResponseError): - await service._reconnect_http_bridge_session( + chunks = [ + chunk + async for chunk in service._stream_http_bridge_session_events( session, request_state=request_state, - require_same_account=True, + text_data="{}", + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state=None, ) + ] - assert selection_kwargs[0]["prefer_earlier_reset_accounts"] is True - assert selection_kwargs[0]["prefer_earlier_reset_window"] == "primary" - assert selection_kwargs[0]["service_tier"] == "priority" - assert selection_kwargs[0]["preferred_account_id"] == session.account.id - assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is False - assert request_state.response_create_sent_at is None + assert len(chunks) == 1 + payload = proxy_service.parse_sse_data_json(chunks[0]) + assert payload is not None + assert payload["type"] == "response.failed" + response = payload["response"] + assert isinstance(response, dict) + error = response["error"] + assert isinstance(error, dict) + assert error["code"] == "proxy_network_unavailable" + assert error["message"] == "Codex upstream websocket send failed: OSError" + retry_precreated.assert_awaited_once_with(session, restart_reader=True) + detach.assert_awaited_once_with(session, request_state=request_state) @pytest.mark.asyncio -async def test_reconnect_account_neutral_recovery_requires_typed_owner_without_callsite_flag( +async def test_http_bridge_capacity_wait_with_response_id_sends_explicit_keepalive( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key=_make_account_neutral_replay_session_key("reconnect-owner")) - settings = SimpleNamespace( - prefer_earlier_reset_accounts=False, - prefer_earlier_reset_window="secondary", - routing_strategy="usage_weighted", - ) - selection_kwargs: list[dict[str, object]] = [] - - async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: - selection_kwargs.append(kwargs) - return proxy_service.AccountSelection( - account=None, - error_message="Required continuity owner account no longer exists", - error_code=CONTINUITY_OWNER_UNAVAILABLE, - ) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace( + sse_keepalive_interval_seconds=0.001, + stream_idle_timeout_seconds=0.001, + ), + ) + monkeypatch.setattr(proxy_service, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.001) + session = _make_bridge_session(key_value="sid-capacity-response") request_state = proxy_service._WebSocketRequestState( - request_id="req-reconnect-recovery-owner", - model="gpt-5.4", + request_id="req-capacity-response", + model="gpt-5.1", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), + event_queue=asyncio.Queue(), + response_id="resp-capacity-response", + transport="http", ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), + request_state.account_capacity_waiting = True + request_state.account_capacity_wait_reason = "Rate limit exceeded. Try again in 120s" + request_state.account_capacity_wait_started_at = time.monotonic() - 3.0 + request_state.account_capacity_wait_retry_after_seconds = 120.0 + + async def fake_submit_http_bridge_request( + target_session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + ) -> None: + del text_data, queue_limit + target_session.pending_requests.append(request_state) + + monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) + + stream = service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data="{}", + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state=None, ) - monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) - with pytest.raises(ProxyResponseError) as exc_info: - await service._reconnect_http_bridge_session(session, request_state=request_state) + keepalive = proxy_service.parse_sse_data_json(await asyncio.wait_for(anext(stream), timeout=1.0)) + in_progress = proxy_service.parse_sse_data_json(await asyncio.wait_for(anext(stream), timeout=1.0)) - assert exc_info.value.status_code == 502 - assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" - assert selection_kwargs[0]["preferred_account_id"] == session.account.id - assert selection_kwargs[0]["preferred_account_is_continuity_owner"] is True - assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is False + assert keepalive is not None + assert keepalive["type"] == "codex.keepalive" + assert keepalive["status"] == "waiting_for_account_capacity" + assert in_progress is not None + assert in_progress["type"] == "response.in_progress" + response = in_progress["response"] + assert isinstance(response, dict) + assert response["id"] == "resp-capacity-response" + + await stream.aclose() @pytest.mark.asyncio -async def test_reconnect_http_bridge_session_uses_bridge_budget_for_capacity_wait( +async def test_get_or_create_http_bridge_session_reuses_live_local_session_without_ring_lookup( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - settings = SimpleNamespace( - prefer_earlier_reset_accounts=False, - prefer_earlier_reset_window="secondary", - routing_strategy="usage_weighted", - ) - sleep_calls: list[dict[str, object]] = [] - - async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: - return proxy_service.AccountSelection( - account=None, - error_message="Rate limit exceeded. Try again in 120s", - error_code="no_accounts", - ) - - async def sleep_for_recovery(*_args: object, **kwargs: object) -> bool: - sleep_calls.append(kwargs) - return False - - request_state = proxy_service._WebSocketRequestState( - request_id="req-reconnect-bridge-budget", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=100.0, + key = proxy_service._HTTPBridgeSessionKey("prompt_cache_key", "bridge-key", None) + existing = proxy_service._HTTPBridgeSession( + key=key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-key"), + request_model="gpt-5.4-mini", + account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE, plan_type="plus")), + upstream=cast(UpstreamWebSocket, SimpleNamespace()), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, ) - monkeypatch.setattr(proxy_service.time, "monotonic", lambda: 100.5) + service._http_bridge_sessions[key] = existing monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: _make_app_settings( - proxy_request_budget_seconds=0.001, - http_responses_session_bridge_request_budget_seconds=120.0, - ), + service, + "_prune_http_bridge_sessions_locked", + Mock(return_value=[]), ) monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), + "get_settings", + lambda: _make_app_settings(), ) - monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) - monkeypatch.setattr(http_bridge_mixin_module, "_sleep_for_account_selection_recovery", sleep_for_recovery) - with pytest.raises(ProxyResponseError): - await service._reconnect_http_bridge_session(session, request_state=request_state) + async def _unexpected_owner_lookup(*args: object, **kwargs: object) -> str: + raise AssertionError("live local session reuse must not hit the ring") - assert sleep_calls - assert sleep_calls[0]["max_sleep_seconds"] == pytest.approx(119.5) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", _unexpected_owner_lookup) + monkeypatch.setattr(proxy_service, "_active_http_bridge_instance_ring", _unexpected_owner_lookup) + + reused = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-key"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + + assert reused is existing + assert reused.request_model == "gpt-5.4" + assert reused.last_used_at > 1.0 @pytest.mark.asyncio -async def test_reconnect_http_bridge_session_skips_capacity_wait_for_usage_limit( +async def test_get_or_create_http_bridge_session_preserves_closed_admission_handoff( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - settings = SimpleNamespace( - prefer_earlier_reset_accounts=False, - prefer_earlier_reset_window="secondary", - routing_strategy="usage_weighted", - ) - - async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: - return proxy_service.AccountSelection( - account=None, - error_message="Rate limit exceeded. Try again in 1h", - error_code="usage_limit_reached", - resets_at=1_700_003_600, - ) - - request_state = proxy_service._WebSocketRequestState( - request_id="req-reconnect-usage-limit-now", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=100.0, - ) - monkeypatch.setattr(proxy_service.time, "monotonic", lambda: 100.5) + key = proxy_service._HTTPBridgeSessionKey("session_header", "bridge-handoff", None) + existing = _make_bridge_session(key_value="bridge-handoff") + existing.key = key + existing.request_model = "gpt-5.4" + existing.closed = True + existing.admission_waiter_count = 1 + service._http_bridge_sessions[key] = existing monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), - ) - monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) - monkeypatch.setattr( - http_bridge_mixin_module, - "_sleep_for_account_selection_recovery", - lambda *_args, **_kwargs: pytest.fail("usage_limit_reached must not enter recovery wait"), - ) + create = AsyncMock() + monkeypatch.setattr(service, "_create_http_bridge_session", create) - with pytest.raises(ProxyResponseError) as exc_info: - await service._reconnect_http_bridge_session(session, request_state=request_state) + resolved = await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-session-id": "bridge-handoff"}, + affinity=proxy_service._AffinityPolicy( + key="bridge-handoff", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) - assert exc_info.value.status_code == 429 - assert exc_info.value.payload["error"]["code"] == "usage_limit_reached" - assert exc_info.value.payload["error"]["type"] == "usage_limit_reached" - assert exc_info.value.payload["error"]["resets_at"] == 1_700_003_600 + assert resolved is existing + assert service._http_bridge_sessions[key] is existing + assert existing.request_model == "gpt-5.4" + create.assert_not_awaited() @pytest.mark.asyncio -async def test_reconnect_http_bridge_session_preserves_owner_error_for_owner_usage_limit( +async def test_get_or_create_http_bridge_session_rejects_anchored_incompatible_closed_admission_handoff( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - settings = SimpleNamespace( - prefer_earlier_reset_accounts=False, - prefer_earlier_reset_window="secondary", - routing_strategy="usage_weighted", - ) + key = proxy_service._HTTPBridgeSessionKey("session_header", "bridge-handoff", None) + existing = _make_bridge_session(key_value="bridge-handoff") + existing.key = key + existing.closed = True + existing.admission_waiter_count = 1 + service._http_bridge_sessions[key] = existing + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + create = AsyncMock() + monkeypatch.setattr(service, "_create_http_bridge_session", create) - async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: - return proxy_service.AccountSelection( - account=None, - error_message="Rate limit exceeded. Try again in 1h", - error_code="usage_limit_reached", - resets_at=1_700_003_600, + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-session-id": "bridge-handoff"}, + affinity=proxy_service._AffinityPolicy(key="bridge-handoff"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + preferred_account_id="different-account", + previous_response_id="resp-anchored", ) - request_state = proxy_service._WebSocketRequestState( - request_id="req-reconnect-owner-usage-limit", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=100.0, - preferred_account_id=session.account.id, - ) - monkeypatch.setattr(proxy_service.time, "monotonic", lambda: 100.5) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + assert exc_info.value.status_code == 503 + assert service._http_bridge_sessions[key] is existing + create.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_get_or_create_http_bridge_session_recovers_unanchored_closed_admission_handoff( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("session_header", "bridge-handoff", None) + existing = _make_bridge_session(key_value="bridge-handoff") + existing.key = key + existing.closed = True + existing.admission_waiter_count = 1 + service._http_bridge_sessions[key] = existing + replacement = _make_bridge_session(key_value="bridge-handoff") + replacement.key = key + settings = _make_app_settings() + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), + "_http_bridge_owner_instance", + AsyncMock(return_value=settings.http_responses_session_bridge_instance_id), ) - monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) monkeypatch.setattr( http_bridge_mixin_module, - "_sleep_for_account_selection_recovery", - lambda *_args, **_kwargs: pytest.fail("owner-only usage_limit_reached must not enter recovery wait"), + "_http_bridge_owner_instance", + AsyncMock(return_value=settings.http_responses_session_bridge_instance_id), ) + monkeypatch.setattr(http_bridge_mixin_module, "_http_bridge_owner_check_required", lambda *args, **kwargs: False) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) + monkeypatch.setattr(service, "_schedule_http_bridge_session_closes", Mock()) + create = AsyncMock(return_value=replacement) + monkeypatch.setattr(service, "_create_http_bridge_session", create) - with pytest.raises(ProxyResponseError) as exc_info: - await service._reconnect_http_bridge_session( - session, - request_state=request_state, - require_preferred_account=True, - ) + resolved = await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-session-id": "bridge-handoff"}, + affinity=proxy_service._AffinityPolicy(key="bridge-handoff"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + preferred_account_id="different-account", + ) - assert exc_info.value.status_code == 502 - assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" - assert exc_info.value.payload["error"]["type"] == "server_error" + assert resolved is replacement + assert service._http_bridge_sessions[key] is replacement + assert existing.closed is True + create.assert_awaited_once() @pytest.mark.asyncio -async def test_reconnect_http_bridge_session_preserves_exclusions_after_capacity_wait( +async def test_get_or_create_http_bridge_session_replaces_routing_unavailable_account( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - settings = SimpleNamespace( - prefer_earlier_reset_accounts=False, - prefer_earlier_reset_window="secondary", - routing_strategy="usage_weighted", + key = proxy_service._HTTPBridgeSessionKey("request", "bridge-routing-unavailable", None) + stale_session = proxy_service._HTTPBridgeSession( + key=key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-routing-unavailable"), + request_model="gpt-5.4-mini", + account=cast(Any, SimpleNamespace(id="acc-unavailable", status=AccountStatus.ACTIVE, plan_type="plus")), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, ) - selection_kwargs: list[dict[str, object]] = [] - account = cast(Any, SimpleNamespace(id=session.account.id, status=AccountStatus.ACTIVE)) + replacement_session = proxy_service._HTTPBridgeSession( + key=key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-routing-unavailable"), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-fresh", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=2.0, + idle_ttl_seconds=120.0, + ) + service._http_bridge_sessions[key] = stale_session + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(service, "_create_http_bridge_session", AsyncMock(return_value=replacement_session)) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) + close_session = AsyncMock() + monkeypatch.setattr(service, "_close_http_bridge_session", close_session) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: - selection_kwargs.append(kwargs) - if len(selection_kwargs) == 1: - return proxy_service.AccountSelection(account=account, error_message=None) - return proxy_service.AccountSelection( - account=None, - error_message="Rate limit exceeded. Try again in 120s", - error_code="no_accounts", + mark_account_routing_unavailable("acc-unavailable") + try: + reused = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-routing-unavailable"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, ) + finally: + clear_account_routing_unavailable("acc-unavailable") - async def fail_refresh(*_args: object, **_kwargs: object) -> Any: - raise RefreshError("invalid_grant", "refresh failed", True) - - sleep_calls = 0 + assert reused is replacement_session + assert service._http_bridge_sessions[key] is replacement_session + assert stale_session.closed is True + await _wait_for_close_await(close_session, stale_session) - async def sleep_for_recovery(*_args: object, **_kwargs: object) -> bool: - nonlocal sleep_calls - sleep_calls += 1 - return sleep_calls == 1 - request_state = proxy_service._WebSocketRequestState( - request_id="req-reconnect-exclusions", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - ) - request_state.excluded_account_ids.add("acc-request-state") - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), +@pytest.mark.asyncio +async def test_close_http_bridge_sessions_for_account_detaches_matching_sessions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + matching = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "bridge-matching", None), + key_value="bridge-matching", ) - monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) - monkeypatch.setattr(service, "_ensure_fresh_with_budget", fail_refresh) - service._load_balancer = cast( - Any, - SimpleNamespace( - mark_permanent_failure=AsyncMock(), - release_account_lease=AsyncMock(), - ), + matching.account.id = "acc-close" + other = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "bridge-other", None), + key_value="bridge-other", ) - monkeypatch.setattr(http_bridge_mixin_module, "_sleep_for_account_selection_recovery", sleep_for_recovery) + other.account.id = "acc-other" + detached_matching = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "bridge-detached-matching", None), + key_value="bridge-detached-matching", + ) + detached_matching.account.id = "acc-close" + # Reader/error paths may reject admission before any resource close starts. + # Account invalidation must not mistake that state for finalized teardown. + detached_matching.closed = True + detached_other = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "bridge-detached-other", None), + key_value="bridge-detached-other", + ) + detached_other.account.id = "acc-other" + service._http_bridge_sessions[matching.key] = matching + service._http_bridge_sessions[other.key] = other + service._http_bridge_detached_sessions[id(detached_matching)] = detached_matching + service._http_bridge_detached_sessions[id(detached_other)] = detached_other + close_session = AsyncMock() + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", close_session) - with pytest.raises(ProxyResponseError): - await service._reconnect_http_bridge_session(session, request_state=request_state) + closed = await service.close_http_bridge_sessions_for_account("acc-close") - assert len(selection_kwargs) == 3 - assert selection_kwargs[2]["exclude_account_ids"] == {"acc-request-state", session.account.id} + assert closed == 2 + assert matching.key not in service._http_bridge_sessions + assert service._http_bridge_sessions[other.key] is other + assert matching.closed is True + assert detached_matching.closed is True + assert detached_other.closed is False + assert close_session.await_args_list == [ + ((matching,), {"reason": "account_binding_changed"}), + ((detached_matching,), {"reason": "account_binding_changed"}), + ] -@pytest.mark.asyncio -async def test_create_http_bridge_session_filters_http_headers_for_upstream_websocket( - monkeypatch: pytest.MonkeyPatch, -) -> None: +def test_http_bridge_request_text_replaces_client_installation_id() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - captured_headers: list[dict[str, str]] = [] - - async def select_account(_deadline: float, **_: object) -> proxy_service.AccountSelection: - return proxy_service.AccountSelection( - account=cast(Any, SimpleNamespace(id="acc-bridge", status=AccountStatus.ACTIVE)), - error_message=None, - error_code=None, - ) + session = _make_bridge_session() + session.account.codex_installation_id = "account-installation" + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}], + "client_metadata": { + "x-codex-installation-id": "client-installation", + "x-codex-turn-metadata": '{"installation_id":"client-installation","turn_id":"payload-turn"}', + }, + } + ) + request_state, text_data = service._prepare_http_bridge_request( + payload, + {}, + api_key=None, + api_key_reservation=None, + ) + request_state.fresh_upstream_request_text = json.dumps( + { + "type": "response.create", + "model": "gpt-5.4", + "input": [], + "client_metadata": {"x-codex-installation-id": "client-replay"}, + }, + separators=(",", ":"), + ) - async def ensure_fresh(account: object, **_: object) -> object: - return account + updated_text = service._http_bridge_text_with_account_installation_id(session, request_state, text_data) - async def open_upstream(_account: object, headers: dict[str, str], **_: object) -> UpstreamWebSocket: - captured_headers.append(dict(headers)) - return cast(UpstreamWebSocket, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + assert json.loads(updated_text)["client_metadata"] == { + "x-codex-installation-id": "account-installation", + "x-codex-turn-metadata": '{"installation_id":"account-installation","turn_id":"payload-turn"}', + } + assert request_state.fresh_upstream_request_text is not None + assert json.loads(request_state.fresh_upstream_request_text)["client_metadata"] == { + "x-codex-installation-id": "account-installation", + } - async def fake_relay(_session: proxy_service._HTTPBridgeSession) -> None: - return None - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - prefer_earlier_reset_accounts=False, - routing_strategy=None, - ) - ) - ), +def test_http_bridge_request_text_rejects_installation_metadata_size_overflow( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + session.account.codex_installation_id = "account-installation" + request_state = proxy_service._WebSocketRequestState( + request_id="req-http-installation-size", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + transport="http", + request_text='{"type":"response.create","input":"x"}', ) - monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) - monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) - monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", open_upstream) - monkeypatch.setattr(service, "_relay_http_bridge_upstream_messages", fake_relay) - - session = await service._create_http_bridge_session( - proxy_service._HTTPBridgeSessionKey("session_header", "sid-filtered", None), - headers={ - "accept": "text/event-stream", - "accept-encoding": "gzip, deflate, br, zstd", - "authorization": "Bearer client-key", - "connection": "keep-alive, x-handshake-debug", - "content-type": "application/json", - "cookie": "session=client-cookie", - "host": "127.0.0.1:3455", - "keep-alive": "timeout=5", - "proxy-authorization": "Basic secret", - "proxy-connection": "keep-alive", - "session_id": "sid-filtered", - "te": "trailers", - "trailer": "x-trailer", - "transfer-encoding": "chunked", - "upgrade": "websocket", - "user-agent": "pi", - "X-Codex-Turn-Metadata": '{"turn_id":"turn-create"}', - "x-OpenAI-Subagent": "collab_spawn", - "X-Codex-Parent-Thread-ID": "parent-create", - "x-CODEX-window-id": "child-create:0", - "x-handshake-debug": "1", - }, - affinity=proxy_service._AffinityPolicy( - key="sid-filtered", - kind=proxy_service.StickySessionKind.CODEX_SESSION, - ), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, + stamped_text = service._http_bridge_text_with_account_installation_id( + session, + request_state, + request_state.request_text or "{}", ) + max_bytes = len(stamped_text.encode("utf-8")) - 1 + request_state.request_text = '{"type":"response.create","input":"x"}' + assert len((request_state.request_text or "").encode("utf-8")) < max_bytes + + monkeypatch.setattr(proxy_service, "_UPSTREAM_RESPONSE_CREATE_WARN_BYTES", max_bytes + 1, raising=False) + monkeypatch.setattr(proxy_service, "_UPSTREAM_RESPONSE_CREATE_MAX_BYTES", max_bytes, raising=False) + + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + service._http_bridge_text_with_account_installation_id( + session, + request_state, + request_state.request_text or "{}", + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.payload["error"]["code"] == "payload_too_large" - if session.upstream_reader is not None: - await session.upstream_reader - assert captured_headers - forwarded = {key.lower(): value for key, value in captured_headers[0].items()} - assert forwarded["session_id"] == "sid-filtered" - assert forwarded["user-agent"] == "pi" - assert "accept" not in forwarded - assert "accept-encoding" not in forwarded - assert "authorization" not in forwarded - assert "connection" not in forwarded - assert "content-type" not in forwarded - assert "cookie" not in forwarded - assert "host" not in forwarded - assert "keep-alive" not in forwarded - assert "proxy-authorization" not in forwarded - assert "proxy-connection" not in forwarded - assert "te" not in forwarded - assert "trailer" not in forwarded - assert "transfer-encoding" not in forwarded - assert "upgrade" not in forwarded - assert "x-codex-turn-metadata" not in forwarded - assert "x-openai-subagent" not in forwarded - assert "x-codex-parent-thread-id" not in forwarded - assert "x-codex-window-id" not in forwarded - assert "x-handshake-debug" not in forwarded + +def test_submit_http_bridge_request_uses_bridge_installation_metadata_helper() -> None: + source = inspect.getsource(proxy_service.ProxyService._submit_http_bridge_request_with_handoff) + + assert "_response_create_text_with_account_installation_id(" not in source + assert source.count("_http_bridge_text_with_account_installation_id(") >= 3 @pytest.mark.asyncio -async def test_reconnect_http_bridge_session_filters_http_headers_for_upstream_websocket( +async def test_get_or_create_http_bridge_session_skips_prune_when_pending_lock_is_wedged( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - session.headers = { - "accept": "text/event-stream", - "accept-encoding": "gzip, deflate, br, zstd", - "authorization": "Bearer client-key", - "connection": "keep-alive, x-handshake-debug", - "content-type": "application/json", - "cookie": "session=client-cookie", - "host": "127.0.0.1:3455", - "keep-alive": "timeout=5", - "proxy-authorization": "Basic secret", - "proxy-connection": "keep-alive", - "session_id": "sid-filtered", - "te": "trailers", - "trailer": "x-trailer", - "transfer-encoding": "chunked", - "upgrade": "websocket", - "user-agent": "pi", - "X-Codex-Turn-Metadata": '{"turn_id":"turn-reconnect"}', - "x-OpenAI-Subagent": "collab_spawn", - "X-Codex-Parent-Thread-ID": "parent-reconnect", - "x-CODEX-window-id": "child-reconnect:0", - "x-handshake-debug": "1", - } - session.upstream_turn_state = "upstream-turn-state" - captured_headers: list[dict[str, str]] = [] - - async def select_account(_deadline: float, **_: object) -> proxy_service.AccountSelection: - return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + key = proxy_service._HTTPBridgeSessionKey("request", "bridge-wedged-idle", None) + existing_session = _make_bridge_session(key=key, key_value="bridge-wedged-idle") + existing_session.last_used_at = time.monotonic() - 300.0 + existing_session.idle_ttl_seconds = 1.0 + service._http_bridge_sessions[key] = existing_session + lock_acquired = asyncio.Event() + release_lock = asyncio.Event() - async def ensure_fresh(account: object, **_: object) -> object: - return account + async def hold_pending_lock() -> None: + async with existing_session.pending_lock: + lock_acquired.set() + await release_lock.wait() - async def open_upstream(_account: object, headers: dict[str, str], **_: object) -> UpstreamWebSocket: - captured_headers.append(dict(headers)) - return cast(UpstreamWebSocket, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + lock_holder = asyncio.create_task(hold_pending_lock()) + await asyncio.wait_for(lock_acquired.wait(), timeout=1.0) - request_state = proxy_service._WebSocketRequestState( - request_id="req-filter-reconnect", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + create_http_bridge_session = AsyncMock() + monkeypatch.setattr(service, "_create_http_bridge_session", create_http_bridge_session) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) + close_http_bridge_session = AsyncMock() + monkeypatch.setattr(service, "_close_http_bridge_session", close_http_bridge_session) monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - prefer_earlier_reset_accounts=False, - routing_strategy=None, - ) - ) - ), + "get_settings", + lambda: _make_app_settings(), ) - monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) - monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) - monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", open_upstream) - await service._reconnect_http_bridge_session(session, request_state=request_state) + try: + resolved = await asyncio.wait_for( + service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-wedged-idle"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ), + timeout=1.0, + ) + finally: + release_lock.set() + await asyncio.wait_for(lock_holder, timeout=1.0) - assert captured_headers - forwarded = {key.lower(): value for key, value in captured_headers[0].items()} - assert forwarded["session_id"] == "sid-filtered" - assert forwarded["user-agent"] == "pi" - assert forwarded["x-codex-turn-state"] == "upstream-turn-state" - assert "accept" not in forwarded - assert "accept-encoding" not in forwarded - assert "authorization" not in forwarded - assert "connection" not in forwarded - assert "content-type" not in forwarded - assert "cookie" not in forwarded - assert "host" not in forwarded - assert "keep-alive" not in forwarded - assert "proxy-authorization" not in forwarded - assert "proxy-connection" not in forwarded - assert "te" not in forwarded - assert "trailer" not in forwarded - assert "transfer-encoding" not in forwarded - assert "upgrade" not in forwarded - assert "x-codex-turn-metadata" not in forwarded - assert "x-openai-subagent" not in forwarded - assert "x-codex-parent-thread-id" not in forwarded - assert "x-codex-window-id" not in forwarded - assert "x-handshake-debug" not in forwarded + assert resolved is existing_session + assert existing_session.closed is False + assert service._http_bridge_sessions[key] is existing_session + close_http_bridge_session.assert_not_awaited() + create_http_bridge_session.assert_not_awaited() @pytest.mark.asyncio -async def test_reconnect_keeps_handoff_protected_during_lease_swap( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_prune_http_bridge_session_skips_wedged_session_with_visible_pending_request() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - old_lease = proxy_service.AccountLease( - lease_id="lease-old-handoff", - account_id=session.account.id, - kind="stream", - acquired_at=1.0, - ) - new_account = cast(Any, SimpleNamespace(id="acc-replacement", status=AccountStatus.ACTIVE, plan_type="plus")) - new_lease = proxy_service.AccountLease( - lease_id="lease-new-handoff", - account_id=new_account.id, - kind="stream", - acquired_at=2.0, - ) - session.account_lease = old_lease + key = proxy_service._HTTPBridgeSessionKey("request", "bridge-wedged-visible", None) request_state = proxy_service._WebSocketRequestState( - request_id="req-handoff-lease-swap", + request_id="req-wedged-visible", model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", ) - replacement = cast( - UpstreamWebSocket, - SimpleNamespace(response_header=lambda _name: None, close=AsyncMock()), - ) - release_account_lease = AsyncMock() + session = _make_bridge_session( + key=key, + key_value="bridge-wedged-visible", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.last_used_at = time.monotonic() - 300.0 + session.idle_ttl_seconds = 1.0 + service._http_bridge_sessions[key] = session + lock_acquired = asyncio.Event() + release_lock = asyncio.Event() - async def release_lease(lease: proxy_service.AccountLease | None) -> None: - assert lease is old_lease - assert session.closed is True - assert session.handoff_in_progress is True - await release_account_lease(lease) + async def hold_pending_lock() -> None: + async with session.pending_lock: + lock_acquired.set() + await release_lock.wait() - async def select_account(_deadline: float, **_: object) -> proxy_service.AccountSelection: - return proxy_service.AccountSelection(account=new_account, error_message=None, lease=new_lease) + lock_holder = asyncio.create_task(hold_pending_lock()) + await asyncio.wait_for(lock_acquired.wait(), timeout=1.0) + try: + async with service._http_bridge_lock: + sessions_to_close = service._prune_http_bridge_sessions_locked() + finally: + release_lock.set() + await asyncio.wait_for(lock_holder, timeout=1.0) - async def ensure_fresh(account: object, **_: object) -> object: - return account + assert sessions_to_close == [] + assert service._http_bridge_sessions[key] is session + assert session.closed is False - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + +@pytest.mark.asyncio +async def test_get_or_create_http_bridge_session_replaces_live_session_when_account_is_no_longer_assigned( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("request", "bridge-key", "key-1") + stale_session = proxy_service._HTTPBridgeSession( + key=key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-key"), + request_model="gpt-5.4-mini", + account=cast(Any, SimpleNamespace(id="acc-stale", status=AccountStatus.ACTIVE, plan_type="plus")), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + ) + replacement_session = proxy_service._HTTPBridgeSession( + key=key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-key"), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-fresh", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=2.0, + idle_ttl_seconds=120.0, + ) + service._http_bridge_sessions[key] = stale_session + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr( + service, + "_create_http_bridge_session", + AsyncMock(return_value=replacement_session), + ) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - prefer_earlier_reset_accounts=False, - routing_strategy=None, - ) - ) - ), + "get_settings", + lambda: _make_app_settings(), ) - monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) - monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) - monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(return_value=replacement)) - monkeypatch.setattr(service._load_balancer, "release_account_lease", release_lease) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a"])), + ) + close_session = AsyncMock() + monkeypatch.setattr(service, "_close_http_bridge_session", close_session) - await service._reconnect_http_bridge_session(session, request_state=request_state) + reused = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-key"), + api_key=_make_api_key(key_id="key-1", assigned_account_ids=["acc-fresh"]), + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) - release_account_lease.assert_awaited_once_with(old_lease) - assert session.account is new_account - assert session.account_lease is new_lease - assert session.closed is False - assert session.handoff_in_progress is False - assert session.handoff_future is None - assert session.key not in service._http_bridge_inflight_sessions + assert reused is replacement_session + assert service._http_bridge_sessions[key] is replacement_session + assert stale_session.closed is True + await _wait_for_close_await(close_session, stale_session) @pytest.mark.asyncio -async def test_reconnect_cancellation_during_wrong_owner_lease_release_completes_handoff( +async def test_get_or_create_http_bridge_session_replaces_prompt_cache_session_promoted_to_codex( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session() - required_account = cast(Any, SimpleNamespace(id="acc-required", status=AccountStatus.ACTIVE, plan_type="plus")) - replacement_account = cast( - Any, - SimpleNamespace(id="acc-replacement", status=AccountStatus.ACTIVE, plan_type="plus"), + key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-key", "key-1") + stale_session = proxy_service._HTTPBridgeSession( + key=key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-key"), + request_model="gpt-5.4-mini", + account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE, plan_type="plus")), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + codex_session=True, + downstream_turn_state="http_turn_legacy", + downstream_turn_state_aliases={"http_turn_legacy"}, + previous_response_ids=set(), ) - replacement_lease = proxy_service.AccountLease( - lease_id="lease-wrong-owner-cancelled", - account_id=replacement_account.id, - kind="stream", - acquired_at=2.0, + replacement_session = proxy_service._HTTPBridgeSession( + key=key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-key"), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=2.0, + idle_ttl_seconds=120.0, ) - release_started = asyncio.Event() - - async def select_account(_deadline: float, **_: object) -> proxy_service.AccountSelection: - return proxy_service.AccountSelection( - account=replacement_account, - error_message=None, - error_code=None, - lease=replacement_lease, - ) - - async def release_lease(_lease: proxy_service.AccountLease | None) -> None: - release_started.set() - await asyncio.Event().wait() - - request_state = proxy_service._WebSocketRequestState( - request_id="req-wrong-owner-cancelled", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - preferred_account_id=required_account.id, + service._http_bridge_sessions[key] = stale_session + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr( + service, + "_create_http_bridge_session", + AsyncMock(return_value=replacement_session), ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - prefer_earlier_reset_accounts=False, - routing_strategy=None, - ) - ) - ), + "get_settings", + lambda: _make_app_settings(), ) - monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) - monkeypatch.setattr(service._load_balancer, "release_account_lease", release_lease) - - reconnect_task = asyncio.create_task( - service._reconnect_http_bridge_session( - session, - request_state=request_state, - require_preferred_account=True, - ) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a"])), ) - await asyncio.wait_for(release_started.wait(), timeout=1.0) - reconnect_task.cancel() + close_session = AsyncMock() + monkeypatch.setattr(service, "_close_http_bridge_session", close_session) - with pytest.raises(asyncio.CancelledError): - await reconnect_task + reused = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-key"), + api_key=_make_api_key(key_id="key-1", assigned_account_ids=["acc-1"], account_assignment_scope_enabled=True), + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) - assert session.closed is True - assert session.handoff_in_progress is False - assert session.handoff_future is None - assert session.key not in service._http_bridge_inflight_sessions + assert reused is replacement_session + assert service._http_bridge_sessions[key] is replacement_session + assert stale_session.closed is True + await _wait_for_close_await(close_session, stale_session) @pytest.mark.asyncio -async def test_reconnect_http_bridge_session_preserves_hard_account_after_1011( +async def test_get_or_create_http_bridge_session_registers_turn_state_alias_without_rekeying_prompt_cache_session( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-hard-1011", None), - key_value="sid-hard-1011", - ) - session.last_upstream_close_code = 1011 - selection_kwargs: list[dict[str, object]] = [] - - async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: - selection_kwargs.append(kwargs) - return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) - - async def ensure_fresh(account: object, **_: object) -> object: - return account - - upstream = cast(Any, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) - request_state = proxy_service._WebSocketRequestState( - request_id="req-hard-1011", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), + prompt_cache_key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-key", "key-1") + session = proxy_service._HTTPBridgeSession( + key=prompt_cache_key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-key"), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE, plan_type="plus")), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + codex_session=False, + downstream_turn_state=None, + downstream_turn_state_aliases=set(), + previous_response_ids={"resp_prev_1"}, ) + service._http_bridge_sessions[prompt_cache_key] = session + service._http_bridge_previous_response_index[ + proxy_service._http_bridge_previous_response_alias_key("resp_prev_1", "key-1") + ] = prompt_cache_key + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) monkeypatch.setattr( proxy_service, - "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - prefer_earlier_reset_accounts=False, - routing_strategy=None, - ) - ) - ), + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a"])), ) - monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) - monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) - monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(return_value=upstream)) + refresh_durable = AsyncMock() + monkeypatch.setattr(service, "_refresh_durable_http_bridge_session", refresh_durable) - await service._reconnect_http_bridge_session(session, request_state=request_state) + resolved = await service._get_or_create_http_bridge_session( + prompt_cache_key, + headers={"x-codex-turn-state": "http_turn_promoted"}, + affinity=proxy_service._AffinityPolicy(key="bridge-key"), + api_key=_make_api_key(key_id="key-1", assigned_account_ids=["acc-1"], account_assignment_scope_enabled=True), + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + previous_response_id="resp_prev_1", + ) - assert selection_kwargs[0]["preferred_account_id"] == "acc-bridge" - exclude_account_ids = cast(set[str], selection_kwargs[0]["exclude_account_ids"]) - assert "acc-bridge" not in exclude_account_ids - assert session.account.id == "acc-bridge" - assert session.last_upstream_close_code is None + assert resolved is session + assert session.key == prompt_cache_key + assert service._http_bridge_sessions[prompt_cache_key] is session + assert ( + service._http_bridge_previous_response_index[ + proxy_service._http_bridge_previous_response_alias_key("resp_prev_1", "key-1") + ] + == prompt_cache_key + ) + assert ( + service._http_bridge_turn_state_index[ + proxy_service._http_bridge_turn_state_alias_key("http_turn_promoted", "key-1") + ] + == prompt_cache_key + ) + refresh_durable.assert_awaited_once_with(session) @pytest.mark.asyncio -async def test_reconnect_http_bridge_session_fails_closed_when_bound_account_is_excluded( +async def test_stream_via_http_bridge_turn_state_request_ignores_prompt_cache_owner_mismatch( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-hard-excluded", None), - key_value="sid-hard-excluded", + payload = proxy_service.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": "hello"} ) request_state = proxy_service._WebSocketRequestState( - request_id="req-hard-excluded", + request_id="req-hard-turn-state", model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), - excluded_account_ids={session.account.id}, + started_at=1.0, + event_queue=asyncio.Queue(), + transport="http", ) - select_account = AsyncMock() - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + event_queue = request_state.event_queue + assert event_queue is not None + await event_queue.put(None) + + def fake_prepare( + _prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del api_key, api_key_reservation, request_id, client_ip + return request_state, '{"type":"response.create"}' + + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_promoted", None), + headers={"x-codex-turn-state": "http_turn_promoted"}, + affinity=proxy_service._AffinityPolicy( + key="http_turn_promoted", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + ) + captured_key: dict[str, object] = {} + captured_lookup: dict[str, object] = {} + + async def fake_get_or_create_http_bridge_session(*args: object, **kwargs: object): + captured_key["value"] = args[0] + captured_lookup["value"] = kwargs.get("durable_lookup") + return session + monkeypatch.setattr( proxy_service, "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - prefer_earlier_reset_accounts=False, - routing_strategy=None, + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) ) + ), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + service._durable_bridge, + "lookup_request_targets", + AsyncMock( + return_value=proxy_service.DurableBridgeLookup( + session_id="durable-prompt-cache", + canonical_kind="prompt_cache", + canonical_key="cache-derived", + api_key_scope="__anonymous__", + account_id="acc-1", + owner_instance_id="instance-remote", + owner_epoch=1, + lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_promoted", + latest_response_id=None, ) ), ) - monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create_http_bridge_session) + monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) - with pytest.raises(proxy_service.ProxyResponseError) as exc_info: - await service._reconnect_http_bridge_session( - session, - request_state=request_state, - require_same_account=True, + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"x-codex-turn-state": "http_turn_promoted"}, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, ) + ] - assert exc_info.value.status_code == 502 - assert session.account.id == "acc-bridge" - select_account.assert_not_awaited() + assert chunks == [] + assert request_state.affinity_policy.key == "http_turn_promoted" + assert request_state.affinity_policy.kind == proxy_service.StickySessionKind.CODEX_SESSION + key = cast(proxy_service._HTTPBridgeSessionKey, captured_key["value"]) + assert key.affinity_kind == "prompt_cache" + assert key.affinity_key == "cache-derived" + lookup = cast(proxy_service.DurableBridgeLookup, captured_lookup["value"]) + assert lookup.canonical_kind == "prompt_cache" + assert lookup.canonical_key == "cache-derived" + assert lookup.owner_instance_id == "instance-remote" + assert lookup.lease_expires_at is not None @pytest.mark.asyncio -async def test_reconnect_http_bridge_session_keeps_hard_1011_pinned_after_lease_fallback( +async def test_stream_via_http_bridge_durable_outage_does_not_reuse_stale_recovery_alias( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-hard-1011-lease", None), - key_value="sid-hard-1011-lease", - ) - session.last_upstream_close_code = 1011 - session.account_lease = proxy_service.AccountLease( - lease_id="lease-hard-1011", - account_id=session.account.id, - kind="stream", - acquired_at=1.0, + recovery = _make_bridge_session(key=_make_account_neutral_replay_session_key("stale-local-recovery")) + recovery.account = cast( + Any, + SimpleNamespace(id="acc-stale-recovery", status=AccountStatus.ACTIVE, plan_type="plus"), ) - selection_kwargs: list[dict[str, object]] = [] - - async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: - selection_kwargs.append(kwargs) - if len(selection_kwargs) == 1: - return proxy_service.AccountSelection( - account=None, - error_message="Account stream capacity is exhausted", - error_code="account_stream_cap", - ) - return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) - - async def ensure_fresh(account: object, **_: object) -> object: - return account - - sleep_calls = 0 - - async def sleep_for_recovery(*_args: object, **_kwargs: object) -> bool: - nonlocal sleep_calls - sleep_calls += 1 - return sleep_calls == 1 - - upstream = cast(Any, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) - request_state = proxy_service._WebSocketRequestState( - request_id="req-hard-1011-lease", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + stale_turn_state = "http_turn_stale_recovery_owner" + recovery.downstream_turn_state = stale_turn_state + recovery.downstream_turn_state_aliases.add(stale_turn_state) + service._http_bridge_sessions[recovery.key] = recovery + alias_key = proxy_service._http_bridge_turn_state_alias_key(stale_turn_state, None) + service._http_bridge_turn_state_index[alias_key] = recovery.key + get_or_create = AsyncMock() monkeypatch.setattr( proxy_service, "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - prefer_earlier_reset_accounts=False, - routing_strategy=None, + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) ) - ) + ), ), ) - monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) - monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) - monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(return_value=upstream)) - monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) - monkeypatch.setattr(http_bridge_mixin_module, "_sleep_for_account_selection_recovery", sleep_for_recovery) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + service._durable_bridge, + "lookup_request_targets", + AsyncMock(side_effect=RuntimeError("durable metadata unavailable")), + ) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + payload = proxy_service.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": "continue"} + ) - await service._reconnect_http_bridge_session(session, request_state=request_state) + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_via_http_bridge( + payload, + headers={"x-codex-turn-state": stale_turn_state}, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ): + pass - assert len(selection_kwargs) == 2 - assert selection_kwargs[0]["preferred_account_id"] == "acc-bridge" - assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is False - assert selection_kwargs[1]["preferred_account_id"] == "acc-bridge" - assert selection_kwargs[1]["fallback_on_preferred_account_unavailable"] is False - assert session.account.id == "acc-bridge" - assert session.last_upstream_close_code is None + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" + assert exc_info.value.payload["error"]["message"] == "HTTP bridge owner metadata unavailable; retry later." + get_or_create.assert_not_awaited() + assert service._http_bridge_turn_state_index[alias_key] == recovery.key @pytest.mark.asyncio -async def test_reconnect_http_bridge_session_fails_closed_after_hard_1011_owner_connect_errors( +async def test_stream_via_http_bridge_keeps_sse_alive_while_session_creation_waits_for_capacity( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-hard-1011-connect-error", None), - key_value="sid-hard-1011-connect-error", + settings = SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + session = _make_bridge_session(key_value="sid-capacity-create") + get_or_create = AsyncMock( + side_effect=[ + ProxyResponseError( + 503, + openai_error("no_accounts", "Rate limit exceeded. Try again in 120s"), + ), + session, + ] ) - session.last_upstream_close_code = 1011 - other_account = cast(Any, SimpleNamespace(id="acc-other", status=AccountStatus.ACTIVE)) - selection_kwargs: list[dict[str, object]] = [] - - async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: - selection_kwargs.append(kwargs) - account = session.account if len(selection_kwargs) <= 2 else other_account - return proxy_service.AccountSelection(account=account, error_message=None, error_code=None) - - async def ensure_fresh(account: object, **_: object) -> object: - return account - - async def open_upstream(account: object, _headers: dict[str, str], **_: object) -> UpstreamWebSocket: - if getattr(account, "id", None) == "acc-bridge": - raise aiohttp.ClientError("owner reconnect failed") - return cast(UpstreamWebSocket, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) - request_state = proxy_service._WebSocketRequestState( - request_id="req-hard-1011-connect-error", + request_id="req-capacity-create", model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), + started_at=time.monotonic() - 10.0, + transport="http", ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + + def fake_prepare( + _prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del api_key, api_key_reservation, request_id, client_ip + return request_state, '{"type":"response.create"}' + + async def fake_stream_events( + _session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + propagate_http_errors: bool, + downstream_turn_state: str | None, + request_deadline: float | None = None, + ): + del request_state, text_data, queue_limit, propagate_http_errors, downstream_turn_state, request_deadline + yield ( + 'data: {"type":"response.completed","response":{"id":"resp_capacity_create_ok",' + '"usage":{"input_tokens":1,"output_tokens":2}}}\n\n' + ) + monkeypatch.setattr( proxy_service, "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - prefer_earlier_reset_accounts=False, - routing_strategy=None, - ) - ) + lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + proxy_request_budget_seconds=0.001, + http_responses_session_bridge_request_budget_seconds=120.0, ), ) - monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) - monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) - monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", open_upstream) + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 0.001) + monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 0.001) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) - with pytest.raises(aiohttp.ClientError): - await service._reconnect_http_bridge_session(session, request_state=request_state) + payload = proxy_service.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": [], "stream": True} + ) - assert len(selection_kwargs) == 2 - assert selection_kwargs[0]["preferred_account_id"] == "acc-bridge" - assert selection_kwargs[1]["preferred_account_id"] == "acc-bridge" - assert session.account.id == "acc-bridge" - assert session.last_upstream_close_code == 1011 + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"session_id": "sid-capacity-create"}, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] + + keepalive = proxy_service.parse_sse_data_json(chunks[0]) + completed = proxy_service.parse_sse_data_json(chunks[-1]) + + assert keepalive is not None + assert completed is not None + assert keepalive["type"] == "codex.keepalive" + assert keepalive["status"] == "waiting_for_account_capacity" + assert completed["type"] == "response.completed" + assert get_or_create.await_count == 2 + expected_deadline = request_state.started_at + 120.0 + assert get_or_create.await_args_list[0].kwargs["request_deadline"] == pytest.approx(expected_deadline) + assert get_or_create.await_args_list[1].kwargs["request_deadline"] == pytest.approx(expected_deadline) @pytest.mark.asyncio -async def test_reconnect_http_bridge_session_ignores_stale_preferred_account_after_1011( +async def test_stream_via_http_bridge_stops_session_creation_retry_after_budget_wait( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-hard-stale-owner-1011", None), - key_value="sid-hard-stale-owner-1011", + settings = SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, ) - session.last_upstream_close_code = 1011 - selection_kwargs: list[dict[str, object]] = [] - - async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: - selection_kwargs.append(kwargs) - return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) - - async def ensure_fresh(account: object, **_: object) -> object: - return account - - upstream = cast(Any, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + get_or_create = AsyncMock( + side_effect=ProxyResponseError( + 503, + openai_error("no_accounts", "Rate limit exceeded. Try again in 120s"), + ) + ) + now = 100.0 request_state = proxy_service._WebSocketRequestState( - request_id="req-hard-stale-owner-1011", + request_id="req-capacity-create-budget", model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), - preferred_account_id="acc-stale-owner", + started_at=now, + transport="http", ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + + def fake_prepare( + _prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del api_key, api_key_reservation, request_id, client_ip + return request_state, '{"type":"response.create"}' + + async def fake_sleep(seconds: float) -> None: + nonlocal now + now += seconds + monkeypatch.setattr( proxy_service, "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - prefer_earlier_reset_accounts=False, - routing_strategy=None, - ) - ) + lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + proxy_request_budget_seconds=1.0, + http_responses_session_bridge_request_budget_seconds=1.0, ), ) - monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) - monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) - monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(return_value=upstream)) + monkeypatch.setattr( + http_bridge_streaming_module, + "_service_time", + lambda: SimpleNamespace(monotonic=lambda: now), + ) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", fake_sleep) + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 120.0) + monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 120.0) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) - await service._reconnect_http_bridge_session(session, request_state=request_state) + payload = proxy_service.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": [], "stream": True} + ) + chunks: list[str] = [] - assert selection_kwargs[0]["preferred_account_id"] == "acc-bridge" - assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is False - assert session.account.id == "acc-bridge" + with pytest.raises(ProxyResponseError): + async for chunk in service._stream_via_http_bridge( + payload, + headers={"session_id": "sid-capacity-create-budget"}, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ): + chunks.append(chunk) + keepalive = proxy_service.parse_sse_data_json(chunks[0]) -async def test_select_account_with_budget_required_file_pin_does_not_fallback_on_account_cap( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - select_account = AsyncMock( - side_effect=[ - proxy_service.AccountSelection( - account=None, - error_message="Account stream capacity is exhausted", - error_code="account_stream_cap", - ), - proxy_service.AccountSelection( - account=cast(Any, SimpleNamespace(id="acc-other")), - error_message=None, - error_code=None, - ), - ] + assert keepalive is not None + assert keepalive["type"] == "codex.keepalive" + assert keepalive["status"] == "waiting_for_account_capacity" + assert get_or_create.await_count == 1 + + +def test_http_bridge_session_key_infers_strength_from_affinity_kind() -> None: + assert proxy_service._HTTPBridgeSessionKey("turn_state_header", "turn", None).strength == "hard" + assert proxy_service._HTTPBridgeSessionKey("session_header", "session", None).strength == "hard" + assert proxy_service._HTTPBridgeSessionKey("thread_header", "thread", None).strength == "hard" + assert proxy_service._HTTPBridgeSessionKey("prompt_cache", "cache", None).strength == "soft" + assert proxy_service._HTTPBridgeSessionKey("request", "request", None).strength == "soft" + + +def test_http_bridge_session_header_key_is_scoped_by_explicit_prompt_cache_key() -> None: + headers = {"session_id": "process-session"} + first = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [], + "prompt_cache_key": "parent-thread", + } ) - service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock(return_value=SimpleNamespace(sticky_reallocation_budget_threshold_pct=95.0)) - ), + child = first.model_copy(update={"prompt_cache_key": "child-thread"}) + + first_key = proxy_service._make_http_bridge_session_key( + first, + headers=headers, + affinity=proxy_service._AffinityPolicy(), + api_key=None, + request_id="request-1", + explicit_prompt_cache_key="parent-thread", + ) + same_thread_key = proxy_service._make_http_bridge_session_key( + first, + headers=headers, + affinity=proxy_service._AffinityPolicy(), + api_key=None, + request_id="request-2", + explicit_prompt_cache_key="parent-thread", + ) + child_key = proxy_service._make_http_bridge_session_key( + child, + headers=headers, + affinity=proxy_service._AffinityPolicy(), + api_key=None, + request_id="request-3", + explicit_prompt_cache_key="child-thread", ) - selection = await service._select_account_with_budget( - time.monotonic() + 60.0, - request_id="req-file-pin", - kind="stream", - request_stage="first_turn", - prefer_earlier_reset_window="secondary", - preferred_account_id="acc-file-owner", - lease_kind="stream", - fallback_on_preferred_account_unavailable=False, + assert first_key.affinity_kind == "session_header" + assert first_key == same_thread_key + assert first_key != child_key + + +def test_http_bridge_session_header_key_without_prompt_cache_key_stays_legacy_compatible() -> None: + payload = proxy_service.ResponsesRequest.model_validate({"model": "gpt-5.6-sol", "instructions": "", "input": []}) + + key = proxy_service._make_http_bridge_session_key( + payload, + headers={"session_id": "legacy-session"}, + affinity=proxy_service._AffinityPolicy(), + api_key=None, + request_id="request-1", + ) + + assert key == proxy_service._HTTPBridgeSessionKey("session_header", "legacy-session", None) + + +def test_http_bridge_thread_keys_isolate_siblings_with_shared_process_cache_identity() -> None: + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [], + "prompt_cache_key": "process-shared", + } + ) + root_headers = {"session-id": "process-shared", "thread-id": "thread-root"} + child_headers = {"session-id": "process-shared", "thread-id": "thread-child"} + + root_key = proxy_service._make_http_bridge_session_key( + payload, + headers=root_headers, + affinity=proxy_service._AffinityPolicy(), + api_key=None, + request_id="request-root", + explicit_prompt_cache_key="process-shared", + ) + root_retry_key = proxy_service._make_http_bridge_session_key( + payload, + headers=root_headers, + affinity=proxy_service._AffinityPolicy(), + api_key=None, + request_id="request-root-retry", + explicit_prompt_cache_key="process-shared", + ) + child_key = proxy_service._make_http_bridge_session_key( + payload, + headers=child_headers, + affinity=proxy_service._AffinityPolicy(), + api_key=None, + request_id="request-child", + explicit_prompt_cache_key="process-shared", + ) + + assert root_key.affinity_kind == "thread_header" + assert root_key.strength == "hard" + assert root_key == root_retry_key + assert root_key != child_key + assert ( + http_bridge_helpers_module._make_http_bridge_session_header_fallback_key( + headers=root_headers, + api_key=None, + explicit_prompt_cache_key="process-shared", + ) + is None ) - assert selection.account is None - assert selection.error_code == "account_stream_cap" - assert select_account.await_count == 1 - first_call = select_account.await_args_list[0] - assert first_call.kwargs["account_ids"] is None - assert first_call.kwargs["required_account_id"] == "acc-file-owner" - assert first_call.kwargs["required_account_is_ownership_constraint"] is True + +def test_http_bridge_same_thread_unanchored_concurrency_keeps_request_scoped_fork() -> None: + canonical = proxy_service._HTTPBridgeSessionKey("thread_header", "thread-canonical", None) + + fork = http_bridge_helpers_module._http_bridge_parallel_fork_key( + key=canonical, + session=None, + inflight_creation=True, + incoming_turn_state=None, + previous_response_id=None, + request_model="gpt-5.6-sol", + request_service_tier=None, + request_scope_id="second-request", + ) + + assert fork is not None + assert fork.affinity_kind == "internal_unanchored_parallel" + assert fork.affinity_key != canonical.affinity_key @pytest.mark.asyncio -async def test_select_account_with_budget_required_file_pin_overrides_single_account_routing( +@pytest.mark.parametrize("turn_state", [None, "turn-exact-thread-alias"]) +async def test_thread_bridge_durable_lookup_preserves_exact_response_alias_without_legacy_session_fallback( monkeypatch: pytest.MonkeyPatch, + turn_state: str | None, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - select_account = AsyncMock( - return_value=proxy_service.AccountSelection( - account=cast(Any, SimpleNamespace(id="acc-file-owner")), - error_message=None, - error_code=None, + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [], + "prompt_cache_key": "process-shared", + "previous_response_id": "resp-exact-legacy-alias", + } + ) + lookup = AsyncMock( + side_effect=ProxyResponseError( + 409, + openai_error("stop_after_lookup", "stop after durable lookup"), ) ) - service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) monkeypatch.setattr( proxy_service, "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - routing_strategy="single_account", - single_account_id="acc-dashboard-selected", - sticky_reallocation_budget_threshold_pct=95.0, + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) ) - ) + ), ), ) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", lookup) - selection = await service._select_account_with_budget( - time.monotonic() + 60.0, - request_id="req-file-pin-single-account", - kind="stream", - request_stage="first_turn", - prefer_earlier_reset_window="secondary", - preferred_account_id="acc-file-owner", - lease_kind="stream", - fallback_on_preferred_account_unavailable=False, + headers = {"session-id": "process-shared", "thread-id": "thread-child"} + if turn_state is not None: + headers["x-codex-turn-state"] = turn_state + stream = service._stream_via_http_bridge( + payload, + headers=headers, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, ) - assert selection.account is not None - assert selection.account.id == "acc-file-owner" - assert select_account.await_count == 1 - first_call = select_account.await_args_list[0] - assert first_call.kwargs["account_ids"] is None - assert first_call.kwargs["required_account_id"] == "acc-file-owner" - assert first_call.kwargs["required_account_is_ownership_constraint"] is True - assert first_call.kwargs["routing_strategy"] == "capacity_weighted" + with pytest.raises(ProxyResponseError) as exc_info: + await anext(stream) + + assert exc_info.value.payload["error"]["code"] == "stop_after_lookup" + assert lookup.await_args is not None + lookup_kwargs = lookup.await_args.kwargs + assert lookup_kwargs["session_key_kind"] == ("thread_header" if turn_state is None else "turn_state_header") + assert lookup_kwargs["previous_response_id"] == "resp-exact-legacy-alias" + assert lookup_kwargs["session_header"] is None + + +def test_http_bridge_owner_check_required_keeps_prompt_cache_soft() -> None: + key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "cache", None) + + assert proxy_service._http_bridge_owner_check_required(key, gateway_safe_mode=False) is False + assert proxy_service._http_bridge_owner_check_required(key, gateway_safe_mode=True) is False + + +def test_http_bridge_owner_check_required_enables_sticky_thread_in_gateway_safe_mode() -> None: + key = proxy_service._HTTPBridgeSessionKey("sticky_thread", "thread-key", None) + + assert proxy_service._http_bridge_owner_check_required(key, gateway_safe_mode=False) is False + assert proxy_service._http_bridge_owner_check_required(key, gateway_safe_mode=True) is True @pytest.mark.asyncio -async def test_select_account_with_budget_rejects_continuity_owner_outside_single_account_policy( +async def test_stream_via_http_bridge_replaces_retired_hard_gate_before_submit( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - select_account = AsyncMock( - return_value=proxy_service.AccountSelection( - account=None, - error_message="Required continuity owner is outside the effective account policy", - error_code="continuity_owner_policy_conflict", - ) + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "hi", + "input": "continue", + "previous_response_id": "resp-before-retired-gate", + } ) - service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - routing_strategy="single_account", - single_account_id="acc-policy-owner", - sticky_reallocation_budget_threshold_pct=95.0, - ) - ) + retired_session = _make_bridge_session(key_value="sid-retired-gate-replace") + replacement_session = _make_bridge_session(key_value="sid-retired-gate-replace") + get_or_create = AsyncMock(side_effect=[retired_session, replacement_session]) + request_state = proxy_service._WebSocketRequestState( + request_id="req-retired-gate-replace", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + request_text=( + '{"type":"response.create","model":"gpt-5.6-sol","previous_response_id":"resp-before-retired-gate"}' ), + previous_response_id="resp-before-retired-gate", + event_queue=asyncio.Queue(), ) - - selection = await service._select_account_with_budget( - time.monotonic() + 60.0, - request_id="req-continuity-single-account-conflict", - kind="stream", - request_stage="reattach", - preferred_account_id="acc-continuity-owner", - preferred_account_is_continuity_owner=True, - lease_kind="stream", - fallback_on_preferred_account_unavailable=False, + gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( + "http_bridge_response_create_gate", + code="response_create_gate_timeout", ) - assert selection.account is None - assert selection.error_code == "continuity_owner_policy_conflict" - select_account.assert_awaited_once() - selection_call = select_account.await_args - assert selection_call is not None - assert selection_call.kwargs["account_ids"] == {"acc-policy-owner"} - assert selection_call.kwargs["required_account_id"] == "acc-continuity-owner" - assert selection_call.kwargs["required_account_is_ownership_constraint"] is True - assert selection_call.kwargs["required_continuity_owner"] is True - assert selection_call.kwargs["routing_strategy"] == "single_account" - + def fake_prepare( + _prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + **_kwargs: object, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + return request_state, request_state.request_text or "{}" -@pytest.mark.asyncio -async def test_select_account_with_budget_required_preferred_does_not_fallback_when_excluded( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - select_account = AsyncMock( - return_value=proxy_service.AccountSelection( - account=cast(Any, SimpleNamespace(id="acc-other")), - error_message=None, - error_code=None, + async def fake_submit( + session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + ) -> None: + del text_data, queue_limit + if session is retired_session: + retired_session.closed = True + request_state.awaiting_response_created = False + request_state.response_create_gate = None + request_state.response_create_gate_acquired = False + raise gate_timeout_error + assert session is replacement_session + assert request_state.event_queue is not None + request_state.event_queue.put_nowait( + 'data: {"type":"response.completed","response":{"id":"resp-replaced-gate"}}\n\n' ) - ) - service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) + request_state.event_queue.put_nowait(None) + monkeypatch.setattr( proxy_service, "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock(return_value=SimpleNamespace(sticky_reallocation_budget_threshold_pct=95.0)) + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), ), ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-bridge")) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + submit = AsyncMock(side_effect=fake_submit) + detach = AsyncMock() + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(service, "_detach_http_bridge_request", detach) - selection = await service._select_account_with_budget( - time.monotonic() + 60.0, - request_id="req-file-pin-excluded", - kind="stream", - request_stage="retry", - preferred_account_id="acc-file-owner", - exclude_account_ids={"acc-file-owner"}, - fallback_on_preferred_account_unavailable=False, - ) + caplog.set_level(logging.INFO, logger="app.modules.proxy.service") + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"session_id": "sid-retired-gate-replace"}, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] - assert selection.account is None - assert selection.error_code == "preferred_account_unavailable" - select_account.assert_not_awaited() + assert chunks == ['data: {"type":"response.completed","response":{"id":"resp-replaced-gate"}}\n\n'] + assert get_or_create.await_count == 2 + initial_call, replacement_call = get_or_create.await_args_list + assert initial_call.args[0] == replacement_call.args[0] + assert replacement_call.kwargs["allow_forward_to_owner"] is False + assert replacement_call.kwargs["allow_previous_response_recovery_rebind"] is True + assert replacement_call.kwargs["preferred_account_id"] == retired_session.account.id + assert replacement_call.kwargs["fallback_on_preferred_account_unavailable"] is False + assert replacement_call.kwargs["request_deadline"] == initial_call.kwargs["request_deadline"] + assert submit.await_count == 2 + detach.assert_awaited_once_with(replacement_session, request_state=request_state) + assert "event=replace_retired_gate" in caplog.text @pytest.mark.asyncio -async def test_select_account_with_budget_soft_preference_can_fallback_after_account_cap( +async def test_stream_via_http_bridge_replaces_retired_hard_gate_excludes_stuck_account( monkeypatch: pytest.MonkeyPatch, ) -> None: + """Unlike a continuity (previous-response-owner) turn — which is + intentionally re-pinned to the same account via preferred_account_id — + a plain waiter's replacement session must exclude the account whose gate + session just proved stuck, or the load balancer could legally reselect + the exact same wedged account for the "replacement".""" service = proxy_service.ProxyService(cast(Any, nullcontext())) - select_account = AsyncMock( - side_effect=[ - proxy_service.AccountSelection( - account=None, - error_message="Account stream capacity is exhausted", - error_code="account_stream_cap", - ), - proxy_service.AccountSelection( - account=cast(Any, SimpleNamespace(id="acc-other")), - error_message=None, - error_code=None, - ), - ] + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "hi", + "input": "continue", + } ) - service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: SimpleNamespace( - get=AsyncMock(return_value=SimpleNamespace(sticky_reallocation_budget_threshold_pct=95.0)) - ), + retired_session = _make_bridge_session(key_value="sid-retired-gate-exclude") + replacement_session = _make_bridge_session(key_value="sid-retired-gate-exclude") + get_or_create = AsyncMock(side_effect=[retired_session, replacement_session]) + request_state = proxy_service._WebSocketRequestState( + request_id="req-retired-gate-exclude", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + request_text='{"type":"response.create","model":"gpt-5.6-sol"}', + event_queue=asyncio.Queue(), ) - - selection = await service._select_account_with_budget( - time.monotonic() + 60.0, - request_id="req-soft-preferred", - kind="stream", - request_stage="first_turn", - prefer_earlier_reset_window="secondary", - preferred_account_id="acc-soft", - lease_kind="stream", - ) - - assert selection.account is not None - assert selection.account.id == "acc-other" - assert select_account.await_count == 2 - first_call = select_account.await_args_list[0] - second_call = select_account.await_args_list[1] - assert first_call.kwargs["account_ids"] is None - assert first_call.kwargs["required_account_id"] == "acc-soft" - assert second_call.kwargs["account_ids"] is None - assert second_call.kwargs["required_account_id"] is None - - -def test_headers_with_authorization_restores_missing_proxy_api_header() -> None: - headers = proxy_service._headers_with_authorization({"x-request-id": "req-1"}, "Bearer proxy-key") - - assert headers["Authorization"] == "Bearer proxy-key" - assert headers["x-request-id"] == "req-1" - - -def test_headers_with_authorization_does_not_override_existing_value() -> None: - headers = proxy_service._headers_with_authorization({"authorization": "Bearer existing"}, "Bearer proxy-key") - - assert headers["authorization"] == "Bearer existing" - - -def test_make_http_bridge_session_key_prefers_signed_forwarded_affinity_over_generated_turn_state() -> None: - payload = proxy_service.ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": "hi"}) - - key = proxy_service._make_http_bridge_session_key( - payload, - headers={ - "x-codex-turn-state": "http_turn_generated", - "x-codex-bridge-affinity-kind": "session_header", - "x-codex-bridge-affinity-key": "sid-123", - }, - affinity=proxy_service._AffinityPolicy(key="sid-123"), - api_key=None, - request_id="req-1", - allow_forwarded_affinity_headers=True, - ) - - assert key.affinity_kind == "session_header" - assert key.affinity_key == "sid-123" - assert key.strength == "hard" - - -def test_make_http_bridge_session_key_keeps_forwarded_parallel_lane_hard() -> None: - payload = proxy_service.ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": "hi"}) - - key = proxy_service._make_http_bridge_session_key( - payload, - headers={ - "x-codex-bridge-affinity-kind": "internal_unanchored_parallel", - "x-codex-bridge-affinity-key": "fork-request-scope", - }, - affinity=proxy_service._AffinityPolicy(key="fork-request-scope"), - api_key=None, - request_id="duplicate-client-request-id", - allow_forwarded_affinity_headers=True, - ) - - assert key.affinity_kind == "internal_unanchored_parallel" - assert key.affinity_key == "fork-request-scope" - assert key.strength == "hard" - - -def test_make_http_bridge_session_key_ignores_forwarded_affinity_headers_on_public_requests() -> None: - payload = proxy_service.ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": "hi"}) - - key = proxy_service._make_http_bridge_session_key( - payload, - headers={ - "x-codex-bridge-affinity-kind": "session_header", - "x-codex-bridge-affinity-key": "sid-123", - }, - affinity=proxy_service._AffinityPolicy(key="cache-123", kind=proxy_service.StickySessionKind.PROMPT_CACHE), - api_key=None, - request_id="req-1", - allow_forwarded_affinity_headers=False, - ) - - assert key.affinity_kind == "prompt_cache" - assert key.affinity_key == "cache-123" - assert key.strength == "soft" - - -def test_http_bridge_requires_cluster_registration_for_non_loopback_advertise_url() -> None: - settings = Settings( - http_responses_session_bridge_instance_id="instance-a", - http_responses_session_bridge_advertise_base_url="http://instance-a.codex-lb-bridge.default.svc.cluster.local:2455", - ) - - assert proxy_service._http_bridge_requires_cluster_registration(settings) is True - - -def test_http_bridge_requires_cluster_registration_skips_loopback_single_replica() -> None: - settings = Settings(http_responses_session_bridge_advertise_base_url="http://127.0.0.1:2455") - - assert proxy_service._http_bridge_requires_cluster_registration(settings) is False - - -def test_parallel_lane_latest_response_is_a_durable_recovery_anchor() -> None: - lookup = proxy_service.DurableBridgeLookup( - session_id="durable-fork", - canonical_kind="internal_unanchored_parallel", - canonical_key="fork-request-scope", - api_key_scope="__anonymous__", - account_id="acc-owner", - owner_instance_id="instance-b", - owner_epoch=2, - lease_expires_at=proxy_service.utcnow() + timedelta(seconds=60), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_fork", - latest_response_id="resp_fork", - ) - - assert proxy_service._http_bridge_has_durable_recovery_anchor( - previous_response_id=None, - durable_lookup=lookup, - ) - - -def test_durable_bridge_lookup_active_owner_accepts_naive_datetime() -> None: - lookup = proxy_service.DurableBridgeLookup( - session_id="sess-1", - canonical_kind="session_header", - canonical_key="sid-123", - api_key_scope="__anonymous__", - account_id="acc-1", - owner_instance_id="instance-a", - owner_epoch=1, - lease_expires_at=datetime(2099, 1, 1, 0, 0, 0), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state=None, - latest_response_id=None, - ) - - assert proxy_service._durable_bridge_lookup_active_owner(lookup) == "instance-a" - - -@pytest.mark.asyncio -async def test_stream_via_http_bridge_injects_durable_previous_response_anchor( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": "hello"}, - ) - request_state = proxy_service._WebSocketRequestState( - request_id="req-1", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - event_queue=asyncio.Queue(), - transport="http", + gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( + "http_bridge_response_create_gate", + code="response_create_gate_timeout", ) - event_queue = request_state.event_queue - assert event_queue is not None - await event_queue.put(None) - captured: dict[str, object] = {} def fake_prepare( - prepared_payload: proxy_service.ResponsesRequest, + _prepared_payload: proxy_service.ResponsesRequest, _headers: dict[str, str] | Any, - *, - api_key: proxy_service.ApiKeyData | None, - api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, - request_id: str, - client_ip: str | None = None, + **_kwargs: object, ) -> tuple[proxy_service._WebSocketRequestState, str]: - del api_key, api_key_reservation, request_id, client_ip - captured["previous_response_id"] = prepared_payload.previous_response_id - return request_state, '{"type":"response.create"}' + return request_state, request_state.request_text or "{}" - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), - headers={"x-codex-session-id": "sid-123"}, - affinity=proxy_service._AffinityPolicy( - key="sid-123", - kind=proxy_service.StickySessionKind.CODEX_SESSION, - ), - request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, - ) + async def fake_submit( + session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + ) -> None: + del text_data, queue_limit + if session is retired_session: + retired_session.closed = True + request_state.awaiting_response_created = False + request_state.response_create_gate = None + request_state.response_create_gate_acquired = False + raise gate_timeout_error + assert session is replacement_session + assert request_state.event_queue is not None + request_state.event_queue.put_nowait( + 'data: {"type":"response.completed","response":{"id":"resp-replaced-gate-exclude"}}\n\n' + ) + request_state.event_queue.put_nowait(None) monkeypatch.setattr( proxy_service, @@ -8759,38 +8450,24 @@ def fake_prepare( ), ) monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr( - service._durable_bridge, - "lookup_request_targets", - AsyncMock( - return_value=proxy_service.DurableBridgeLookup( - session_id="sess-1", - canonical_kind="session_header", - canonical_key="sid-123", - api_key_scope="__anonymous__", - account_id="acc-1", - owner_instance_id="instance-a", - owner_epoch=1, - lease_expires_at=datetime.now(timezone.utc), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_1", - latest_response_id="resp_latest", - ) - ), - ) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) - monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) - monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + submit = AsyncMock(side_effect=fake_submit) + detach = AsyncMock() + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(service, "_detach_http_bridge_request", detach) chunks = [ chunk async for chunk in service._stream_via_http_bridge( payload, - headers={"x-codex-session-id": "sid-123"}, + headers={"session_id": "sid-retired-gate-exclude"}, codex_session_affinity=True, - propagate_http_errors=False, - openai_cache_affinity=False, + propagate_http_errors=True, + openai_cache_affinity=True, api_key=None, api_key_reservation=None, suppress_text_done_events=False, @@ -8801,92 +8478,79 @@ def fake_prepare( ) ] - assert chunks == [] - assert captured["previous_response_id"] == "resp_latest" + assert chunks == ['data: {"type":"response.completed","response":{"id":"resp-replaced-gate-exclude"}}\n\n'] + assert get_or_create.await_count == 2 + _initial_call, replacement_call = get_or_create.await_args_list + assert replacement_call.kwargs["preferred_account_id"] is None + assert replacement_call.kwargs["exclude_account_ids"] == {retired_session.account.id} @pytest.mark.asyncio -async def test_stream_via_http_bridge_trims_replayed_tool_call_items_with_previous_response_id( +async def test_stream_via_http_bridge_replaces_retired_hard_gate_keeps_pinned_account_unexcluded( monkeypatch: pytest.MonkeyPatch, ) -> None: + """A waiter whose replacement is already required to land on a specific + account (a resolved previous-response owner, or a file-pinned account — + simulated here directly via a pre-set preferred_account_id with no + previous_response_id) must keep that account, unexcluded, even though it + is the same account whose gate just proved stuck. Excluding a waiter's + own required account would make its required-account replacement + impossible and poison every later recovery call on the request.""" service = proxy_service.ProxyService(cast(Any, nullcontext())) payload = proxy_service.ResponsesRequest.model_validate( { - "model": "gpt-5.4", + "model": "gpt-5.6-sol", "instructions": "hi", - "previous_response_id": "resp_prev_tool_call", - "input": [ - {"id": "rs_repeat", "type": "reasoning", "summary": []}, - { - "id": "msg_repeat", - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "running command"}], - }, - { - "id": "fc_repeat", - "type": "function_call", - "call_id": "call_repeat", - "name": "exec_command", - "arguments": '{"cmd":"date"}', - }, - { - "type": "function_call_output", - "call_id": "call_repeat", - "output": "Wed May 6 16:00:00 UTC 2026", - }, - ], + "input": "continue", } ) + retired_session = _make_bridge_session(key_value="sid-retired-gate-pinned") + replacement_session = _make_bridge_session(key_value="sid-retired-gate-pinned") + get_or_create = AsyncMock(side_effect=[retired_session, replacement_session]) request_state = proxy_service._WebSocketRequestState( - request_id="req-trim-tool-call", - model="gpt-5.4", + request_id="req-retired-gate-pinned", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, - event_queue=asyncio.Queue(), + started_at=time.monotonic(), transport="http", + request_text='{"type":"response.create","model":"gpt-5.6-sol"}', + event_queue=asyncio.Queue(), + preferred_account_id=retired_session.account.id, + ) + gate_timeout_error = http_bridge_helpers_module._http_bridge_startup_wait_timeout_error( + "http_bridge_response_create_gate", + code="response_create_gate_timeout", ) - event_queue = request_state.event_queue - assert event_queue is not None - await event_queue.put(None) - captured_input: list[proxy_service.JsonValue] = [] def fake_prepare( - prepared_payload: proxy_service.ResponsesRequest, + _prepared_payload: proxy_service.ResponsesRequest, _headers: dict[str, str] | Any, - *, - api_key: proxy_service.ApiKeyData | None, - api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, - request_id: str, - client_ip: str | None = None, + **_kwargs: object, ) -> tuple[proxy_service._WebSocketRequestState, str]: - del api_key, api_key_reservation, request_id, client_ip - assert isinstance(prepared_payload.input, list) - captured_input[:] = cast(list[proxy_service.JsonValue], prepared_payload.input) - request_state.previous_response_id = prepared_payload.previous_response_id - return request_state, json.dumps({"type": "response.create", "input": prepared_payload.input}) + return request_state, request_state.request_text or "{}" - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), - headers={"x-codex-session-id": "sid-123"}, - affinity=proxy_service._AffinityPolicy( - key="sid-123", - kind=proxy_service.StickySessionKind.CODEX_SESSION, - ), - request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, - ) + async def fake_submit( + session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + ) -> None: + del text_data, queue_limit + if session is retired_session: + retired_session.closed = True + request_state.awaiting_response_created = False + request_state.response_create_gate = None + request_state.response_create_gate_acquired = False + raise gate_timeout_error + assert session is replacement_session + assert request_state.event_queue is not None + request_state.event_queue.put_nowait( + 'data: {"type":"response.completed","response":{"id":"resp-replaced-gate-pinned"}}\n\n' + ) + request_state.event_queue.put_nowait(None) monkeypatch.setattr( proxy_service, @@ -8907,20 +8571,23 @@ def fake_prepare( ) monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-1")) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) - monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) - monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + submit = AsyncMock(side_effect=fake_submit) + detach = AsyncMock() + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(service, "_detach_http_bridge_request", detach) chunks = [ chunk async for chunk in service._stream_via_http_bridge( payload, - headers={"x-codex-session-id": "sid-123"}, + headers={"session_id": "sid-retired-gate-pinned"}, codex_session_affinity=True, - propagate_http_errors=False, - openai_cache_affinity=False, + propagate_http_errors=True, + openai_cache_affinity=True, api_key=None, api_key_reservation=None, suppress_text_done_events=False, @@ -8931,71 +8598,57 @@ def fake_prepare( ) ] - assert chunks == [] - assert captured_input == [ - { - "type": "function_call_output", - "call_id": "call_repeat", - "output": "Wed May 6 16:00:00 UTC 2026", - } - ] + assert chunks == ['data: {"type":"response.completed","response":{"id":"resp-replaced-gate-pinned"}}\n\n'] + assert get_or_create.await_count == 2 + _initial_call, replacement_call = get_or_create.await_args_list + assert replacement_call.kwargs["preferred_account_id"] == retired_session.account.id + assert replacement_call.kwargs["exclude_account_ids"] is None @pytest.mark.asyncio -async def test_stream_via_http_bridge_does_not_inject_session_anchor_for_soft_reuse( +async def test_stream_via_http_bridge_soft_prompt_cache_queue_full_reroutes( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": "hello"}, + { + "model": "gpt-5.4", + "instructions": "hi", + "input": "hello", + "prompt_cache_key": "soft-queue-full", + } ) - request_state = proxy_service._WebSocketRequestState( - request_id="req-soft", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - event_queue=asyncio.Queue(), - transport="http", + saturated_session = _make_bridge_session(key_value="soft-queue-full", queued_request_count=8) + saturated_session.key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "soft-queue-full", None) + reroute_session = _make_bridge_session(key_value="soft-reroute") + capacity_unavailable = ProxyResponseError( + 503, + proxy_service.openai_error("no_accounts", "Rate limit exceeded. Try again in 120s"), ) - event_queue = request_state.event_queue - assert event_queue is not None - await event_queue.put(None) - prepared_previous_response_ids: list[str | None] = [] + get_or_create = AsyncMock(side_effect=[saturated_session, capacity_unavailable, reroute_session]) - def fake_prepare( - prepared_payload: proxy_service.ResponsesRequest, - _headers: dict[str, str] | Any, + async def fake_stream_events( + session: proxy_service._HTTPBridgeSession, *, - api_key: proxy_service.ApiKeyData | None, - api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, - request_id: str, - client_ip: str | None = None, - ) -> tuple[proxy_service._WebSocketRequestState, str]: - del api_key, api_key_reservation, request_id, client_ip - prepared_previous_response_ids.append(prepared_payload.previous_response_id) - return request_state, '{"type":"response.create"}' - - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "cache-123", None), - headers={}, - affinity=proxy_service._AffinityPolicy( - key="cache-123", - kind=proxy_service.StickySessionKind.PROMPT_CACHE, - ), - request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, - last_completed_response_id="resp_soft_latest", - ) + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + propagate_http_errors: bool, + downstream_turn_state: str | None, + request_deadline: float | None = None, + ): + del request_state, text_data, queue_limit, propagate_http_errors, downstream_turn_state, request_deadline + if session is saturated_session: + raise ProxyResponseError( + 429, + proxy_service.openai_error( + "bridge_queue_full", + "HTTP responses session bridge queue is full", + error_type="rate_limit_error", + ), + ) + yield 'data: {"type":"response.completed"}\n\n' monkeypatch.setattr( proxy_service, @@ -9007,7 +8660,6 @@ def fake_prepare( return_value=SimpleNamespace( sticky_threads_enabled=False, openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_enabled=True, http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, http_responses_session_bridge_gateway_safe_mode=False, ) @@ -9015,20 +8667,22 @@ def fake_prepare( ), ), ) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) - monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) - monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 0.001) + monkeypatch.setattr(http_bridge_streaming_module, "_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS", 0.001) + caplog.set_level(logging.INFO, logger="app.modules.proxy.service") chunks = [ chunk async for chunk in service._stream_via_http_bridge( payload, headers={}, codex_session_affinity=False, - propagate_http_errors=False, + propagate_http_errors=True, openai_cache_affinity=True, api_key=None, api_key_reservation=None, @@ -9040,85 +8694,62 @@ def fake_prepare( ) ] - assert chunks == [] - assert prepared_previous_response_ids == [None] - - -@pytest.mark.asyncio -async def test_stream_via_http_bridge_skips_session_anchor_injection_when_trim_would_not_apply( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Guard session-level previous_response_id injection. - - The session anchor must only be injected when the trim branch would - actually strip the already-stored prefix. If the incoming payload is - a full resend whose prefix cannot be trimmed (non-list input, shorter - history, or a prefix fingerprint mismatch), injecting an anchor would - send both the full history and a previous_response_id upstream, which - duplicates context and distorts output/cost. - """ + assert chunks == ['data: {"type":"response.completed"}\n\n'] + assert get_or_create.await_count == 3 + reroute_key = get_or_create.await_args_list[1].args[0] + retry_reroute_key = get_or_create.await_args_list[2].args[0] + assert reroute_key.affinity_kind == "internal_soft_affinity_reroute" + assert reroute_key.strength == "soft" + assert retry_reroute_key.affinity_kind == "internal_soft_affinity_reroute" + assert retry_reroute_key.strength == "soft" + assert get_or_create.await_args_list[1].kwargs["previous_response_id"] is None + assert get_or_create.await_args_list[2].kwargs["previous_response_id"] is None + assert "internal_soft_affinity_reroute" in caplog.text + + +@pytest.mark.asyncio +async def test_stream_via_http_bridge_file_pin_queue_full_does_not_reroute( + monkeypatch: pytest.MonkeyPatch, +) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - # Non-list input: trim cannot possibly apply, so no anchor should be - # injected even though the session has a completed response. payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": "fresh turn text"}, - ) - request_state = proxy_service._WebSocketRequestState( - request_id="req-session-anchor-guard", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - event_queue=asyncio.Queue(), - transport="http", + { + "model": "gpt-5.4", + "instructions": "hi", + "input": [{"type": "input_file", "file_id": "file_doc"}], + } ) - event_queue = request_state.event_queue - assert event_queue is not None - await event_queue.put(None) - prepared_previous_response_ids: list[str | None] = [] + saturated_session = _make_bridge_session(key_value="file-pin-queue-full", queued_request_count=8) + get_or_create = AsyncMock(return_value=saturated_session) - def fake_prepare( - prepared_payload: proxy_service.ResponsesRequest, - _headers: dict[str, str] | Any, + async def fake_stream_events( + session: proxy_service._HTTPBridgeSession, *, - api_key: proxy_service.ApiKeyData | None, - api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, - request_id: str, - client_ip: str | None = None, - ) -> tuple[proxy_service._WebSocketRequestState, str]: - del api_key, api_key_reservation, request_id, client_ip - prepared_previous_response_ids.append(prepared_payload.previous_response_id) - return request_state, '{"type":"response.create"}' - - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-anchor-guard", None), - headers={"x-codex-session-id": "sid-anchor-guard"}, - affinity=proxy_service._AffinityPolicy( - key="sid-anchor-guard", - kind=proxy_service.StickySessionKind.CODEX_SESSION, - ), - request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, - codex_session=True, - last_completed_response_id="resp_session_latest", - last_completed_input_count=3, - last_completed_input_prefix_fingerprint=proxy_service._fingerprint_input_items( - [ - {"role": "user", "content": [{"type": "input_text", "text": "a"}]}, - {"role": "assistant", "content": [{"type": "output_text", "text": "b"}]}, - {"role": "user", "content": [{"type": "input_text", "text": "c"}]}, - ] - ), - ) + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + propagate_http_errors: bool, + downstream_turn_state: str | None, + request_deadline: float | None = None, + ): + del ( + session, + request_state, + text_data, + queue_limit, + propagate_http_errors, + downstream_turn_state, + request_deadline, + ) + raise ProxyResponseError( + 429, + proxy_service.openai_error( + "bridge_queue_full", + "HTTP responses session bridge queue is full", + error_type="rate_limit_error", + ), + ) + yield "" monkeypatch.setattr( proxy_service, @@ -9130,7 +8761,6 @@ def fake_prepare( return_value=SimpleNamespace( sticky_threads_enabled=False, openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_enabled=True, http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, http_responses_session_bridge_gateway_safe_mode=False, ) @@ -9138,20 +8768,17 @@ def fake_prepare( ), ), ) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) - monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) - monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( + with pytest.raises(ProxyResponseError) as info: + async for _ in service._stream_via_http_bridge( payload, - headers={"x-codex-session-id": "sid-anchor-guard"}, - codex_session_affinity=True, - propagate_http_errors=False, + headers={}, + codex_session_affinity=False, + propagate_http_errors=True, openai_cache_affinity=False, api_key=None, api_key_reservation=None, @@ -9160,455 +8787,2467 @@ def fake_prepare( codex_idle_ttl_seconds=1800.0, max_sessions=8, queue_limit=4, - ) - ] + rewritten_file_account_id="acc-file", + ): + pass - assert chunks == [] - # No anchor should have been injected because the non-list input - # would have left the trim branch inert, which would have duplicated - # context upstream. - assert prepared_previous_response_ids == [None] + assert info.value.status_code == 429 + assert get_or_create.await_count == 1 + create_call = get_or_create.await_args + assert create_call is not None + assert create_call.kwargs["preferred_account_id"] == "acc-file" + assert create_call.kwargs["fallback_on_preferred_account_unavailable"] is False -async def _run_session_anchor_owner_stream( +@pytest.mark.asyncio +async def test_select_account_with_budget_prefers_durable_account_id_when_available( monkeypatch: pytest.MonkeyPatch, - *, - account_id: str, - anchor_owner_account_id: str | None, -) -> list[proxy_service.ResponsesRequest]: - """Drive _stream_via_http_bridge for a trimmable session-anchor turn. - - The stored prefix matches the incoming input (so the trim branch WOULD - apply); the only variable is whether the serving account owns the anchor. - Returns the payloads passed to each prepare call. - """ +) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - prefix_items: list[proxy_service.JsonValue] = [ - {"role": "user", "content": [{"type": "input_text", "text": "a"}]}, - {"role": "assistant", "content": [{"type": "output_text", "text": "b"}]}, - {"role": "user", "content": [{"type": "input_text", "text": "c"}]}, - ] - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "hi", - "input": [*prefix_items, {"role": "user", "content": [{"type": "input_text", "text": "d"}]}], - }, - ) - request_state = proxy_service._WebSocketRequestState( - request_id="req-session-anchor-owner", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - event_queue=asyncio.Queue(), - transport="http", - ) - event_queue = request_state.event_queue - assert event_queue is not None - await event_queue.put(None) - prepared_payloads: list[proxy_service.ResponsesRequest] = [] - - def fake_prepare( - prepared_payload: proxy_service.ResponsesRequest, - _headers: dict[str, str] | Any, - *, - api_key: proxy_service.ApiKeyData | None, - api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, - request_id: str, - client_ip: str | None = None, - ) -> tuple[proxy_service._WebSocketRequestState, str]: - del api_key, api_key_reservation, request_id, client_ip - prepared_payloads.append(prepared_payload) - return request_state, '{"type":"response.create"}' - - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-anchor-owner", None), - headers={"x-codex-session-id": "sid-anchor-owner"}, - affinity=proxy_service._AffinityPolicy( - key="sid-anchor-owner", - kind=proxy_service.StickySessionKind.CODEX_SESSION, - ), - request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id=account_id, status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, - codex_session=True, - last_completed_response_id="resp_session_latest", - last_completed_response_account_id=anchor_owner_account_id, - last_completed_input_count=3, - last_completed_input_prefix_fingerprint=proxy_service._fingerprint_input_items(prefix_items), + select_account = AsyncMock( + return_value=proxy_service.AccountSelection( + account=cast(Any, SimpleNamespace(id="acc-preferred")), + error_message=None, + error_code=None, + ) ) - + service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) monkeypatch.setattr( proxy_service, "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_enabled=True, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), + lambda: SimpleNamespace( + get=AsyncMock(return_value=SimpleNamespace(sticky_reallocation_budget_threshold_pct=95.0)) ), ) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) - monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) - monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) - async for _chunk in service._stream_via_http_bridge( - payload, - headers={"x-codex-session-id": "sid-anchor-owner"}, - codex_session_affinity=True, - propagate_http_errors=False, - openai_cache_affinity=False, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - ): - pass - return prepared_payloads + selection = await service._select_account_with_budget( + time.monotonic() + 60.0, + request_id="req-1", + kind="http_bridge", + request_stage="reattach", + prefer_earlier_reset_window="primary", + preferred_account_id="acc-preferred", + ) + + assert selection.account is not None + assert selection.account.id == "acc-preferred" + assert select_account.await_count == 1 + first_call = select_account.await_args_list[0] + assert first_call.kwargs["account_ids"] is None + assert first_call.kwargs["required_account_id"] == "acc-preferred" @pytest.mark.asyncio -async def test_stream_via_http_bridge_injects_session_anchor_when_account_owns_it( +async def test_select_account_with_budget_skips_preferred_account_outside_assignment_scope( monkeypatch: pytest.MonkeyPatch, ) -> None: - # Serving account owns the anchor -> the compact anchor is injected as normal. - prepared = await _run_session_anchor_owner_stream(monkeypatch, account_id="acc-1", anchor_owner_account_id="acc-1") - # Injection re-prepares the payload, so the final (sent) request carries the anchor. - assert prepared[-1].previous_response_id == "resp_session_latest" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + select_account = AsyncMock( + return_value=proxy_service.AccountSelection( + account=cast(Any, SimpleNamespace(id="acc-allowed")), + error_message=None, + error_code=None, + ) + ) + service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock(return_value=SimpleNamespace(sticky_reallocation_budget_threshold_pct=95.0)) + ), + ) + selection = await service._select_account_with_budget( + time.monotonic() + 60.0, + request_id="req-2", + kind="http_bridge", + request_stage="reattach", + api_key=_make_api_key(key_id="key-1", assigned_account_ids=["acc-allowed"]), + prefer_earlier_reset_window="primary", + preferred_account_id="acc-preferred", + ) -@pytest.mark.asyncio -async def test_stream_via_http_bridge_skips_session_anchor_after_cross_account_failover( - monkeypatch: pytest.MonkeyPatch, -) -> None: - # Anchor was created on acc-1 but the session now serves on acc-2 (failover). - # A previous_response_id is account-scoped upstream, so injecting it here would - # send an unresolvable anchor with the history trimmed away -> upstream never - # emits response.created -> the response-create gate wedges. It must be skipped - # and the full history resent instead. - prepared = await _run_session_anchor_owner_stream(monkeypatch, account_id="acc-2", anchor_owner_account_id="acc-1") - assert all(payload.previous_response_id != "resp_session_latest" for payload in prepared) - assert prepared[-1].input == [ - {"role": "user", "content": [{"type": "input_text", "text": "a"}]}, - {"role": "assistant", "content": [{"type": "output_text", "text": "b"}]}, - {"role": "user", "content": [{"type": "input_text", "text": "c"}]}, - {"role": "user", "content": [{"type": "input_text", "text": "d"}]}, - ] + assert selection.account is not None + assert selection.account.id == "acc-allowed" + assert select_account.await_count == 1 + first_call = select_account.await_args_list[0] + assert first_call.kwargs["account_ids"] == {"acc-allowed"} @pytest.mark.asyncio -async def test_stream_via_http_bridge_does_not_inject_durable_previous_response_anchor_for_full_resend_payload( +async def test_select_account_with_budget_classifies_continuity_owner_outside_assignment_scope( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "hi", - "input": [ - {"role": "user", "content": "hello"}, - {"role": "assistant", "content": "world"}, - {"role": "user", "content": "follow up"}, - ], - }, - ) - request_state = proxy_service._WebSocketRequestState( - request_id="req-full-resend", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - event_queue=asyncio.Queue(), - transport="http", - ) - event_queue = request_state.event_queue - assert event_queue is not None - await event_queue.put(None) - captured: dict[str, object] = {} - prepared_input_lengths: list[int] = [] - - def fake_prepare( - prepared_payload: proxy_service.ResponsesRequest, - _headers: dict[str, str] | Any, - *, - api_key: proxy_service.ApiKeyData | None, - api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, - request_id: str, - client_ip: str | None = None, - ) -> tuple[proxy_service._WebSocketRequestState, str]: - del api_key, api_key_reservation, request_id, client_ip - captured["previous_response_id"] = prepared_payload.previous_response_id - inp = prepared_payload.input - prepared_input_lengths.append(len(inp) if isinstance(inp, list) else 1) - return request_state, '{"type":"response.create"}' - - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), - headers={"x-codex-session-id": "sid-123"}, - affinity=proxy_service._AffinityPolicy( - key="sid-123", - kind=proxy_service.StickySessionKind.CODEX_SESSION, - ), - request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, + select_account = AsyncMock( + return_value=proxy_service.AccountSelection( + account=None, + error_message="Required continuity owner is outside the effective account policy", + error_code="continuity_owner_policy_conflict", + ) ) - + service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) monkeypatch.setattr( proxy_service, "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), + lambda: SimpleNamespace( + get=AsyncMock(return_value=SimpleNamespace(sticky_reallocation_budget_threshold_pct=95.0)) ), ) + + selection = await service._select_account_with_budget( + time.monotonic() + 60.0, + request_id="req-continuity-owner-scope", + kind="http_bridge", + request_stage="reattach", + api_key=_make_api_key(key_id="key-1", assigned_account_ids=["acc-allowed"]), + prefer_earlier_reset_window="primary", + preferred_account_id="acc-continuity-owner", + preferred_account_is_continuity_owner=True, + fallback_on_preferred_account_unavailable=False, + ) + + assert selection.error_code == "continuity_owner_policy_conflict" + select_account.assert_awaited_once() + selection_call = select_account.await_args + assert selection_call is not None + assert selection_call.kwargs["account_ids"] == {"acc-allowed"} + assert selection_call.kwargs["required_account_id"] == "acc-continuity-owner" + assert selection_call.kwargs["required_account_is_ownership_constraint"] is True + assert selection_call.kwargs["required_continuity_owner"] is True + + +@pytest.mark.asyncio +async def test_create_http_bridge_session_passes_dashboard_reset_window_to_selection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + settings = SimpleNamespace( + prefer_earlier_reset_accounts=True, + prefer_earlier_reset_window="primary", + routing_strategy="usage_weighted", + ) + selection_kwargs: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + selection_kwargs.append(kwargs) + return proxy_service.AccountSelection(account=None, error_message="No active accounts available") + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) monkeypatch.setattr( - service._durable_bridge, - "lookup_request_targets", - AsyncMock( - return_value=proxy_service.DurableBridgeLookup( - session_id="sess-1", - canonical_kind="session_header", - canonical_key="sid-123", - api_key_scope="__anonymous__", - account_id="acc-1", - owner_instance_id="instance-a", - owner_epoch=1, - lease_expires_at=datetime.now(timezone.utc), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_1", - latest_response_id="resp_latest", - ) - ), + proxy_service, "get_settings_cache", lambda: SimpleNamespace(get=AsyncMock(return_value=settings)) ) - monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) - monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) - monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( - payload, - headers={"x-codex-session-id": "sid-123"}, - codex_session_affinity=True, - propagate_http_errors=False, - openai_cache_affinity=False, + with pytest.raises(ProxyResponseError): + await service._create_http_bridge_session( + proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), + headers={}, + affinity=proxy_service._AffinityPolicy(key="sid-123"), api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, + request_model="gpt-5.4", idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, ) - ] - assert chunks == [] - assert captured["previous_response_id"] is None - # Full-resend payloads are explicitly excluded from durable anchor - # injection, so the bridge prepares the original request exactly once. - assert prepared_input_lengths == [3] - # This path never reaches the trim branch, so the fake request_state - # returned by fake_prepare keeps its default metadata. - assert request_state.input_full_fingerprint is None + assert selection_kwargs[0]["prefer_earlier_reset_accounts"] is True + assert selection_kwargs[0]["prefer_earlier_reset_window"] == "primary" + + +def _pre_dispatch_proxy_error(message: str = "sanitized proxy connect failure") -> ProxyResponseError: + return ProxyResponseError( + 502, + openai_error("upstream_unavailable", message), + failure_phase="connect", + retryable_same_contract=True, + failure_detail="proxy_connect_pre_dispatch", + failure_exception_type="ClientProxyConnectionError", + ) + + +def _bridge_selection_settings() -> SimpleNamespace: + return SimpleNamespace( + prefer_earlier_reset_accounts=False, + prefer_earlier_reset_window="secondary", + routing_strategy="usage_weighted", + ) + + +@pytest.mark.asyncio +async def test_create_http_bridge_session_defers_confirmed_proxy_backoff_until_reservation_release( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + account_a = cast(Any, SimpleNamespace(id="acc-proxy-a", status=AccountStatus.ACTIVE, plan_type="plus")) + account_b = cast(Any, SimpleNamespace(id="acc-proxy-b", status=AccountStatus.ACTIVE, plan_type="plus")) + lease_a = proxy_service.AccountLease("lease-bridge-a", account_a.id, "stream", time.monotonic()) + lease_b = proxy_service.AccountLease("lease-bridge-b", account_b.id, "stream", time.monotonic()) + selections: list[set[str]] = [] + reallocate_flags: list[bool] = [] + released_leases: list[proxy_service.AccountLease] = [] + backed_off_accounts: list[object] = [] + settlement_order: list[str] = [] + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv-http-bridge-proxy-failover", + key_id="key-http-bridge-proxy-failover", + model="gpt-5.4", + ) + lifecycle = proxy_support_module._DeferredAccountBackoffLifecycle(reservation=reservation) + upstream = cast(Any, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + excluded = set(cast(set[str], kwargs["exclude_account_ids"])) + selections.append(excluded) + affinity_policy = cast(proxy_service._AffinityPolicy, kwargs["affinity_policy"]) + reallocate_flags.append(affinity_policy.reallocate_sticky) + if not excluded: + return proxy_service.AccountSelection(account=account_a, error_message=None, lease=lease_a) + return proxy_service.AccountSelection(account=account_b, error_message=None, lease=lease_b) + + async def release_account_lease(lease: proxy_service.AccountLease | None) -> None: + if lease is not None: + released_leases.append(lease) + + async def record_error_backoff(account: object) -> None: + backed_off_accounts.append(account) + # The dead route's stream lease must settle before the health write. + assert lease_a in released_leases + assert settlement_order == ["settle"] + settlement_order.append("backoff") + + async def release_reservation(candidate: object) -> None: + assert candidate is reservation + settlement_order.append("settle") + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=_bridge_selection_settings())), + ) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=[account_a, account_b])) + monkeypatch.setattr( + service, + "_open_upstream_websocket_with_budget", + AsyncMock(side_effect=[_pre_dispatch_proxy_error(), upstream]), + ) + monkeypatch.setattr(service._load_balancer, "release_account_lease", release_account_lease) + monkeypatch.setattr(service._load_balancer, "record_error_backoff", record_error_backoff) + monkeypatch.setattr(service._load_balancer, "record_error", AsyncMock()) + monkeypatch.setattr(service, "_release_websocket_reservation", release_reservation) + monkeypatch.setattr(service, "_relay_http_bridge_upstream_messages", AsyncMock()) + + session = await service._create_http_bridge_session( + proxy_service._HTTPBridgeSessionKey("session_header", "sid-proxy-failover", None), + headers={}, + affinity=proxy_service._AffinityPolicy(key="sid-proxy-failover"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + deferred_account_backoff_lifecycle=lifecycle, + defer_account_health_writes=True, + ) + + assert backed_off_accounts == [] + assert lifecycle.pending_backoffs == {account_a.id: account_a} + await service._release_websocket_reservation(reservation) + lifecycle.settlement_confirmed = True + await service._drain_deferred_account_error_backoffs(lifecycle.pending_backoffs) + + assert session.account is account_b + assert selections == [set(), {account_a.id}] + assert reallocate_flags == [False, True] + assert backed_off_accounts == [account_a] + assert settlement_order == ["settle", "backoff"] + assert lease_a in released_leases + assert lease_b not in released_leases + + +@pytest.mark.asyncio +async def test_create_http_bridge_session_confirmed_proxy_failure_keeps_hard_owner_pinned( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + account = cast(Any, SimpleNamespace(id="acc-proxy-owner", status=AccountStatus.ACTIVE, plan_type="plus")) + lease = proxy_service.AccountLease("lease-bridge-owner", account.id, "stream", time.monotonic()) + select_account = AsyncMock( + return_value=proxy_service.AccountSelection(account=account, error_message=None, lease=lease) + ) + release_account_lease = AsyncMock() + record_error_backoff = AsyncMock() + original_error = _pre_dispatch_proxy_error("owner proxy unavailable") + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=_bridge_selection_settings())), + ) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(side_effect=original_error)) + monkeypatch.setattr(service._load_balancer, "release_account_lease", release_account_lease) + monkeypatch.setattr(service._load_balancer, "record_error_backoff", record_error_backoff) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._create_http_bridge_session( + proxy_service._HTTPBridgeSessionKey("turn_state_header", "turn-owner", None, strength="hard"), + headers={"x-codex-turn-state": "turn-owner"}, + affinity=proxy_service._AffinityPolicy(key="turn-owner"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + preferred_account_id=account.id, + require_preferred_account=True, + fallback_on_preferred_account_unavailable=False, + ) + + # The hard-required owner fails closed on the original sanitized failure. + assert exc_info.value is original_error + select_account.assert_awaited_once() + record_error_backoff.assert_awaited_once_with(account) + assert release_account_lease.await_args_list[0].args == (lease,) + + +@pytest.mark.asyncio +async def test_create_http_bridge_session_preserves_proxy_failure_when_no_replacement( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + account = cast(Any, SimpleNamespace(id="acc-proxy-only", status=AccountStatus.ACTIVE, plan_type="plus")) + lease = proxy_service.AccountLease("lease-bridge-only", account.id, "stream", time.monotonic()) + selections: list[set[str]] = [] + original_error = _pre_dispatch_proxy_error("original bridge proxy failure") + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + excluded = set(cast(set[str], kwargs["exclude_account_ids"])) + selections.append(excluded) + if not excluded: + return proxy_service.AccountSelection(account=account, error_message=None, lease=lease) + return proxy_service.AccountSelection( + account=None, + error_message="No active accounts available", + error_code="no_accounts", + ) + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=_bridge_selection_settings())), + ) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(side_effect=original_error)) + monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) + monkeypatch.setattr(service._load_balancer, "record_error_backoff", AsyncMock()) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._create_http_bridge_session( + proxy_service._HTTPBridgeSessionKey("session_header", "sid-proxy-no-replacement", None), + headers={}, + affinity=proxy_service._AffinityPolicy(key="sid-proxy-no-replacement"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + ) + + # The original sanitized failure is preserved instead of ``no_accounts``. + assert exc_info.value is original_error + assert selections == [set(), {account.id}] + + +@pytest.mark.asyncio +async def test_create_http_bridge_session_idle_close_error_is_not_treated_as_dead_route( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + account = cast(Any, SimpleNamespace(id="acc-proxy-idle", status=AccountStatus.ACTIVE, plan_type="plus")) + lease = proxy_service.AccountLease("lease-bridge-idle", account.id, "stream", time.monotonic()) + record_error_backoff = AsyncMock() + idle_error = ProxyResponseError( + 502, + openai_error("upstream_unavailable", "Upstream websocket closed while idle"), + failure_phase="upstream", + failure_detail="stream_idle_timeout", + ) + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=_bridge_selection_settings())), + ) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=proxy_service.AccountSelection(account=account, error_message=None, lease=lease)), + ) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(side_effect=idle_error)) + monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) + monkeypatch.setattr(service._load_balancer, "record_error_backoff", record_error_backoff) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._create_http_bridge_session( + proxy_service._HTTPBridgeSessionKey("session_header", "sid-proxy-idle", None), + headers={}, + affinity=proxy_service._AffinityPolicy(key="sid-proxy-idle"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + ) + + # An idle disconnect is not provable pre-dispatch evidence: no account + # exclusion, no transient-backoff health write. + assert exc_info.value is idle_error + record_error_backoff.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_passes_dashboard_reset_window_to_selection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + session.request_service_tier = "priority" + settings = SimpleNamespace( + prefer_earlier_reset_accounts=True, + prefer_earlier_reset_window="primary", + routing_strategy="usage_weighted", + ) + selection_kwargs: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + selection_kwargs.append(kwargs) + return proxy_service.AccountSelection(account=None, error_message="No active accounts available") + + request_state = proxy_service._WebSocketRequestState( + request_id="req-reconnect", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + response_create_sent_at=1.0, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, "get_settings_cache", lambda: SimpleNamespace(get=AsyncMock(return_value=settings)) + ) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) + + with pytest.raises(ProxyResponseError): + await service._reconnect_http_bridge_session( + session, + request_state=request_state, + require_same_account=True, + ) + + assert selection_kwargs[0]["prefer_earlier_reset_accounts"] is True + assert selection_kwargs[0]["prefer_earlier_reset_window"] == "primary" + assert selection_kwargs[0]["service_tier"] == "priority" + assert selection_kwargs[0]["preferred_account_id"] == session.account.id + assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is False + assert request_state.response_create_sent_at is None + + +@pytest.mark.asyncio +async def test_reconnect_goal_restart_can_leave_owner_that_failed_before_dispatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-goal-restart", None), + key_value="sid-goal-restart", + ) + old_account = session.account + replacement = cast( + Any, + SimpleNamespace( + id="acc-goal-restart-replacement", + status=AccountStatus.ACTIVE, + plan_type="plus", + ), + ) + selection_kwargs: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + selection_kwargs.append(kwargs) + return proxy_service.AccountSelection(account=replacement, error_message=None) + + replacement_upstream = cast(Any, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-goal-restart-reconnect", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + affinity_policy=proxy_service._AffinityPolicy( + key="sid-goal-restart", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + abandon_unavailable_legacy_owner=True, + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=_bridge_selection_settings())), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=replacement)) + monkeypatch.setattr( + service, + "_open_upstream_websocket_with_budget", + AsyncMock(return_value=replacement_upstream), + ) + + await service._reconnect_http_bridge_session( + session, + request_state=request_state, + require_same_account=True, + ) + + assert selection_kwargs[0]["affinity_policy"] == request_state.affinity_policy + assert selection_kwargs[0]["preferred_account_id"] == old_account.id + assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is True + assert session.account is replacement + + +@pytest.mark.asyncio +async def test_reconnect_account_neutral_recovery_requires_typed_owner_without_callsite_flag( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key=_make_account_neutral_replay_session_key("reconnect-owner")) + settings = SimpleNamespace( + prefer_earlier_reset_accounts=False, + prefer_earlier_reset_window="secondary", + routing_strategy="usage_weighted", + ) + selection_kwargs: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + selection_kwargs.append(kwargs) + return proxy_service.AccountSelection( + account=None, + error_message="Required continuity owner account no longer exists", + error_code=CONTINUITY_OWNER_UNAVAILABLE, + ) + + request_state = proxy_service._WebSocketRequestState( + request_id="req-reconnect-recovery-owner", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), + ) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session(session, request_state=request_state) + + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert selection_kwargs[0]["preferred_account_id"] == session.account.id + assert selection_kwargs[0]["preferred_account_is_continuity_owner"] is True + assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is False + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_uses_bridge_budget_for_capacity_wait( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + settings = SimpleNamespace( + prefer_earlier_reset_accounts=False, + prefer_earlier_reset_window="secondary", + routing_strategy="usage_weighted", + ) + sleep_calls: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection( + account=None, + error_message="Rate limit exceeded. Try again in 120s", + error_code="no_accounts", + ) + + async def sleep_for_recovery(*_args: object, **kwargs: object) -> bool: + sleep_calls.append(kwargs) + return False + + request_state = proxy_service._WebSocketRequestState( + request_id="req-reconnect-bridge-budget", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=100.0, + ) + monkeypatch.setattr(proxy_service.time, "monotonic", lambda: 100.5) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + proxy_request_budget_seconds=0.001, + http_responses_session_bridge_request_budget_seconds=120.0, + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), + ) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) + monkeypatch.setattr(http_bridge_mixin_module, "_sleep_for_account_selection_recovery", sleep_for_recovery) + + with pytest.raises(ProxyResponseError): + await service._reconnect_http_bridge_session(session, request_state=request_state) + + assert sleep_calls + assert sleep_calls[0]["max_sleep_seconds"] == pytest.approx(119.5) + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_skips_capacity_wait_for_usage_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + settings = SimpleNamespace( + prefer_earlier_reset_accounts=False, + prefer_earlier_reset_window="secondary", + routing_strategy="usage_weighted", + ) + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection( + account=None, + error_message="Rate limit exceeded. Try again in 1h", + error_code="usage_limit_reached", + resets_at=1_700_003_600, + ) + + request_state = proxy_service._WebSocketRequestState( + request_id="req-reconnect-usage-limit-now", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=100.0, + ) + monkeypatch.setattr(proxy_service.time, "monotonic", lambda: 100.5) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), + ) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) + monkeypatch.setattr( + http_bridge_mixin_module, + "_sleep_for_account_selection_recovery", + lambda *_args, **_kwargs: pytest.fail("usage_limit_reached must not enter recovery wait"), + ) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session(session, request_state=request_state) + + assert exc_info.value.status_code == 429 + assert exc_info.value.payload["error"]["code"] == "usage_limit_reached" + assert exc_info.value.payload["error"]["type"] == "usage_limit_reached" + assert exc_info.value.payload["error"]["resets_at"] == 1_700_003_600 + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_preserves_owner_error_for_owner_usage_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + settings = SimpleNamespace( + prefer_earlier_reset_accounts=False, + prefer_earlier_reset_window="secondary", + routing_strategy="usage_weighted", + ) + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection( + account=None, + error_message="Rate limit exceeded. Try again in 1h", + error_code="usage_limit_reached", + resets_at=1_700_003_600, + ) + + request_state = proxy_service._WebSocketRequestState( + request_id="req-reconnect-owner-usage-limit", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=100.0, + preferred_account_id=session.account.id, + ) + monkeypatch.setattr(proxy_service.time, "monotonic", lambda: 100.5) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), + ) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) + monkeypatch.setattr( + http_bridge_mixin_module, + "_sleep_for_account_selection_recovery", + lambda *_args, **_kwargs: pytest.fail("owner-only usage_limit_reached must not enter recovery wait"), + ) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session( + session, + request_state=request_state, + require_preferred_account=True, + ) + + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert exc_info.value.payload["error"]["type"] == "server_error" + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_preserves_exclusions_after_capacity_wait( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + settings = SimpleNamespace( + prefer_earlier_reset_accounts=False, + prefer_earlier_reset_window="secondary", + routing_strategy="usage_weighted", + ) + selection_kwargs: list[dict[str, object]] = [] + account = cast(Any, SimpleNamespace(id=session.account.id, status=AccountStatus.ACTIVE)) + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + selection_kwargs.append(kwargs) + if len(selection_kwargs) == 1: + return proxy_service.AccountSelection(account=account, error_message=None) + return proxy_service.AccountSelection( + account=None, + error_message="Rate limit exceeded. Try again in 120s", + error_code="no_accounts", + ) + + async def fail_refresh(*_args: object, **_kwargs: object) -> Any: + raise RefreshError("invalid_grant", "refresh failed", True) + + sleep_calls = 0 + + async def sleep_for_recovery(*_args: object, **_kwargs: object) -> bool: + nonlocal sleep_calls + sleep_calls += 1 + return sleep_calls == 1 + + request_state = proxy_service._WebSocketRequestState( + request_id="req-reconnect-exclusions", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + ) + request_state.excluded_account_ids.add("acc-request-state") + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), + ) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", fail_refresh) + service._load_balancer = cast( + Any, + SimpleNamespace( + mark_permanent_failure=AsyncMock(), + release_account_lease=AsyncMock(), + ), + ) + monkeypatch.setattr(http_bridge_mixin_module, "_sleep_for_account_selection_recovery", sleep_for_recovery) + + with pytest.raises(ProxyResponseError): + await service._reconnect_http_bridge_session(session, request_state=request_state) + + assert len(selection_kwargs) == 3 + assert selection_kwargs[2]["exclude_account_ids"] == {"acc-request-state", session.account.id} + + +@pytest.mark.asyncio +async def test_create_http_bridge_session_filters_http_headers_for_upstream_websocket( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + captured_headers: list[dict[str, str]] = [] + + async def select_account(_deadline: float, **_: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection( + account=cast(Any, SimpleNamespace(id="acc-bridge", status=AccountStatus.ACTIVE)), + error_message=None, + error_code=None, + ) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + async def open_upstream(_account: object, headers: dict[str, str], **_: object) -> UpstreamWebSocket: + captured_headers.append(dict(headers)) + return cast(UpstreamWebSocket, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + + async def fake_relay(_session: proxy_service._HTTPBridgeSession) -> None: + return None + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", open_upstream) + monkeypatch.setattr(service, "_relay_http_bridge_upstream_messages", fake_relay) + + session = await service._create_http_bridge_session( + proxy_service._HTTPBridgeSessionKey("session_header", "sid-filtered", None), + headers={ + "accept": "text/event-stream", + "accept-encoding": "gzip, deflate, br, zstd", + "authorization": "Bearer client-key", + "connection": "keep-alive, x-handshake-debug", + "content-type": "application/json", + "cookie": "session=client-cookie", + "host": "127.0.0.1:3455", + "keep-alive": "timeout=5", + "proxy-authorization": "Basic secret", + "proxy-connection": "keep-alive", + "session_id": "sid-filtered", + "te": "trailers", + "trailer": "x-trailer", + "transfer-encoding": "chunked", + "upgrade": "websocket", + "user-agent": "pi", + "X-Codex-Turn-Metadata": '{"turn_id":"turn-create"}', + "x-OpenAI-Subagent": "collab_spawn", + "X-Codex-Parent-Thread-ID": "parent-create", + "x-CODEX-window-id": "child-create:0", + "x-handshake-debug": "1", + }, + affinity=proxy_service._AffinityPolicy( + key="sid-filtered", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + ) + + if session.upstream_reader is not None: + await session.upstream_reader + assert captured_headers + forwarded = {key.lower(): value for key, value in captured_headers[0].items()} + assert forwarded["session_id"] == "sid-filtered" + assert forwarded["user-agent"] == "pi" + assert "accept" not in forwarded + assert "accept-encoding" not in forwarded + assert "authorization" not in forwarded + assert "connection" not in forwarded + assert "content-type" not in forwarded + assert "cookie" not in forwarded + assert "host" not in forwarded + assert "keep-alive" not in forwarded + assert "proxy-authorization" not in forwarded + assert "proxy-connection" not in forwarded + assert "te" not in forwarded + assert "trailer" not in forwarded + assert "transfer-encoding" not in forwarded + assert "upgrade" not in forwarded + assert "x-codex-turn-metadata" not in forwarded + assert "x-openai-subagent" not in forwarded + assert "x-codex-parent-thread-id" not in forwarded + assert "x-codex-window-id" not in forwarded + assert "x-handshake-debug" not in forwarded + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_filters_http_headers_for_upstream_websocket( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + session.headers = { + "accept": "text/event-stream", + "accept-encoding": "gzip, deflate, br, zstd", + "authorization": "Bearer client-key", + "connection": "keep-alive, x-handshake-debug", + "content-type": "application/json", + "cookie": "session=client-cookie", + "host": "127.0.0.1:3455", + "keep-alive": "timeout=5", + "proxy-authorization": "Basic secret", + "proxy-connection": "keep-alive", + "session_id": "sid-filtered", + "te": "trailers", + "trailer": "x-trailer", + "transfer-encoding": "chunked", + "upgrade": "websocket", + "user-agent": "pi", + "X-Codex-Turn-Metadata": '{"turn_id":"turn-reconnect"}', + "x-OpenAI-Subagent": "collab_spawn", + "X-Codex-Parent-Thread-ID": "parent-reconnect", + "x-CODEX-window-id": "child-reconnect:0", + "x-handshake-debug": "1", + } + session.upstream_turn_state = "upstream-turn-state" + captured_headers: list[dict[str, str]] = [] + + async def select_account(_deadline: float, **_: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + async def open_upstream(_account: object, headers: dict[str, str], **_: object) -> UpstreamWebSocket: + captured_headers.append(dict(headers)) + return cast(UpstreamWebSocket, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + + request_state = proxy_service._WebSocketRequestState( + request_id="req-filter-reconnect", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", open_upstream) + + await service._reconnect_http_bridge_session(session, request_state=request_state) + + assert captured_headers + forwarded = {key.lower(): value for key, value in captured_headers[0].items()} + assert forwarded["session_id"] == "sid-filtered" + assert forwarded["user-agent"] == "pi" + assert forwarded["x-codex-turn-state"] == "upstream-turn-state" + assert "accept" not in forwarded + assert "accept-encoding" not in forwarded + assert "authorization" not in forwarded + assert "connection" not in forwarded + assert "content-type" not in forwarded + assert "cookie" not in forwarded + assert "host" not in forwarded + assert "keep-alive" not in forwarded + assert "proxy-authorization" not in forwarded + assert "proxy-connection" not in forwarded + assert "te" not in forwarded + assert "trailer" not in forwarded + assert "transfer-encoding" not in forwarded + assert "upgrade" not in forwarded + assert "x-codex-turn-metadata" not in forwarded + assert "x-openai-subagent" not in forwarded + assert "x-codex-parent-thread-id" not in forwarded + assert "x-codex-window-id" not in forwarded + assert "x-handshake-debug" not in forwarded + + +@pytest.mark.asyncio +async def test_reconnect_keeps_handoff_protected_during_lease_swap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + old_lease = proxy_service.AccountLease( + lease_id="lease-old-handoff", + account_id=session.account.id, + kind="stream", + acquired_at=1.0, + ) + new_account = cast(Any, SimpleNamespace(id="acc-replacement", status=AccountStatus.ACTIVE, plan_type="plus")) + new_lease = proxy_service.AccountLease( + lease_id="lease-new-handoff", + account_id=new_account.id, + kind="stream", + acquired_at=2.0, + ) + session.account_lease = old_lease + request_state = proxy_service._WebSocketRequestState( + request_id="req-handoff-lease-swap", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + ) + replacement = cast( + UpstreamWebSocket, + SimpleNamespace(response_header=lambda _name: None, close=AsyncMock()), + ) + release_account_lease = AsyncMock() + + async def release_lease(lease: proxy_service.AccountLease | None) -> None: + assert lease is old_lease + assert session.closed is True + assert session.handoff_in_progress is True + await release_account_lease(lease) + + async def select_account(_deadline: float, **_: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection(account=new_account, error_message=None, lease=new_lease) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(return_value=replacement)) + monkeypatch.setattr(service._load_balancer, "release_account_lease", release_lease) + + await service._reconnect_http_bridge_session(session, request_state=request_state) + + release_account_lease.assert_awaited_once_with(old_lease) + assert session.account is new_account + assert session.account_lease is new_lease + assert session.closed is False + assert session.handoff_in_progress is False + assert session.handoff_future is None + assert session.key not in service._http_bridge_inflight_sessions + + +@pytest.mark.asyncio +async def test_reconnect_cancellation_during_wrong_owner_lease_release_completes_handoff( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + required_account = cast(Any, SimpleNamespace(id="acc-required", status=AccountStatus.ACTIVE, plan_type="plus")) + replacement_account = cast( + Any, + SimpleNamespace(id="acc-replacement", status=AccountStatus.ACTIVE, plan_type="plus"), + ) + replacement_lease = proxy_service.AccountLease( + lease_id="lease-wrong-owner-cancelled", + account_id=replacement_account.id, + kind="stream", + acquired_at=2.0, + ) + release_started = asyncio.Event() + + async def select_account(_deadline: float, **_: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection( + account=replacement_account, + error_message=None, + error_code=None, + lease=replacement_lease, + ) + + async def release_lease(_lease: proxy_service.AccountLease | None) -> None: + release_started.set() + await asyncio.Event().wait() + + request_state = proxy_service._WebSocketRequestState( + request_id="req-wrong-owner-cancelled", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + preferred_account_id=required_account.id, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service._load_balancer, "release_account_lease", release_lease) + + reconnect_task = asyncio.create_task( + service._reconnect_http_bridge_session( + session, + request_state=request_state, + require_preferred_account=True, + ) + ) + await asyncio.wait_for(release_started.wait(), timeout=1.0) + reconnect_task.cancel() + + with pytest.raises(asyncio.CancelledError): + await reconnect_task + + assert session.closed is True + assert session.handoff_in_progress is False + assert session.handoff_future is None + assert session.key not in service._http_bridge_inflight_sessions + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_preserves_hard_account_after_1011( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-hard-1011", None), + key_value="sid-hard-1011", + ) + session.last_upstream_close_code = 1011 + selection_kwargs: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + selection_kwargs.append(kwargs) + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + upstream = cast(Any, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-1011", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(return_value=upstream)) + + await service._reconnect_http_bridge_session(session, request_state=request_state) + + assert selection_kwargs[0]["preferred_account_id"] == "acc-bridge" + exclude_account_ids = cast(set[str], selection_kwargs[0]["exclude_account_ids"]) + assert "acc-bridge" not in exclude_account_ids + assert session.account.id == "acc-bridge" + assert session.last_upstream_close_code is None + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_keeps_soft_file_pin_owner_after_1011( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-soft-file-1011", None), + key_value="sid-soft-file-1011", + ) + session.last_upstream_close_code = 1011 + selection_kwargs: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + selection_kwargs.append(kwargs) + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + upstream = cast(Any, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-soft-file-1011", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + preferred_account_id="acc-bridge", + file_required_preferred_account=True, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(return_value=upstream)) + + # when + await service._reconnect_http_bridge_session(session, request_state=request_state) + + # then + assert selection_kwargs[0]["preferred_account_id"] == "acc-bridge" + exclude_account_ids = cast(set[str], selection_kwargs[0]["exclude_account_ids"]) + assert "acc-bridge" not in exclude_account_ids + assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is False + assert session.account.id == "acc-bridge" + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_skips_soft_account_after_1011_without_file_pin( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-soft-1011", None), + key_value="sid-soft-1011", + ) + session.last_upstream_close_code = 1011 + other_account = cast(Any, SimpleNamespace(id="acc-other", status=AccountStatus.ACTIVE, plan_type="plus")) + selection_kwargs: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + selection_kwargs.append(kwargs) + return proxy_service.AccountSelection(account=other_account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + upstream = cast(Any, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-soft-1011", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + preferred_account_id="acc-bridge", + file_required_preferred_account=False, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(return_value=upstream)) + + # when + await service._reconnect_http_bridge_session(session, request_state=request_state) + + # then + exclude_account_ids = cast(set[str], selection_kwargs[0]["exclude_account_ids"]) + assert "acc-bridge" in exclude_account_ids + assert selection_kwargs[0]["preferred_account_id"] is None + assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "selection_error_code", + ["no_accounts", "preferred_account_unavailable", "account_stream_cap"], +) +async def test_reconnect_http_bridge_session_fails_closed_when_file_pin_owner_cannot_be_selected( + monkeypatch: pytest.MonkeyPatch, + selection_error_code: str, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-soft-file-1011-miss", None), + key_value="sid-soft-file-1011-miss", + ) + session.last_upstream_close_code = 1011 + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection( + account=None, + error_message="No available accounts", + error_code=selection_error_code, + ) + + async def sleep_for_recovery(*_args: object, **_kwargs: object) -> bool: + return False + + request_state = proxy_service._WebSocketRequestState( + request_id="req-soft-file-1011-miss", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + preferred_account_id="acc-bridge", + file_required_preferred_account=True, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(http_bridge_mixin_module, "_sleep_for_account_selection_recovery", sleep_for_recovery) + + # when + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session(session, request_state=request_state) + + # then + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert exc_info.value.payload["error"]["type"] == "server_error" + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_fails_closed_when_file_pin_owner_connect_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-soft-file-1011-connect", None), + key_value="sid-soft-file-1011-connect", + ) + session.last_upstream_close_code = 1011 + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + async def open_upstream(*_args: object, **_kwargs: object) -> Any: + raise proxy_service.ProxyResponseError( + 503, + proxy_service.openai_error("upstream_proxy_unavailable", "Upstream proxy unavailable"), + ) + + request_state = proxy_service._WebSocketRequestState( + request_id="req-soft-file-1011-connect", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + preferred_account_id="acc-bridge", + file_required_preferred_account=True, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", open_upstream) + + # when + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session(session, request_state=request_state) + + # then + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert exc_info.value.payload["error"]["type"] == "server_error" + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_fails_closed_when_file_pin_owner_transport_times_out( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-soft-file-1011-timeout", None), + key_value="sid-soft-file-1011-timeout", + ) + session.last_upstream_close_code = 1011 + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + async def open_upstream(*_args: object, **_kwargs: object) -> Any: + raise aiohttp.ClientError("replacement socket timed out") + + request_state = proxy_service._WebSocketRequestState( + request_id="req-soft-file-1011-timeout", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic() - 1.0, + preferred_account_id="acc-bridge", + file_required_preferred_account=True, + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + proxy_request_budget_seconds=0.001, + http_responses_session_bridge_request_budget_seconds=0.001, + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", open_upstream) + + # when + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session(session, request_state=request_state) + + # then + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert exc_info.value.payload["error"]["type"] == "server_error" + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_fails_closed_when_file_pin_owner_refresh_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-soft-file-1011-refresh", None), + key_value="sid-soft-file-1011-refresh", + ) + session.last_upstream_close_code = 1011 + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(*_args: object, **_kwargs: object) -> Any: + raise RefreshError("invalid_grant", "refresh failed", True) + + request_state = proxy_service._WebSocketRequestState( + request_id="req-soft-file-1011-refresh", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic() - 1.0, + preferred_account_id="acc-bridge", + file_required_preferred_account=True, + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + proxy_request_budget_seconds=0.001, + http_responses_session_bridge_request_budget_seconds=0.001, + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + service._load_balancer = cast(Any, SimpleNamespace(mark_permanent_failure=AsyncMock())) + + # when + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session(session, request_state=request_state) + + # then + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert exc_info.value.payload["error"]["type"] == "server_error" + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_fails_closed_when_bound_account_is_excluded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-hard-excluded", None), + key_value="sid-hard-excluded", + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-excluded", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + excluded_account_ids={session.account.id}, + ) + select_account = AsyncMock() + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session( + session, + request_state=request_state, + require_same_account=True, + ) + + assert exc_info.value.status_code == 502 + assert session.account.id == "acc-bridge" + select_account.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_keeps_hard_1011_pinned_after_lease_fallback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-hard-1011-lease", None), + key_value="sid-hard-1011-lease", + ) + session.last_upstream_close_code = 1011 + session.account_lease = proxy_service.AccountLease( + lease_id="lease-hard-1011", + account_id=session.account.id, + kind="stream", + acquired_at=1.0, + ) + selection_kwargs: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + selection_kwargs.append(kwargs) + if len(selection_kwargs) == 1: + return proxy_service.AccountSelection( + account=None, + error_message="Account stream capacity is exhausted", + error_code="account_stream_cap", + ) + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + sleep_calls = 0 + + async def sleep_for_recovery(*_args: object, **_kwargs: object) -> bool: + nonlocal sleep_calls + sleep_calls += 1 + return sleep_calls == 1 + + upstream = cast(Any, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-1011-lease", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(return_value=upstream)) + monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) + monkeypatch.setattr(http_bridge_mixin_module, "_sleep_for_account_selection_recovery", sleep_for_recovery) + + await service._reconnect_http_bridge_session(session, request_state=request_state) + + assert len(selection_kwargs) == 2 + assert selection_kwargs[0]["preferred_account_id"] == "acc-bridge" + assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is False + assert selection_kwargs[1]["preferred_account_id"] == "acc-bridge" + assert selection_kwargs[1]["fallback_on_preferred_account_unavailable"] is False + assert session.account.id == "acc-bridge" + assert session.last_upstream_close_code is None + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_fails_closed_after_hard_1011_owner_connect_errors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-hard-1011-connect-error", None), + key_value="sid-hard-1011-connect-error", + ) + session.last_upstream_close_code = 1011 + other_account = cast(Any, SimpleNamespace(id="acc-other", status=AccountStatus.ACTIVE)) + selection_kwargs: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + selection_kwargs.append(kwargs) + account = session.account if len(selection_kwargs) <= 2 else other_account + return proxy_service.AccountSelection(account=account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + async def open_upstream(account: object, _headers: dict[str, str], **_: object) -> UpstreamWebSocket: + if getattr(account, "id", None) == "acc-bridge": + raise aiohttp.ClientError("owner reconnect failed") + return cast(UpstreamWebSocket, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-1011-connect-error", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", open_upstream) + + with pytest.raises(aiohttp.ClientError): + await service._reconnect_http_bridge_session(session, request_state=request_state) + + assert len(selection_kwargs) == 2 + assert selection_kwargs[0]["preferred_account_id"] == "acc-bridge" + assert selection_kwargs[1]["preferred_account_id"] == "acc-bridge" + assert session.account.id == "acc-bridge" + assert session.last_upstream_close_code == 1011 + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_ignores_stale_preferred_account_after_1011( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-hard-stale-owner-1011", None), + key_value="sid-hard-stale-owner-1011", + ) + session.last_upstream_close_code = 1011 + selection_kwargs: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + selection_kwargs.append(kwargs) + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + upstream = cast(Any, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-stale-owner-1011", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + preferred_account_id="acc-stale-owner", + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(return_value=upstream)) + + await service._reconnect_http_bridge_session(session, request_state=request_state) + + assert selection_kwargs[0]["preferred_account_id"] == "acc-bridge" + assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is False + assert session.account.id == "acc-bridge" + + +async def test_select_account_with_budget_required_file_pin_does_not_fallback_on_account_cap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + select_account = AsyncMock( + side_effect=[ + proxy_service.AccountSelection( + account=None, + error_message="Account stream capacity is exhausted", + error_code="account_stream_cap", + ), + proxy_service.AccountSelection( + account=cast(Any, SimpleNamespace(id="acc-other")), + error_message=None, + error_code=None, + ), + ] + ) + service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock(return_value=SimpleNamespace(sticky_reallocation_budget_threshold_pct=95.0)) + ), + ) + + selection = await service._select_account_with_budget( + time.monotonic() + 60.0, + request_id="req-file-pin", + kind="stream", + request_stage="first_turn", + prefer_earlier_reset_window="secondary", + preferred_account_id="acc-file-owner", + lease_kind="stream", + fallback_on_preferred_account_unavailable=False, + ) + + assert selection.account is None + assert selection.error_code == "account_stream_cap" + assert select_account.await_count == 1 + first_call = select_account.await_args_list[0] + assert first_call.kwargs["account_ids"] is None + assert first_call.kwargs["required_account_id"] == "acc-file-owner" + assert first_call.kwargs["required_account_is_ownership_constraint"] is True + + +@pytest.mark.asyncio +async def test_select_account_with_budget_required_file_pin_overrides_single_account_routing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + select_account = AsyncMock( + return_value=proxy_service.AccountSelection( + account=cast(Any, SimpleNamespace(id="acc-file-owner")), + error_message=None, + error_code=None, + ) + ) + service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + routing_strategy="single_account", + single_account_id="acc-dashboard-selected", + sticky_reallocation_budget_threshold_pct=95.0, + ) + ) + ), + ) + + selection = await service._select_account_with_budget( + time.monotonic() + 60.0, + request_id="req-file-pin-single-account", + kind="stream", + request_stage="first_turn", + prefer_earlier_reset_window="secondary", + preferred_account_id="acc-file-owner", + lease_kind="stream", + fallback_on_preferred_account_unavailable=False, + ) + + assert selection.account is not None + assert selection.account.id == "acc-file-owner" + assert select_account.await_count == 1 + first_call = select_account.await_args_list[0] + assert first_call.kwargs["account_ids"] is None + assert first_call.kwargs["required_account_id"] == "acc-file-owner" + assert first_call.kwargs["required_account_is_ownership_constraint"] is True + assert first_call.kwargs["routing_strategy"] == "capacity_weighted" + + +@pytest.mark.asyncio +async def test_select_account_with_budget_rejects_continuity_owner_outside_single_account_policy( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + select_account = AsyncMock( + return_value=proxy_service.AccountSelection( + account=None, + error_message="Required continuity owner is outside the effective account policy", + error_code="continuity_owner_policy_conflict", + ) + ) + service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + routing_strategy="single_account", + single_account_id="acc-policy-owner", + sticky_reallocation_budget_threshold_pct=95.0, + ) + ) + ), + ) + + selection = await service._select_account_with_budget( + time.monotonic() + 60.0, + request_id="req-continuity-single-account-conflict", + kind="stream", + request_stage="reattach", + preferred_account_id="acc-continuity-owner", + preferred_account_is_continuity_owner=True, + lease_kind="stream", + fallback_on_preferred_account_unavailable=False, + ) + + assert selection.account is None + assert selection.error_code == "continuity_owner_policy_conflict" + select_account.assert_awaited_once() + selection_call = select_account.await_args + assert selection_call is not None + assert selection_call.kwargs["account_ids"] == {"acc-policy-owner"} + assert selection_call.kwargs["required_account_id"] == "acc-continuity-owner" + assert selection_call.kwargs["required_account_is_ownership_constraint"] is True + assert selection_call.kwargs["required_continuity_owner"] is True + assert selection_call.kwargs["routing_strategy"] == "single_account" + + +@pytest.mark.asyncio +async def test_select_account_with_budget_required_preferred_does_not_fallback_when_excluded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + select_account = AsyncMock( + return_value=proxy_service.AccountSelection( + account=cast(Any, SimpleNamespace(id="acc-other")), + error_message=None, + error_code=None, + ) + ) + service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock(return_value=SimpleNamespace(sticky_reallocation_budget_threshold_pct=95.0)) + ), + ) + + selection = await service._select_account_with_budget( + time.monotonic() + 60.0, + request_id="req-file-pin-excluded", + kind="stream", + request_stage="retry", + preferred_account_id="acc-file-owner", + exclude_account_ids={"acc-file-owner"}, + fallback_on_preferred_account_unavailable=False, + ) + + assert selection.account is None + assert selection.error_code == "preferred_account_unavailable" + select_account.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_select_account_with_budget_soft_preference_can_fallback_after_account_cap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + select_account = AsyncMock( + side_effect=[ + proxy_service.AccountSelection( + account=None, + error_message="Account stream capacity is exhausted", + error_code="account_stream_cap", + ), + proxy_service.AccountSelection( + account=cast(Any, SimpleNamespace(id="acc-other")), + error_message=None, + error_code=None, + ), + ] + ) + service._load_balancer = cast(Any, SimpleNamespace(select_account=select_account)) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock(return_value=SimpleNamespace(sticky_reallocation_budget_threshold_pct=95.0)) + ), + ) + + selection = await service._select_account_with_budget( + time.monotonic() + 60.0, + request_id="req-soft-preferred", + kind="stream", + request_stage="first_turn", + prefer_earlier_reset_window="secondary", + preferred_account_id="acc-soft", + lease_kind="stream", + ) + + assert selection.account is not None + assert selection.account.id == "acc-other" + assert select_account.await_count == 2 + first_call = select_account.await_args_list[0] + second_call = select_account.await_args_list[1] + assert first_call.kwargs["account_ids"] is None + assert first_call.kwargs["required_account_id"] == "acc-soft" + assert second_call.kwargs["account_ids"] is None + assert second_call.kwargs["required_account_id"] is None + + +def test_headers_with_authorization_restores_missing_proxy_api_header() -> None: + headers = proxy_service._headers_with_authorization({"x-request-id": "req-1"}, "Bearer proxy-key") + + assert headers["Authorization"] == "Bearer proxy-key" + assert headers["x-request-id"] == "req-1" + + +def test_headers_with_authorization_does_not_override_existing_value() -> None: + headers = proxy_service._headers_with_authorization({"authorization": "Bearer existing"}, "Bearer proxy-key") + + assert headers["authorization"] == "Bearer existing" + + +def test_make_http_bridge_session_key_prefers_signed_forwarded_affinity_over_generated_turn_state() -> None: + payload = proxy_service.ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": "hi"}) + + key = proxy_service._make_http_bridge_session_key( + payload, + headers={ + "x-codex-turn-state": "http_turn_generated", + "x-codex-bridge-affinity-kind": "session_header", + "x-codex-bridge-affinity-key": "sid-123", + }, + affinity=proxy_service._AffinityPolicy(key="sid-123"), + api_key=None, + request_id="req-1", + allow_forwarded_affinity_headers=True, + ) + + assert key.affinity_kind == "session_header" + assert key.affinity_key == "sid-123" + assert key.strength == "hard" + + +def test_make_http_bridge_session_key_keeps_forwarded_thread_affinity_verbatim() -> None: + payload = proxy_service.ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": "hi"}) + + key = proxy_service._make_http_bridge_session_key( + payload, + headers={ + "session-id": "process-local", + "thread-id": "thread-local", + "x-codex-bridge-affinity-kind": "thread_header", + "x-codex-bridge-affinity-key": "opaque-key-from-owner", + }, + affinity=proxy_service._AffinityPolicy(), + api_key=None, + request_id="req-forwarded-thread", + allow_forwarded_affinity_headers=True, + ) + + assert key == proxy_service._HTTPBridgeSessionKey( + "thread_header", + "opaque-key-from-owner", + None, + ) + + +def test_make_http_bridge_session_key_keeps_forwarded_parallel_lane_hard() -> None: + payload = proxy_service.ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": "hi"}) + + key = proxy_service._make_http_bridge_session_key( + payload, + headers={ + "x-codex-bridge-affinity-kind": "internal_unanchored_parallel", + "x-codex-bridge-affinity-key": "fork-request-scope", + }, + affinity=proxy_service._AffinityPolicy(key="fork-request-scope"), + api_key=None, + request_id="duplicate-client-request-id", + allow_forwarded_affinity_headers=True, + ) + + assert key.affinity_kind == "internal_unanchored_parallel" + assert key.affinity_key == "fork-request-scope" + assert key.strength == "hard" + + +def test_make_http_bridge_session_key_ignores_forwarded_affinity_headers_on_public_requests() -> None: + payload = proxy_service.ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": "hi"}) + + key = proxy_service._make_http_bridge_session_key( + payload, + headers={ + "x-codex-bridge-affinity-kind": "session_header", + "x-codex-bridge-affinity-key": "sid-123", + }, + affinity=proxy_service._AffinityPolicy(key="cache-123", kind=proxy_service.StickySessionKind.PROMPT_CACHE), + api_key=None, + request_id="req-1", + allow_forwarded_affinity_headers=False, + ) + + assert key.affinity_kind == "prompt_cache" + assert key.affinity_key == "cache-123" + assert key.strength == "soft" + + +def test_http_bridge_requires_cluster_registration_for_non_loopback_advertise_url() -> None: + settings = Settings( + http_responses_session_bridge_instance_id="instance-a", + http_responses_session_bridge_advertise_base_url="http://instance-a.codex-lb-bridge.default.svc.cluster.local:2455", + ) + + assert proxy_service._http_bridge_requires_cluster_registration(settings) is True + + +def test_http_bridge_requires_cluster_registration_skips_loopback_single_replica() -> None: + settings = Settings(http_responses_session_bridge_advertise_base_url="http://127.0.0.1:2455") + + assert proxy_service._http_bridge_requires_cluster_registration(settings) is False + + +def test_parallel_lane_latest_response_is_a_durable_recovery_anchor() -> None: + lookup = proxy_service.DurableBridgeLookup( + session_id="durable-fork", + canonical_kind="internal_unanchored_parallel", + canonical_key="fork-request-scope", + api_key_scope="__anonymous__", + account_id="acc-owner", + owner_instance_id="instance-b", + owner_epoch=2, + lease_expires_at=proxy_service.utcnow() + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_fork", + latest_response_id="resp_fork", + ) + + assert proxy_service._http_bridge_has_durable_recovery_anchor( + previous_response_id=None, + durable_lookup=lookup, + ) + + +def test_durable_bridge_lookup_active_owner_accepts_naive_datetime() -> None: + lookup = proxy_service.DurableBridgeLookup( + session_id="sess-1", + canonical_kind="session_header", + canonical_key="sid-123", + api_key_scope="__anonymous__", + account_id="acc-1", + owner_instance_id="instance-a", + owner_epoch=1, + lease_expires_at=datetime(2099, 1, 1, 0, 0, 0), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state=None, + latest_response_id=None, + ) + + assert proxy_service._durable_bridge_lookup_active_owner(lookup) == "instance-a" + + +@pytest.mark.asyncio +async def test_stream_via_http_bridge_injects_durable_previous_response_anchor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + payload = proxy_service.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": "hello"}, + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-1", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + event_queue=asyncio.Queue(), + transport="http", + ) + event_queue = request_state.event_queue + assert event_queue is not None + await event_queue.put(None) + captured: dict[str, object] = {} + def fake_prepare( + prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del api_key, api_key_reservation, request_id, client_ip + captured["previous_response_id"] = prepared_payload.previous_response_id + return request_state, '{"type":"response.create"}' -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("suffix_items", "pending_tool_calls", "preserves_full_resend", "forwardable_owner"), - [ - pytest.param( - [ - {"role": "assistant", "content": "hello back"}, - {"role": "user", "content": "follow up"}, - ], - {}, - True, - False, - id="retained-assistant-output", + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), + headers={"x-codex-session-id": "sid-123"}, + affinity=proxy_service._AffinityPolicy( + key="sid-123", + kind=proxy_service.StickySessionKind.CODEX_SESSION, ), - pytest.param( - [ - { - "type": "message", - "role": "assistant", - "phase": "final_answer", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-previous"}, - "content": [{"type": "output_text", "text": "hello back"}], - }, - { - "type": "message", - "role": "user", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, - "content": [{"type": "input_text", "text": "follow up"}], - }, - { - "type": "message", - "role": "developer", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, - "content": [{"type": "input_text", "text": "new control message"}], - }, - ], - {}, - True, - False, - id="retained-assistant-output-with-fresh-developer-followup", + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + ) + + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), ), - pytest.param( - [ - { - "type": "function_call", - "call_id": "call-1", - "name": "lookup", - "arguments": "{}", - }, - { - "type": "function_call_output", - "call_id": "call-1", - "output": "result", - }, - ], - {"call-1": "function_call"}, - True, - False, - id="self-contained-tool-loop", + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + service._durable_bridge, + "lookup_request_targets", + AsyncMock( + return_value=proxy_service.DurableBridgeLookup( + session_id="sess-1", + canonical_kind="session_header", + canonical_key="sid-123", + api_key_scope="__anonymous__", + account_id="acc-1", + owner_instance_id="instance-a", + owner_epoch=1, + lease_expires_at=datetime.now(timezone.utc), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_1", + latest_response_id="resp_latest", + ) ), - pytest.param( - [ - { - "type": "custom_tool_call", - "call_id": "call-1", - "name": "shell", - "input": "pwd", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, - }, + ) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) + monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"x-codex-session-id": "sid-123"}, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] + + assert chunks == [] + assert captured["previous_response_id"] == "resp_latest" + + +@pytest.mark.asyncio +async def test_stream_via_http_bridge_trims_replayed_tool_call_items_with_previous_response_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "previous_response_id": "resp_prev_tool_call", + "input": [ + {"id": "rs_repeat", "type": "reasoning", "summary": []}, { + "id": "msg_repeat", "type": "message", - "role": "developer", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, - "content": [{"type": "input_text", "text": "new control message"}], - }, - { - "type": "custom_tool_call_output", - "call_id": "call-1", - "output": "/workspace", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "running command"}], }, - ], - {"call-1": "custom_tool_call"}, - True, - False, - id="self-contained-tool-loop-with-fresh-developer-interleave", - ), - pytest.param( - [ { + "id": "fc_repeat", "type": "function_call", - "call_id": "call-1", - "name": "lookup", - "arguments": "{}", + "call_id": "call_repeat", + "name": "exec_command", + "arguments": '{"cmd":"date"}', }, { "type": "function_call_output", - "call_id": "call-1", - "output": "result", + "call_id": "call_repeat", + "output": "Wed May 6 16:00:00 UTC 2026", }, ], - None, - False, - False, - id="tool-loop-with-unknown-manifest", - ), - pytest.param( - [{"role": "user", "content": "revise that answer"}], - None, - False, - False, - id="missing-prior-output", + } + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-trim-tool-call", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + event_queue=asyncio.Queue(), + transport="http", + ) + event_queue = request_state.event_queue + assert event_queue is not None + await event_queue.put(None) + captured_input: list[proxy_service.JsonValue] = [] + + def fake_prepare( + prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del api_key, api_key_reservation, request_id, client_ip + assert isinstance(prepared_payload.input, list) + captured_input[:] = cast(list[proxy_service.JsonValue], prepared_payload.input) + request_state.previous_response_id = prepared_payload.previous_response_id + return request_state, json.dumps({"type": "response.create", "input": prepared_payload.input}) + + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), + headers={"x-codex-session-id": "sid-123"}, + affinity=proxy_service._AffinityPolicy( + key="sid-123", + kind=proxy_service.StickySessionKind.CODEX_SESSION, ), - pytest.param( - [{"role": "user", "content": "revise that answer"}], - None, - False, - True, - id="owner-forward-race-missing-prior-output", + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + ) + + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), ), - ], -) -async def test_stream_via_http_bridge_preserves_only_safe_trimmable_full_resend_on_fresh_bridge( + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-1")) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) + monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"x-codex-session-id": "sid-123"}, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] + + assert chunks == [] + assert captured_input == [ + { + "type": "function_call_output", + "call_id": "call_repeat", + "output": "Wed May 6 16:00:00 UTC 2026", + } + ] + + +@pytest.mark.asyncio +async def test_stream_via_http_bridge_does_not_inject_session_anchor_for_soft_reuse( monkeypatch: pytest.MonkeyPatch, - suffix_items: list[proxy_service.JsonValue], - pending_tool_calls: dict[str, str] | None, - preserves_full_resend: bool, - forwardable_owner: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - stored_input_items: list[proxy_service.JsonValue] = [ - { - "type": "additional_tools", - "role": "developer", - "tools": [{"type": "custom", "name": "shell"}], - }, - {"role": "user", "content": "hello"}, - ] - input_items = [*stored_input_items, *suffix_items] payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "hi", - "input": input_items, - "reasoning": { - "context": "last_turn", - "effort": "high", - "summary": "auto", - "vendor_hint": 7, - }, - }, + {"model": "gpt-5.4", "instructions": "hi", "input": "hello"}, ) request_state = proxy_service._WebSocketRequestState( - request_id="req-full-resend-trim", + request_id="req-soft", model="gpt-5.4", service_tier=None, reasoning_effort=None, @@ -9621,10 +11260,6 @@ async def test_stream_via_http_bridge_preserves_only_safe_trimmable_full_resend_ assert event_queue is not None await event_queue.put(None) prepared_previous_response_ids: list[str | None] = [] - prepared_input_lengths: list[int] = [] - prepared_frames: list[dict[str, Any]] = [] - prepare_call_count = 0 - real_prepare = service._prepare_http_bridge_request def fake_prepare( prepared_payload: proxy_service.ResponsesRequest, @@ -9634,38 +11269,130 @@ def fake_prepare( api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, request_id: str, client_ip: str | None = None, - **kwargs: Any, ) -> tuple[proxy_service._WebSocketRequestState, str]: - # The recovery journal fingerprint is prepared from the same payload - # before the request is sent. It is internal bookkeeping, not a - # second upstream dispatch, so keep it out of dispatch assertions. - nonlocal prepare_call_count - prepare_call_count += 1 - record_dispatch = not (preserves_full_resend and prepare_call_count == 1) - if record_dispatch: - prepared_previous_response_ids.append(prepared_payload.previous_response_id) - inp = prepared_payload.input - if record_dispatch: - prepared_input_lengths.append(len(inp) if isinstance(inp, list) else 1) - _, text_data = real_prepare( - prepared_payload, - _headers, - api_key=api_key, - api_key_reservation=api_key_reservation, - request_id=request_id, - client_ip=client_ip, - **kwargs, + del api_key, api_key_reservation, request_id, client_ip + prepared_previous_response_ids.append(prepared_payload.previous_response_id) + return request_state, '{"type":"response.create"}' + + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "cache-123", None), + headers={}, + affinity=proxy_service._AffinityPolicy( + key="cache-123", + kind=proxy_service.StickySessionKind.PROMPT_CACHE, + ), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + last_completed_response_id="resp_soft_latest", + ) + + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_enabled=True, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) + monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, ) - if record_dispatch: - prepared_frames.append(json.loads(text_data)) - request_state.previous_response_id = prepared_payload.previous_response_id - return request_state, text_data + ] + + assert chunks == [] + assert prepared_previous_response_ids == [None] + + +@pytest.mark.asyncio +async def test_stream_via_http_bridge_skips_session_anchor_injection_when_trim_would_not_apply( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Guard session-level previous_response_id injection. + + The session anchor must only be injected when the trim branch would + actually strip the already-stored prefix. If the incoming payload is + a full resend whose prefix cannot be trimmed (non-list input, shorter + history, or a prefix fingerprint mismatch), injecting an anchor would + send both the full history and a previous_response_id upstream, which + duplicates context and distorts output/cost. + """ + service = proxy_service.ProxyService(cast(Any, nullcontext())) + # Non-list input: trim cannot possibly apply, so no anchor should be + # injected even though the session has a completed response. + payload = proxy_service.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": "fresh turn text"}, + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-session-anchor-guard", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + event_queue=asyncio.Queue(), + transport="http", + ) + event_queue = request_state.event_queue + assert event_queue is not None + await event_queue.put(None) + prepared_previous_response_ids: list[str | None] = [] + + def fake_prepare( + prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del api_key, api_key_reservation, request_id, client_ip + prepared_previous_response_ids.append(prepared_payload.previous_response_id) + return request_state, '{"type":"response.create"}' session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), - headers={"x-codex-session-id": "sid-123"}, + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-anchor-guard", None), + headers={"x-codex-session-id": "sid-anchor-guard"}, affinity=proxy_service._AffinityPolicy( - key="sid-123", + key="sid-anchor-guard", kind=proxy_service.StickySessionKind.CODEX_SESSION, ), request_model="gpt-5.4", @@ -9678,6 +11405,16 @@ def fake_prepare( queued_request_count=0, last_used_at=1.0, idle_ttl_seconds=120.0, + codex_session=True, + last_completed_response_id="resp_session_latest", + last_completed_input_count=3, + last_completed_input_prefix_fingerprint=proxy_service._fingerprint_input_items( + [ + {"role": "user", "content": [{"type": "input_text", "text": "a"}]}, + {"role": "assistant", "content": [{"type": "output_text", "text": "b"}]}, + {"role": "user", "content": [{"type": "input_text", "text": "c"}]}, + ] + ), ) monkeypatch.setattr( @@ -9690,6 +11427,7 @@ def fake_prepare( return_value=SimpleNamespace( sticky_threads_enabled=False, openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_enabled=True, http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, http_responses_session_bridge_gateway_safe_mode=False, ) @@ -9697,49 +11435,10 @@ def fake_prepare( ), ), ) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr( - service._durable_bridge, - "lookup_request_targets", - AsyncMock( - return_value=proxy_service.DurableBridgeLookup( - session_id="sess-1", - canonical_kind="session_header", - canonical_key="sid-123", - api_key_scope="__anonymous__", - account_id="acc-1", - owner_instance_id="instance-b" if forwardable_owner else "instance-a", - owner_epoch=1, - lease_expires_at=( - datetime.now(timezone.utc) + timedelta(seconds=60) - if forwardable_owner - else datetime.now(timezone.utc) - ), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_1", - latest_response_id="resp_latest", - latest_input_item_count=len(stored_input_items), - latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input_items), - latest_pending_tool_calls=pending_tool_calls, - ) - ), - ) - session.codex_session = True - account_neutral_classifier = Mock(return_value=True) - monkeypatch.setattr( - http_bridge_streaming_module, - "_http_bridge_payload_is_account_neutral_fresh_replay", - account_neutral_classifier, - ) - get_or_create = AsyncMock(return_value=session) monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr( - service, - "_http_bridge_can_forward_to_active_owner", - AsyncMock(return_value=forwardable_owner), - ) - monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-1")) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) @@ -9747,7 +11446,7 @@ def fake_prepare( chunk async for chunk in service._stream_via_http_bridge( payload, - headers={"x-codex-session-id": "sid-123"}, + headers={"x-codex-session-id": "sid-anchor-guard"}, codex_session_affinity=True, propagate_http_errors=False, openai_cache_affinity=False, @@ -9762,576 +11461,672 @@ def fake_prepare( ] assert chunks == [] - assert prepared_previous_response_ids == ([None] if preserves_full_resend else [None, "resp_latest", "resp_latest"]) - assert prepared_input_lengths == ( - [len(input_items)] if preserves_full_resend else [len(input_items), len(input_items), len(suffix_items)] - ) - assert all("tools" not in frame for frame in prepared_frames) - normalized_input_items = cast(list[proxy_service.JsonValue], payload.input) - expected_input_items = ( - normalized_input_items if preserves_full_resend else normalized_input_items[-len(suffix_items) :] - ) - assert prepared_frames[-1]["input"] == expected_input_items - assert [frame["client_metadata"][CODEX_RESPONSES_LITE_WEBSOCKET_METADATA_KEY] for frame in prepared_frames] == [ - "true", - ] * len(prepared_frames) - assert all( - frame["reasoning"] - == { - "context": "all_turns", - "effort": "high", - "summary": "auto", - "vendor_hint": 7, - } - for frame in prepared_frames - ) - assert cast(dict[str, Any], payload.to_payload()["reasoning"])["context"] == "last_turn" - creation = get_or_create.await_args - assert creation is not None - assert creation.kwargs["previous_response_id"] == ( - None if preserves_full_resend or forwardable_owner else "resp_latest" - ) - assert creation.kwargs["preferred_account_id"] == "acc-1" - assert session.last_completed_response_id == (None if preserves_full_resend else "resp_latest") - assert session.last_completed_response_account_id == (None if preserves_full_resend else "acc-1") - if not preserves_full_resend: - assert request_state.proxy_injected_previous_response_id is True - assert request_state.fresh_upstream_request_is_retry_safe is False - assert request_state.fresh_upstream_request_is_account_neutral is False - if preserves_full_resend: - account_neutral_classifier.assert_called_once() - else: - account_neutral_classifier.assert_not_called() - create_call = get_or_create.await_args - assert create_call is not None - create_kwargs = create_call.kwargs - create_headers = {key.lower(): value for key, value in create_kwargs["headers"].items()} - create_affinity = cast(proxy_service._AffinityPolicy, create_kwargs["affinity"]) - if preserves_full_resend and not forwardable_owner: - assert "x-codex-session-id" not in create_headers - assert create_affinity.kind == proxy_service.StickySessionKind.CODEX_SESSION - assert create_affinity.key is None - assert create_affinity.codex_session_source is None - assert create_kwargs["session_header_fallback_key"] is None - assert create_kwargs["preferred_account_id"] == "acc-1" - assert create_kwargs["preferred_account_has_continuity_provenance"] is True - else: - assert create_headers["x-codex-session-id"] == "sid-123" - assert create_affinity.key == "sid-123" - assert create_affinity.codex_session_source == "session_header" - - -def test_verified_durable_full_resend_proof_is_sealed_immutable_and_request_bound() -> None: - stored_input_items: list[proxy_service.JsonValue] = [ - { - "type": "additional_tools", - "role": "developer", - "tools": [{"type": "custom", "name": "shell"}], - }, - {"role": "user", "content": "hello"}, - ] - full_input = [ - *stored_input_items, - {"role": "assistant", "content": "hello back"}, - {"role": "user", "content": "follow up"}, - ] - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "hi", - "input": full_input, - } - ) - durable_lookup = proxy_service.DurableBridgeLookup( - session_id="sess-proof", - canonical_kind="session_header", - canonical_key="sid-proof", - api_key_scope="__anonymous__", - account_id="acc-proof", - owner_instance_id=None, - owner_epoch=3, - lease_expires_at=None, - state=HttpBridgeSessionState.CLOSED, - latest_turn_state="http_turn_proof", - latest_response_id="resp-proof", - latest_input_item_count=len(stored_input_items), - latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input_items), - latest_pending_tool_calls={}, - model="gpt-5.4", - ) - - with pytest.raises(TypeError, match="created only by the verifier"): - http_bridge_streaming_module._VerifiedDurableFullResend( - _token=object(), - durable_session_id=durable_lookup.session_id, - owner_account_id=cast(str, durable_lookup.account_id), - latest_response_id=cast(str, durable_lookup.latest_response_id), - stored_input_item_count=len(stored_input_items), - stored_input_fingerprint=cast(str, durable_lookup.latest_input_full_fingerprint), - full_input_fingerprint=proxy_service._fingerprint_input_items(full_input), - pending_tool_calls=None, - ) - - proof = http_bridge_streaming_module._verify_durable_full_resend(payload, durable_lookup) - assert proof is not None - assert proof.matches(payload, durable_lookup) is True - assert copy.copy(proof) is proof - assert copy.deepcopy(proof) is proof - with pytest.raises(AttributeError, match="immutable"): - proof._owner_account_id = "acc-forged" # type: ignore[misc] - with pytest.raises(TypeError, match="cannot be serialized"): - pickle.dumps(proof) - - changed_payload = payload.model_copy( - update={ - "input": [ - *stored_input_items, - {"role": "assistant", "content": "different output"}, - {"role": "user", "content": "follow up"}, - ] - } - ) - assert proof.matches(changed_payload, durable_lookup) is False - substituted_durable_lookups = ( - replace(durable_lookup, session_id="sess-other"), - replace(durable_lookup, account_id="acc-other"), - replace(durable_lookup, latest_response_id="resp-other"), - replace(durable_lookup, latest_input_item_count=len(stored_input_items) + 1), - replace(durable_lookup, latest_input_full_fingerprint="fingerprint-other"), - replace(durable_lookup, latest_pending_tool_calls={"call-other": "function_call"}), - ) - assert all(proof.matches(payload, lookup) is False for lookup in substituted_durable_lookups) - - incomplete_payload = payload.model_copy( - update={"input": [*stored_input_items, {"role": "user", "content": "follow up"}]} - ) - assert http_bridge_streaming_module._verify_durable_full_resend(incomplete_payload, durable_lookup) is None - - -def test_verified_durable_full_resend_accepts_manifest_bound_fourcam_shape() -> None: - stored_input: list[JsonValue] = [{"type": "message", "role": "user", "content": "stored"}] - response_output: list[JsonValue] = [ - { - "type": "reasoning", - "id": "rs_manifest_full_resend", - "encrypted_content": "opaque", - "summary": [], - "status": "completed", - }, - { - "type": "message", - "id": "msg_manifest_full_resend", - "role": "assistant", - "phase": "commentary", - "content": [{"type": "output_text", "text": "checking"}], - }, - { - "type": "custom_tool_call", - "id": "ctc_manifest_full_resend", - "call_id": "call_manifest_full_resend", - "name": "shell", - "input": "rustfmt --check", - "status": "completed", - }, - ] - pending = {"call_manifest_full_resend": "custom_tool_call"} - manifest = build_response_transition_manifest( - { - "response": { - "id": "resp_manifest_full_resend", - "status": "completed", - "output": response_output, - } - }, - pending_tool_calls=pending, - ) - assert manifest is not None - full_input: list[JsonValue] = [ - *stored_input, - *response_output, - { - "type": "custom_tool_call_output", - "id": "ctco_manifest_full_resend", - "call_id": "call_manifest_full_resend", - "output": "verified", - "status": "completed", - }, - { - "type": "message", - "id": "msg_00000000-0000-4000-8000-000000000401", - "role": "developer", - "content": [{"type": "input_text", "text": "retry context"}], - "internal_chat_message_metadata_passthrough": {"turn_id": "00000000-0000-4000-8000-000000000402"}, - }, - { - "type": "message", - "id": "msg_00000000-0000-4000-8000-000000000403", - "role": "user", - "content": [{"type": "input_text", "text": "retry"}], - "internal_chat_message_metadata_passthrough": { - "turn_id": "00000000-0000-4000-8000-000000000402", - "create_time": 1.0, - }, - }, - ] - payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.6-sol", "instructions": "continue", "input": full_input} - ) - lookup = proxy_service.DurableBridgeLookup( - session_id="sess-manifest-full-resend", - canonical_kind="session_header", - canonical_key="sid-manifest-full-resend", - api_key_scope="__anonymous__", - account_id="acc-manifest-full-resend", - owner_instance_id=None, - owner_epoch=1, - lease_expires_at=None, - state=HttpBridgeSessionState.CLOSED, - latest_turn_state="http_turn_manifest_full_resend", - latest_response_id="resp_manifest_full_resend", - latest_input_item_count=len(stored_input), - latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input), - latest_pending_tool_calls=pending, - latest_response_transition_manifest=manifest, - model="gpt-5.6-sol", - ) - - proof = http_bridge_streaming_module._verify_durable_full_resend(payload, lookup) + # No anchor should have been injected because the non-list input + # would have left the trim branch inert, which would have duplicated + # context upstream. + assert prepared_previous_response_ids == [None] - assert proof is not None - assert proof.matches(payload, lookup) - changed_output = copy.deepcopy(response_output) - cast(dict[str, JsonValue], changed_output[1])["content"] = [{"type": "output_text", "text": "different"}] - changed_manifest = build_response_transition_manifest( - { - "response": { - "id": "resp_manifest_full_resend", - "status": "completed", - "output": changed_output, - } - }, - pending_tool_calls=pending, - ) - assert changed_manifest is not None - assert not proof.matches( - payload, - replace(lookup, latest_response_transition_manifest=changed_manifest), - ) +async def _run_session_anchor_owner_stream( + monkeypatch: pytest.MonkeyPatch, + *, + account_id: str, + anchor_owner_account_id: str | None, +) -> list[proxy_service.ResponsesRequest]: + """Drive _stream_via_http_bridge for a trimmable session-anchor turn. -def test_verified_durable_full_resend_accepts_response_bound_pending_tool_calls() -> None: - stored_input_items: list[proxy_service.JsonValue] = [ - {"role": "user", "content": "look that up"}, - ] - full_input: list[proxy_service.JsonValue] = [ - *stored_input_items, - { - "type": "function_call", - "call_id": "call-1", - "name": "lookup", - "arguments": "{}", - }, - { - "type": "function_call_output", - "call_id": "call-1", - "output": "result", - }, + The stored prefix matches the incoming input (so the trim branch WOULD + apply); the only variable is whether the serving account owns the anchor. + Returns the payloads passed to each prepare call. + """ + service = proxy_service.ProxyService(cast(Any, nullcontext())) + prefix_items: list[proxy_service.JsonValue] = [ + {"role": "user", "content": [{"type": "input_text", "text": "a"}]}, + {"role": "assistant", "content": [{"type": "output_text", "text": "b"}]}, + {"role": "user", "content": [{"type": "input_text", "text": "c"}]}, ] payload = proxy_service.ResponsesRequest.model_validate( { "model": "gpt-5.4", "instructions": "hi", - "input": full_input, - } + "input": [*prefix_items, {"role": "user", "content": [{"type": "input_text", "text": "d"}]}], + }, ) - durable_lookup = proxy_service.DurableBridgeLookup( - session_id="sess-tool-proof", - canonical_kind="session_header", - canonical_key="sid-tool-proof", - api_key_scope="__anonymous__", - account_id="acc-proof", - owner_instance_id=None, - owner_epoch=3, - lease_expires_at=None, - state=HttpBridgeSessionState.CLOSED, - latest_turn_state="http_turn_tool_proof", - latest_response_id="resp-tool-proof", - latest_input_item_count=len(stored_input_items), - latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input_items), + request_state = proxy_service._WebSocketRequestState( + request_id="req-session-anchor-owner", model="gpt-5.4", - latest_pending_tool_calls={"call-1": "function_call"}, - ) - - proof = http_bridge_streaming_module._verify_durable_full_resend(payload, durable_lookup) - - assert proof is not None - assert proof.matches(payload, durable_lookup) is True - assert ( - proof.matches( - payload, - replace(durable_lookup, latest_pending_tool_calls={"call-other": "function_call"}), - ) - is False - ) - assert ( - http_bridge_streaming_module._verify_durable_full_resend( - payload, - replace(durable_lookup, latest_pending_tool_calls=None), - ) - is None + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + event_queue=asyncio.Queue(), + transport="http", ) + event_queue = request_state.event_queue + assert event_queue is not None + await event_queue.put(None) + prepared_payloads: list[proxy_service.ResponsesRequest] = [] + def fake_prepare( + prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del api_key, api_key_reservation, request_id, client_ip + prepared_payloads.append(prepared_payload) + return request_state, '{"type":"response.create"}' -def test_verified_agent_message_recovery_requires_explicitly_empty_tool_manifest() -> None: - stored_input_items: list[proxy_service.JsonValue] = [ - {"role": "user", "content": "first question"}, - ] - agent_message: dict[str, proxy_service.JsonValue] = { - "type": "agent_message", - "id": "amsg_01a02b33-3b30-7742-bdb3-091f07cf2ea0", - "author": "/root/episode_identity_final_audit", - "recipient": "/root", - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - "create_time": 1787431172.912141, - }, - "content": [{"type": "input_text", "text": "verified inter-agent result"}], - } - full_input: list[proxy_service.JsonValue] = [ - *stored_input_items, - agent_message, - {"role": "user", "content": "continue"}, - ] - payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": full_input} - ) - durable_lookup = proxy_service.DurableBridgeLookup( - session_id="sess-agent-message-proof", - canonical_kind="session_header", - canonical_key="sid-agent-message-proof", - api_key_scope="__anonymous__", - account_id="acc-proof", - owner_instance_id=None, - owner_epoch=3, - lease_expires_at=None, - state=HttpBridgeSessionState.CLOSED, - latest_turn_state="http_turn_agent_message_proof", - latest_response_id="resp-agent-message-proof", - latest_input_item_count=len(stored_input_items), - latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input_items), - latest_pending_tool_calls={}, - model="gpt-5.4", + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-anchor-owner", None), + headers={"x-codex-session-id": "sid-anchor-owner"}, + affinity=proxy_service._AffinityPolicy( + key="sid-anchor-owner", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id=account_id, status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + codex_session=True, + last_completed_response_id="resp_session_latest", + last_completed_response_account_id=anchor_owner_account_id, + last_completed_input_count=3, + last_completed_input_prefix_fingerprint=proxy_service._fingerprint_input_items(prefix_items), ) - assert http_bridge_streaming_module._verify_durable_full_resend(payload, durable_lookup) is not None - assert ( - http_bridge_streaming_module._verify_durable_full_resend( - payload, - replace(durable_lookup, latest_pending_tool_calls=None), - ) - is None - ) - assert ( - http_bridge_streaming_module._verify_durable_full_resend( - payload, - replace(durable_lookup, latest_pending_tool_calls={"call-pending": "function_call"}), - ) - is None + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_enabled=True, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), ) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) + monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) - session = _make_bridge_session(key_value="store-agent-message-proof") - session.last_completed_response_id = "resp-store-agent-message-proof" - session.last_completed_response_account_id = session.account.id - session.last_completed_input_count = len(stored_input_items) - session.last_completed_input_prefix_fingerprint = proxy_service._fingerprint_input_items(stored_input_items) - session.last_pending_tool_calls = {} - proof = http_bridge_streaming_module._verify_store_context_full_resend(payload, session) - assert proof is not None + async for _chunk in service._stream_via_http_bridge( + payload, + headers={"x-codex-session-id": "sid-anchor-owner"}, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ): + pass + return prepared_payloads - session.last_pending_tool_call_manifest_invalid = True - assert proof.matches(payload, session) is False - assert http_bridge_streaming_module._verify_store_context_full_resend(payload, session) is None - session.last_pending_tool_call_manifest_invalid = False - session.last_pending_tool_calls = {"call-pending": "function_call"} - assert http_bridge_streaming_module._verify_store_context_full_resend(payload, session) is None +@pytest.mark.asyncio +async def test_stream_via_http_bridge_injects_session_anchor_when_account_owns_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Serving account owns the anchor -> the compact anchor is injected as normal. + prepared = await _run_session_anchor_owner_stream(monkeypatch, account_id="acc-1", anchor_owner_account_id="acc-1") + # Injection re-prepares the payload, so the final (sent) request carries the anchor. + assert prepared[-1].previous_response_id == "resp_session_latest" -def test_verified_abandoned_pending_agent_boundary_is_stale_anchor_only_and_state_bound() -> None: - stored_input_items: list[proxy_service.JsonValue] = [ - {"role": "user", "content": "first question"}, - { - "type": "agent_message", - "id": "amsg_01a02b33-3b30-7742-bdb3-091f07cf2ea1", - "author": "/root/historical_worker", - "recipient": "/root", - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2daa", - "create_time": 1787431100.0, - }, - "content": [{"type": "input_text", "text": "historical inter-agent result"}], - }, +@pytest.mark.asyncio +async def test_stream_via_http_bridge_skips_session_anchor_after_cross_account_failover( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Anchor was created on acc-1 but the session now serves on acc-2 (failover). + # A previous_response_id is account-scoped upstream, so injecting it here would + # send an unresolvable anchor with the history trimmed away -> upstream never + # emits response.created -> the response-create gate wedges. It must be skipped + # and the full history resent instead. + prepared = await _run_session_anchor_owner_stream(monkeypatch, account_id="acc-2", anchor_owner_account_id="acc-1") + assert all(payload.previous_response_id != "resp_session_latest" for payload in prepared) + assert prepared[-1].input == [ + {"role": "user", "content": [{"type": "input_text", "text": "a"}]}, + {"role": "assistant", "content": [{"type": "output_text", "text": "b"}]}, + {"role": "user", "content": [{"type": "input_text", "text": "c"}]}, + {"role": "user", "content": [{"type": "input_text", "text": "d"}]}, ] - full_input: list[proxy_service.JsonValue] = [ - *stored_input_items, - { - "type": "reasoning", - "id": "rs_08639659bba14680016a8a0904462c87d08ae1115f2aef2ccc", - "encrypted_content": "opaque", - "summary": [], - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - }, - }, - { - "type": "agent_message", - "id": "amsg_01a02b33-3b30-7742-bdb3-091f07cf2ea0", - "author": "/root/episode_identity_final_audit", - "recipient": "/root", - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - "create_time": 1787431172.912141, - }, - "content": [{"type": "input_text", "text": "verified inter-agent result"}], - }, + + +@pytest.mark.asyncio +async def test_stream_via_http_bridge_does_not_inject_durable_previous_response_anchor_for_full_resend_payload( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + payload = proxy_service.ResponsesRequest.model_validate( { - "type": "message", - "id": "msg_01a02c43-4980-7afb-97f5-2e2d30aa73de", - "role": "user", - "content": [{"type": "input_text", "text": "continue"}], - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02c43-4980-7afb-97f5-2e2d30aa73de", - "create_time": 1787433402.605, - }, + "model": "gpt-5.4", + "instructions": "hi", + "input": [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "world"}, + {"role": "user", "content": "follow up"}, + ], }, - ] - payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": full_input} ) - durable_lookup = proxy_service.DurableBridgeLookup( - session_id="sess-abandoned-pending-proof", - canonical_kind="session_header", - canonical_key="sid-abandoned-pending-proof", - api_key_scope="__anonymous__", - account_id="acc-proof", - owner_instance_id=None, - owner_epoch=3, - lease_expires_at=None, - state=HttpBridgeSessionState.CLOSED, - latest_turn_state="http_turn_abandoned_pending_proof", - latest_response_id="resp-abandoned-pending-proof", - latest_input_item_count=len(stored_input_items), - latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input_items), - latest_pending_tool_calls={"call-undelivered": "custom_tool_call"}, + request_state = proxy_service._WebSocketRequestState( + request_id="req-full-resend", model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + event_queue=asyncio.Queue(), + transport="http", ) + event_queue = request_state.event_queue + assert event_queue is not None + await event_queue.put(None) + captured: dict[str, object] = {} + prepared_input_lengths: list[int] = [] - # This proof is intentionally excluded from ordinary fresh replay. - assert http_bridge_streaming_module._verify_durable_full_resend(payload, durable_lookup) is None - proof_with_reason, rejection_reason = ( - http_bridge_streaming_module._verify_durable_abandoned_pending_full_resend_with_reason( - payload, - durable_lookup, - ) + def fake_prepare( + prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del api_key, api_key_reservation, request_id, client_ip + captured["previous_response_id"] = prepared_payload.previous_response_id + inp = prepared_payload.input + prepared_input_lengths.append(len(inp) if isinstance(inp, list) else 1) + return request_state, '{"type":"response.create"}' + + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), + headers={"x-codex-session-id": "sid-123"}, + affinity=proxy_service._AffinityPolicy( + key="sid-123", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, ) - assert proof_with_reason is not None - assert rejection_reason is None - proof = http_bridge_streaming_module._verify_durable_abandoned_pending_full_resend( - payload, - durable_lookup, + + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), ) - assert proof is not None - assert proof.matches(payload, durable_lookup) - assert not proof.matches( - payload, - replace(durable_lookup, latest_pending_tool_calls={"call-other": "custom_tool_call"}), + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + service._durable_bridge, + "lookup_request_targets", + AsyncMock( + return_value=proxy_service.DurableBridgeLookup( + session_id="sess-1", + canonical_kind="session_header", + canonical_key="sid-123", + api_key_scope="__anonymous__", + account_id="acc-1", + owner_instance_id="instance-a", + owner_epoch=1, + lease_expires_at=datetime.now(timezone.utc), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_1", + latest_response_id="resp_latest", + ) + ), ) - assert ( - http_bridge_streaming_module._verify_durable_abandoned_pending_full_resend( - payload.model_copy(update={"input": [*stored_input_items, full_input[-1]]}), - durable_lookup, + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) + monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"x-codex-session-id": "sid-123"}, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, ) - is None - ) + ] + + assert chunks == [] + assert captured["previous_response_id"] is None + # Full-resend payloads are explicitly excluded from durable anchor + # injection, so the bridge prepares the original request exactly once. + assert prepared_input_lengths == [3] + # This path never reaches the trim branch, so the fake request_state + # returned by fake_prepare keeps its default metadata. + assert request_state.input_full_fingerprint is None -def test_abandoned_pending_full_resend_diagnostic_reports_reason_without_content( - caplog: pytest.LogCaptureFixture, +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("suffix_items", "pending_tool_calls", "preserves_full_resend", "forwardable_owner"), + [ + pytest.param( + [ + {"role": "assistant", "content": "hello back"}, + {"role": "user", "content": "follow up"}, + ], + None, + True, + False, + id="retained-assistant-output", + ), + pytest.param( + [ + { + "type": "message", + "role": "assistant", + "phase": "final_answer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-previous"}, + "content": [{"type": "output_text", "text": "hello back"}], + }, + { + "type": "message", + "role": "user", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "follow up"}], + }, + { + "type": "message", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "new control message"}], + }, + ], + None, + True, + False, + id="retained-assistant-output-with-fresh-developer-followup", + ), + pytest.param( + [ + { + "type": "function_call", + "call_id": "call-1", + "name": "lookup", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "call-1", + "output": "result", + }, + ], + {"call-1": "function_call"}, + True, + False, + id="self-contained-tool-loop", + ), + pytest.param( + [ + { + "type": "custom_tool_call", + "call_id": "call-1", + "name": "shell", + "input": "pwd", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + }, + { + "type": "message", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "new control message"}], + }, + { + "type": "custom_tool_call_output", + "call_id": "call-1", + "output": "/workspace", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + }, + ], + {"call-1": "custom_tool_call"}, + True, + False, + id="self-contained-tool-loop-with-fresh-developer-interleave", + ), + pytest.param( + [ + { + "type": "function_call", + "call_id": "call-1", + "name": "lookup", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "call-1", + "output": "result", + }, + ], + None, + False, + False, + id="tool-loop-with-unknown-manifest", + ), + pytest.param( + [{"role": "user", "content": "revise that answer"}], + None, + False, + False, + id="missing-prior-output", + ), + pytest.param( + [{"role": "user", "content": "revise that answer"}], + None, + False, + True, + id="owner-forward-race-missing-prior-output", + ), + ], +) +async def test_stream_via_http_bridge_preserves_only_safe_trimmable_full_resend_on_fresh_bridge( + monkeypatch: pytest.MonkeyPatch, + suffix_items: list[proxy_service.JsonValue], + pending_tool_calls: dict[str, str] | None, + preserves_full_resend: bool, + forwardable_owner: bool, ) -> None: - secret_marker = "never-log-this-message-body" + service = proxy_service.ProxyService(cast(Any, nullcontext())) stored_input_items: list[proxy_service.JsonValue] = [ - {"role": "user", "content": secret_marker}, + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "custom", "name": "shell"}], + }, + {"role": "user", "content": "hello"}, ] + input_items = [*stored_input_items, *suffix_items] payload = proxy_service.ResponsesRequest.model_validate( { "model": "gpt-5.4", - "input": [ - *stored_input_items, - { - "type": "agent_message", - "id": "amsg_01a02b33-3b30-7742-bdb3-091f07cf2ea0", - "author": "/root/fixture_worker", - "recipient": "/root", - "content": [{"type": "input_text", "text": secret_marker}], - }, - { - "type": "message", - "id": "msg_01a02b33-3b30-7742-bdb3-091f07cf2ea2", - "role": "developer", - "content": [{"type": "input_text", "text": secret_marker}], - }, - ], - } + "instructions": "hi", + "input": input_items, + "reasoning": { + "context": "last_turn", + "effort": "high", + "summary": "auto", + "vendor_hint": 7, + }, + }, ) - durable_lookup = proxy_service.DurableBridgeLookup( - session_id="sess-diagnostic", - canonical_kind="session_header", - canonical_key="sid-diagnostic", - api_key_scope="__anonymous__", - account_id="acc-proof", - owner_instance_id=None, - owner_epoch=3, - lease_expires_at=None, - state=HttpBridgeSessionState.CLOSED, - latest_turn_state="http_turn_diagnostic", - latest_response_id="resp-diagnostic", - latest_input_item_count=1, - latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input_items), - latest_pending_tool_calls={"call-undelivered": "custom_tool_call"}, + request_state = proxy_service._WebSocketRequestState( + request_id="req-full-resend-trim", model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + event_queue=asyncio.Queue(), + transport="http", ) + event_queue = request_state.event_queue + assert event_queue is not None + await event_queue.put(None) + prepared_previous_response_ids: list[str | None] = [] + prepared_input_lengths: list[int] = [] + prepared_frames: list[dict[str, Any]] = [] + prepare_call_count = 0 + real_prepare = service._prepare_http_bridge_request - proof, reason_code = http_bridge_streaming_module._verify_durable_abandoned_pending_full_resend_with_reason( - payload, - durable_lookup, + def fake_prepare( + prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + **kwargs: Any, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + # The recovery journal fingerprint is prepared from the same payload + # before the request is sent. It is internal bookkeeping, not a + # second upstream dispatch, so keep it out of dispatch assertions. + nonlocal prepare_call_count + prepare_call_count += 1 + record_dispatch = not (preserves_full_resend and prepare_call_count == 1) + if record_dispatch: + prepared_previous_response_ids.append(prepared_payload.previous_response_id) + inp = prepared_payload.input + if record_dispatch: + prepared_input_lengths.append(len(inp) if isinstance(inp, list) else 1) + _, text_data = real_prepare( + prepared_payload, + _headers, + api_key=api_key, + api_key_reservation=api_key_reservation, + request_id=request_id, + client_ip=client_ip, + **kwargs, + ) + if record_dispatch: + prepared_frames.append(json.loads(text_data)) + request_state.previous_response_id = prepared_payload.previous_response_id + return request_state, text_data + + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), + headers={"x-codex-session-id": "sid-123"}, + affinity=proxy_service._AffinityPolicy( + key="sid-123", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, ) - assert proof is None - assert reason_code == "followup_missing" - caplog.set_level(logging.INFO, logger="app.modules.proxy.service") - http_bridge_streaming_module._log_abandoned_pending_full_resend_rejection( - bridge_session_key=proxy_service._HTTPBridgeSessionKey( - "session_header", - "sid-diagnostic", - None, + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), ), - payload=payload, - durable_lookup=durable_lookup, - reason_code=reason_code, - stage="unit_test", ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + service._durable_bridge, + "lookup_request_targets", + AsyncMock( + return_value=proxy_service.DurableBridgeLookup( + session_id="sess-1", + canonical_kind="session_header", + canonical_key="sid-123", + api_key_scope="__anonymous__", + account_id="acc-1", + owner_instance_id="instance-b" if forwardable_owner else "instance-a", + owner_epoch=1, + lease_expires_at=( + datetime.now(timezone.utc) + timedelta(seconds=60) + if forwardable_owner + else datetime.now(timezone.utc) + ), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_1", + latest_response_id="resp_latest", + latest_input_item_count=len(stored_input_items), + latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input_items), + latest_pending_tool_calls=pending_tool_calls, + ) + ), + ) + session.codex_session = True + account_neutral_classifier = Mock(return_value=True) + monkeypatch.setattr( + http_bridge_streaming_module, + "_http_bridge_payload_is_account_neutral_fresh_replay", + account_neutral_classifier, + ) + get_or_create = AsyncMock(return_value=session) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr( + service, + "_http_bridge_can_forward_to_active_owner", + AsyncMock(return_value=forwardable_owner), + ) + monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-1")) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"x-codex-session-id": "sid-123"}, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] - rendered = "\n".join(record.getMessage() for record in caplog.records) - assert "reason_code=followup_missing" in rendered - assert "suffix_shape=agent_message" in rendered - assert secret_marker not in rendered + assert chunks == [] + assert prepared_previous_response_ids == ([None] if preserves_full_resend else [None, "resp_latest", "resp_latest"]) + assert prepared_input_lengths == ( + [len(input_items)] if preserves_full_resend else [len(input_items), len(input_items), len(suffix_items)] + ) + assert all("tools" not in frame for frame in prepared_frames) + normalized_input_items = cast(list[proxy_service.JsonValue], payload.input) + expected_input_items = ( + normalized_input_items if preserves_full_resend else normalized_input_items[-len(suffix_items) :] + ) + assert prepared_frames[-1]["input"] == expected_input_items + assert [frame["client_metadata"][CODEX_RESPONSES_LITE_WEBSOCKET_METADATA_KEY] for frame in prepared_frames] == [ + "true", + ] * len(prepared_frames) + assert all( + frame["reasoning"] + == { + "context": "all_turns", + "effort": "high", + "summary": "auto", + "vendor_hint": 7, + } + for frame in prepared_frames + ) + assert cast(dict[str, Any], payload.to_payload()["reasoning"])["context"] == "last_turn" + creation = get_or_create.await_args + assert creation is not None + assert creation.kwargs["previous_response_id"] == ( + None if preserves_full_resend or forwardable_owner else "resp_latest" + ) + assert creation.kwargs["preferred_account_id"] == "acc-1" + assert session.last_completed_response_id == (None if preserves_full_resend else "resp_latest") + assert session.last_completed_response_account_id == (None if preserves_full_resend else "acc-1") + if not preserves_full_resend: + assert request_state.proxy_injected_previous_response_id is True + assert request_state.fresh_upstream_request_is_retry_safe is False + if preserves_full_resend: + account_neutral_classifier.assert_called_once() + else: + account_neutral_classifier.assert_not_called() + create_call = get_or_create.await_args + assert create_call is not None + create_kwargs = create_call.kwargs + create_headers = {key.lower(): value for key, value in create_kwargs["headers"].items()} + create_affinity = cast(proxy_service._AffinityPolicy, create_kwargs["affinity"]) + if preserves_full_resend and not forwardable_owner: + assert "x-codex-session-id" not in create_headers + assert create_affinity.kind == proxy_service.StickySessionKind.CODEX_SESSION + assert create_affinity.key is None + assert create_affinity.codex_session_source is None + assert create_kwargs["session_header_fallback_key"] is None + assert create_kwargs["preferred_account_id"] == "acc-1" + assert create_kwargs["preferred_account_has_continuity_provenance"] is True + else: + assert create_headers["x-codex-session-id"] == "sid-123" + assert create_affinity.key == "sid-123" + assert create_affinity.codex_session_source == "session_header" -def test_verified_store_context_full_resend_proof_is_sealed_and_live_session_bound() -> None: +def test_verified_durable_full_resend_proof_is_sealed_immutable_and_request_bound() -> None: stored_input_items: list[proxy_service.JsonValue] = [ + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "custom", "name": "shell"}], + }, {"role": "user", "content": "hello"}, ] - full_input: list[proxy_service.JsonValue] = [ + full_input = [ *stored_input_items, {"role": "assistant", "content": "hello back"}, {"role": "user", "content": "follow up"}, @@ -10343,30 +12138,38 @@ def test_verified_store_context_full_resend_proof_is_sealed_and_live_session_bou "input": full_input, } ) - session = _make_bridge_session(key_value="store-context-proof") - session.last_completed_response_id = "resp-store-context" - session.last_completed_response_account_id = session.account.id - session.last_completed_input_count = len(stored_input_items) - session.last_completed_input_prefix_fingerprint = proxy_service._fingerprint_input_items(stored_input_items) + durable_lookup = proxy_service.DurableBridgeLookup( + session_id="sess-proof", + canonical_kind="session_header", + canonical_key="sid-proof", + api_key_scope="__anonymous__", + account_id="acc-proof", + owner_instance_id=None, + owner_epoch=3, + lease_expires_at=None, + state=HttpBridgeSessionState.CLOSED, + latest_turn_state="http_turn_proof", + latest_response_id="resp-proof", + latest_input_item_count=len(stored_input_items), + latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input_items), + model="gpt-5.4", + ) with pytest.raises(TypeError, match="created only by the verifier"): - http_bridge_streaming_module._VerifiedStoreContextFullResend( + http_bridge_streaming_module._VerifiedDurableFullResend( _token=object(), - affinity_kind=session.key.affinity_kind, - affinity_key=session.key.affinity_key, - api_key_id=session.key.api_key_id, - owner_account_id=session.account.id, - latest_response_id=cast(str, session.last_completed_response_id), + durable_session_id=durable_lookup.session_id, + owner_account_id=cast(str, durable_lookup.account_id), + latest_response_id=cast(str, durable_lookup.latest_response_id), stored_input_item_count=len(stored_input_items), - stored_input_fingerprint=session.last_completed_input_prefix_fingerprint, + stored_input_fingerprint=cast(str, durable_lookup.latest_input_full_fingerprint), full_input_fingerprint=proxy_service._fingerprint_input_items(full_input), pending_tool_calls=None, ) - proof = http_bridge_streaming_module._verify_store_context_full_resend(payload, session) - + proof = http_bridge_streaming_module._verify_durable_full_resend(payload, durable_lookup) assert proof is not None - assert proof.matches(payload, session) is True + assert proof.matches(payload, durable_lookup) is True assert copy.copy(proof) is proof assert copy.deepcopy(proof) is proof with pytest.raises(AttributeError, match="immutable"): @@ -10374,11 +12177,6 @@ def test_verified_store_context_full_resend_proof_is_sealed_and_live_session_bou with pytest.raises(TypeError, match="cannot be serialized"): pickle.dumps(proof) - incomplete_payload = payload.model_copy( - update={"input": [*stored_input_items, {"role": "user", "content": "follow up"}]} - ) - assert http_bridge_streaming_module._verify_store_context_full_resend(incomplete_payload, session) is None - changed_payload = payload.model_copy( update={ "input": [ @@ -10388,10 +12186,84 @@ def test_verified_store_context_full_resend_proof_is_sealed_and_live_session_bou ] } ) - assert proof.matches(changed_payload, session) is False + assert proof.matches(changed_payload, durable_lookup) is False + substituted_durable_lookups = ( + replace(durable_lookup, session_id="sess-other"), + replace(durable_lookup, account_id="acc-other"), + replace(durable_lookup, latest_response_id="resp-other"), + replace(durable_lookup, latest_input_item_count=len(stored_input_items) + 1), + replace(durable_lookup, latest_input_full_fingerprint="fingerprint-other"), + replace(durable_lookup, latest_pending_tool_calls={"call-other": "function_call"}), + ) + assert all(proof.matches(payload, lookup) is False for lookup in substituted_durable_lookups) + + incomplete_payload = payload.model_copy( + update={"input": [*stored_input_items, {"role": "user", "content": "follow up"}]} + ) + assert http_bridge_streaming_module._verify_durable_full_resend(incomplete_payload, durable_lookup) is None + + +def test_verified_durable_full_resend_accepts_response_bound_pending_tool_calls() -> None: + stored_input_items: list[proxy_service.JsonValue] = [ + {"role": "user", "content": "look that up"}, + ] + full_input: list[proxy_service.JsonValue] = [ + *stored_input_items, + { + "type": "function_call", + "call_id": "call-1", + "name": "lookup", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "call-1", + "output": "result", + }, + ] + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": full_input, + } + ) + durable_lookup = proxy_service.DurableBridgeLookup( + session_id="sess-tool-proof", + canonical_kind="session_header", + canonical_key="sid-tool-proof", + api_key_scope="__anonymous__", + account_id="acc-proof", + owner_instance_id=None, + owner_epoch=3, + lease_expires_at=None, + state=HttpBridgeSessionState.CLOSED, + latest_turn_state="http_turn_tool_proof", + latest_response_id="resp-tool-proof", + latest_input_item_count=len(stored_input_items), + latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input_items), + model="gpt-5.4", + latest_pending_tool_calls={"call-1": "function_call"}, + ) + + proof = http_bridge_streaming_module._verify_durable_full_resend(payload, durable_lookup) - session.last_completed_response_id = "resp-replaced" - assert proof.matches(payload, session) is False + assert proof is not None + assert proof.matches(payload, durable_lookup) is True + assert ( + proof.matches( + payload, + replace(durable_lookup, latest_pending_tool_calls={"call-other": "function_call"}), + ) + is False + ) + assert ( + http_bridge_streaming_module._verify_durable_full_resend( + payload, + replace(durable_lookup, latest_pending_tool_calls=None), + ) + is None + ) @pytest.mark.asyncio @@ -10793,6 +12665,46 @@ async def fake_get_or_create( assert captured["preferred_account_has_continuity_provenance"] is True +@pytest.mark.asyncio +async def test_local_terminal_reset_tracks_detached_generation_during_pending_settlement( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="local-terminal-reset-detached") + settlement_started = asyncio.Event() + release_settlement = asyncio.Event() + + async def slow_pending_settlement(*args: object, **kwargs: object) -> bool: + del args, kwargs + settlement_started.set() + await release_settlement.wait() + return True + + service._http_bridge_sessions[session.key] = session + monkeypatch.setattr(service, "_fail_pending_websocket_requests", slow_pending_settlement) + monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) + + reset_task = asyncio.create_task( + service._reset_http_bridge_session_after_local_terminal_error( + session, + error_code="stream_incomplete", + error_message="local recovery failed", + ) + ) + try: + await asyncio.wait_for(settlement_started.wait(), timeout=1.0) + assert service._http_bridge_sessions == {} + assert service._http_bridge_detached_sessions[id(session)] is session + assert http_bridge_helpers_module._http_bridge_capacity_generation_count(service) == 1 + assert session.resource_close_task is None + finally: + release_settlement.set() + + await asyncio.wait_for(reset_task, timeout=1.0) + + assert service._http_bridge_detached_sessions == {} + + @pytest.mark.asyncio async def test_close_http_bridge_session_fails_pending_downstream_requests() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) @@ -10827,6 +12739,8 @@ async def test_close_http_bridge_session_fails_pending_downstream_requests() -> idle_ttl_seconds=120.0, ) + service._http_bridge_detached_sessions[id(session)] = session + await service._close_http_bridge_session(session) failed_event = await asyncio.wait_for(event_queue.get(), timeout=1.0) @@ -10836,6 +12750,40 @@ async def test_close_http_bridge_session_fails_pending_downstream_requests() -> assert await asyncio.wait_for(event_queue.get(), timeout=1.0) is None assert list(session.pending_requests) == [] assert session.queued_request_count == 0 + assert service._http_bridge_detached_sessions == {} + + +@pytest.mark.asyncio +async def test_close_http_bridge_session_uses_one_resource_owner_for_concurrent_callers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="close-single-flight") + close_started = asyncio.Event() + release_close = asyncio.Event() + upstream_close_calls = 0 + + async def close_upstream() -> None: + nonlocal upstream_close_calls + upstream_close_calls += 1 + close_started.set() + await release_close.wait() + + session.upstream = cast(UpstreamWebSocket, SimpleNamespace(close=close_upstream)) + service._http_bridge_detached_sessions[id(session)] = session + release_account_lease = AsyncMock() + monkeypatch.setattr(service._load_balancer, "release_account_lease", release_account_lease) + + first_caller = asyncio.create_task(service._close_http_bridge_session(session)) + await asyncio.wait_for(close_started.wait(), timeout=1.0) + second_caller = asyncio.create_task(service._close_http_bridge_session(session)) + await asyncio.sleep(0) + release_close.set() + await asyncio.gather(first_caller, second_caller) + + assert upstream_close_calls == 1 + release_account_lease.assert_awaited_once() + assert service._http_bridge_detached_sessions == {} @pytest.mark.asyncio @@ -10867,6 +12815,9 @@ async def release_account_lease(account_lease: proxy_service.AccountLease | None lock_holder = asyncio.create_task(hold_pending_lock()) await asyncio.wait_for(lock_acquired.wait(), timeout=1.0) monkeypatch.setattr(service._load_balancer, "release_account_lease", release_account_lease) + upstream_close = AsyncMock() + session.upstream = cast(UpstreamWebSocket, SimpleNamespace(close=upstream_close)) + service._http_bridge_detached_sessions[id(session)] = session close_task = asyncio.create_task(service._close_http_bridge_session(session)) try: @@ -10874,8 +12825,14 @@ async def release_account_lease(account_lease: proxy_service.AccountLease | None assert session.account_lease is None assert not close_task.done() close_task.cancel() + await asyncio.sleep(0) + assert not close_task.done() + assert service._http_bridge_detached_sessions[id(session)] is session + release_lock.set() with pytest.raises(asyncio.CancelledError): - await close_task + await asyncio.wait_for(close_task, timeout=1.0) + upstream_close.assert_awaited_once() + assert service._http_bridge_detached_sessions == {} finally: release_lock.set() await asyncio.wait_for(lock_holder, timeout=1.0) @@ -11193,6 +13150,35 @@ def test_http_bridge_drain_detach_removes_old_previous_response_alias() -> None: assert old_session.previous_response_ids == set() +@pytest.mark.asyncio +async def test_detached_predecessor_cannot_publish_turn_state_to_replacement() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("session_header", "detached-turn-state", None) + predecessor = _make_bridge_session(key=key, key_value=key.affinity_key) + replacement = _make_bridge_session(key=key, key_value=key.affinity_key) + predecessor.upstream_control.reconnect_requested = True + predecessor.upstream_control.retire_after_drain = True + service._http_bridge_sessions[key] = predecessor + + async with service._http_bridge_lock: + assert ( + service._detach_http_bridge_session_locked( + key, + expected_session=predecessor, + mark_closed=False, + ) + is predecessor + ) + service._http_bridge_sessions[key] = replacement + + turn_state = "http_turn_detached_predecessor" + assert await service._register_http_bridge_turn_state(predecessor, turn_state) is False + assert predecessor.downstream_turn_state_aliases == set() + assert proxy_service._http_bridge_turn_state_alias_key(turn_state, None) not in ( + service._http_bridge_turn_state_index + ) + + @pytest.mark.asyncio async def test_stream_via_http_bridge_does_not_inject_durable_anchor_for_live_turn_state_session( monkeypatch: pytest.MonkeyPatch, @@ -11844,7 +13830,6 @@ async def fake_stream_http_bridge_session_events(*args: object, **kwargs: object latest_response_id="resp_latest", latest_input_item_count=len(prefix_items), latest_input_full_fingerprint=proxy_service._fingerprint_input_items(payload_prefix_items), - latest_pending_tool_calls={}, ) takeover_lookup = replace( durable_lookup, @@ -13654,8 +15639,6 @@ async def _run_owner_forward_recovery_with_session( input_items: list[dict[str, Any]], capacity_error_on_first_submit: bool = False, submit_attempts: list[str] | None = None, - rewritten_file_account_id: str | None = None, - submitted_states: list[proxy_service._WebSocketRequestState] | None = None, ) -> list[Any]: """Drive owner-forward failure -> local recovery; return prepared inputs. @@ -13722,8 +15705,6 @@ async def fake_submit_http_bridge_request( del _session, text_data, queue_limit if submit_attempts is not None: submit_attempts.append(request_state.request_id) - if submitted_states is not None: - submitted_states.append(request_state) if capacity_error_on_first_submit and submit_attempts is not None and len(submit_attempts) == 1: raise ProxyResponseError( 429, @@ -13781,7 +15762,6 @@ async def fake_submit_http_bridge_request( codex_idle_ttl_seconds=900.0, max_sessions=8, queue_limit=4, - rewritten_file_account_id=rewritten_file_account_id, ) ] if capacity_error_on_first_submit: @@ -13834,25 +15814,6 @@ async def test_stream_via_http_bridge_owner_forward_recovery_waits_for_local_sub assert submit_attempts == ["req-2", "req-2"] -@pytest.mark.asyncio -async def test_stream_via_http_bridge_owner_forward_recovery_preserves_uploaded_file_account_pin( - monkeypatch: pytest.MonkeyPatch, -) -> None: - submitted_states: list[proxy_service._WebSocketRequestState] = [] - - await _run_owner_forward_recovery_with_session( - monkeypatch, - recovery_session=_make_owner_forward_recovery_session(), - input_items=[{"role": "user", "content": "continue with uploaded file"}], - rewritten_file_account_id="acc-1", - submitted_states=submitted_states, - ) - - assert len(submitted_states) == 1 - assert submitted_states[0].preferred_account_id == "acc-1" - assert submitted_states[0].file_required_preferred_account is True - - @pytest.mark.asyncio async def test_stream_via_http_bridge_owner_forward_recovery_injects_outputs_from_local_session_state( monkeypatch: pytest.MonkeyPatch, @@ -13907,7 +15868,6 @@ async def test_stream_via_http_bridge_local_recovery_retry_keeps_injected_interr retry_session = _make_owner_forward_recovery_session() prepared_inputs: list[Any] = [] - submitted_states: list[proxy_service._WebSocketRequestState] = [] def fake_prepare( prepared_payload: proxy_service.ResponsesRequest, @@ -13946,7 +15906,6 @@ async def fake_submit_http_bridge_request( nonlocal submit_calls del _session, text_data, queue_limit submit_calls += 1 - submitted_states.append(request_state) if submit_calls == 1: raise ProxyResponseError(400, proxy_service.openai_error("previous_response_not_found", "missing")) event_queue = request_state.event_queue @@ -13997,7 +15956,6 @@ async def fake_submit_http_bridge_request( codex_idle_ttl_seconds=900.0, max_sessions=8, queue_limit=4, - rewritten_file_account_id="acc-1", ) ] @@ -14015,10 +15973,6 @@ async def fake_submit_http_bridge_request( assert prepared_inputs[0] == input_items assert prepared_inputs[1] == [synthetic_item, *input_items] assert prepared_inputs[2] == [synthetic_item, *input_items] - assert len(submitted_states) == 2 - assert submitted_states[0].file_required_preferred_account is True - assert submitted_states[1].file_required_preferred_account is True - assert submitted_states[1].preferred_account_id == "acc-1" @pytest.mark.asyncio @@ -15203,36 +17157,159 @@ async def fake_create_http_bridge_session( return created_session monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) - monkeypatch.setattr(service, "_create_http_bridge_session", fake_create_http_bridge_session) + monkeypatch.setattr(service, "_create_http_bridge_session", fake_create_http_bridge_session) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a", "instance-b"])), + ) + + resolved = await service._get_or_create_http_bridge_session( + requested_key, + headers={ + "x-codex-turn-state": "http_turn_generated", + "x-codex-session-id": "sid-123", + }, + affinity=proxy_service._AffinityPolicy( + key="sid-123", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + previous_response_id="resp_prev_1", + session_header_fallback_key=fallback_key, + ) + + assert resolved is created_session + assert captured["key"] == fallback_key + + +@pytest.mark.asyncio +async def test_missing_turn_alias_with_thread_uses_thread_canonical_not_legacy_process_lane( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + headers = { + "session-id": "process-shared", + "thread-id": "thread-current", + "x-codex-turn-state": "http_turn_missing_thread_alias", + } + requested_key = proxy_service._HTTPBridgeSessionKey( + "turn_state_header", + "http_turn_missing_thread_alias", + None, + ) + thread_key_value = proxy_affinity._codex_backend_identity(headers).thread_selection_key + assert thread_key_value is not None + thread_key = proxy_service._HTTPBridgeSessionKey("thread_header", thread_key_value, None) + legacy_process_key = proxy_service._HTTPBridgeSessionKey("session_header", "process-shared", None) + service._http_bridge_sessions[legacy_process_key] = _make_bridge_session( + key=legacy_process_key, + key_value="legacy-sibling", + ) + captured: dict[str, object] = {} + + async def create_session(key: proxy_service._HTTPBridgeSessionKey, **kwargs: object): + del kwargs + captured["key"] = key + return _make_bridge_session(key=key, key_value="current-thread") + + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(service, "_create_http_bridge_session", create_session) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a"])), + ) + + resolved = await service._get_or_create_http_bridge_session( + requested_key, + headers=headers, + affinity=proxy_service._AffinityPolicy( + key=thread_key_value, + kind=proxy_service.StickySessionKind.PROMPT_CACHE, + codex_session_source="thread_header", + ), + api_key=None, + request_model="gpt-5.6-sol", + idle_ttl_seconds=120.0, + max_sessions=8, + previous_response_id="resp-missing-thread-alias", + ) + + assert resolved.key == thread_key + assert captured["key"] == thread_key + assert resolved is not service._http_bridge_sessions[legacy_process_key] + + +@pytest.mark.asyncio +async def test_legacy_forward_missing_turn_alias_with_thread_avoids_process_lane( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + headers = { + "session-id": "process-shared", + "thread-id": "thread-forwarded", + "x-codex-turn-state": "http_turn_missing_forwarded_alias", + } + requested_key = proxy_service._HTTPBridgeSessionKey( + "turn_state_header", + "http_turn_missing_forwarded_alias", + None, + ) + thread_key_value = proxy_affinity._codex_backend_identity(headers).thread_selection_key + assert thread_key_value is not None + thread_key = proxy_service._HTTPBridgeSessionKey("thread_header", thread_key_value, None) + legacy_process_key = proxy_service._HTTPBridgeSessionKey("session_header", "process-shared", None) + service._http_bridge_sessions[legacy_process_key] = _make_bridge_session( + key=legacy_process_key, + key_value="legacy-sibling", + ) + captured: dict[str, object] = {} + + async def create_session(key: proxy_service._HTTPBridgeSessionKey, **kwargs: object): + del kwargs + captured["key"] = key + return _make_bridge_session(key=key, key_value="forwarded-thread") + + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(service, "_create_http_bridge_session", create_session) monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) monkeypatch.setattr( proxy_service, "_active_http_bridge_instance_ring", - AsyncMock(return_value=("instance-a", ["instance-a", "instance-b"])), + AsyncMock(return_value=("instance-a", ["instance-a"])), ) resolved = await service._get_or_create_http_bridge_session( requested_key, - headers={ - "x-codex-turn-state": "http_turn_generated", - "x-codex-session-id": "sid-123", - }, + headers=headers, affinity=proxy_service._AffinityPolicy( - key="sid-123", - kind=proxy_service.StickySessionKind.CODEX_SESSION, + key=thread_key_value, + kind=proxy_service.StickySessionKind.PROMPT_CACHE, + codex_session_source="thread_header", ), api_key=None, - request_model="gpt-5.4", + request_model="gpt-5.6-sol", idle_ttl_seconds=120.0, max_sessions=8, - previous_response_id="resp_prev_1", - session_header_fallback_key=fallback_key, + previous_response_id="resp-missing-forwarded-alias", + forwarded_request=True, ) - assert resolved is created_session - assert captured["key"] == fallback_key + assert resolved.key == thread_key + assert captured["key"] == thread_key + assert resolved is not service._http_bridge_sessions[legacy_process_key] @pytest.mark.asyncio @@ -15411,8 +17488,11 @@ async def fake_create_http_bridge_session(create_key, **_kwargs): monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) monkeypatch.setattr(service, "_create_http_bridge_session", fake_create_http_bridge_session) - monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + claim_durable = AsyncMock() + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", claim_durable) + settings = _make_app_settings() + settings.http_responses_session_bridge_instance_id = "instance-a" + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) monkeypatch.setattr( proxy_service, @@ -15458,6 +17538,11 @@ async def fake_create_http_bridge_session(create_key, **_kwargs): assert parent.closed is False assert service._http_bridge_sessions[parent_key] is parent assert service._http_bridge_previous_response_index[previous_alias] == parent_key + claim_durable.assert_awaited_once_with( + child, + allow_takeover=True, + force_owner_epoch_advance=True, + ) @pytest.mark.asyncio @@ -15653,6 +17738,7 @@ async def fake_create_http_bridge_session( async def track_close(session: proxy_service._HTTPBridgeSession, *, reason: str) -> None: assert reason == "registry_detach" closed_sessions.append(session) + service._http_bridge_detached_sessions.pop(id(session), None) monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) monkeypatch.setattr(service, "_create_http_bridge_session", fake_create_http_bridge_session) @@ -15719,6 +17805,7 @@ async def fake_create_http_bridge_session( async def track_close(session: proxy_service._HTTPBridgeSession, *, reason: str) -> None: assert reason == "registry_detach" closed_sessions.append(session) + service._http_bridge_detached_sessions.pop(id(session), None) monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) monkeypatch.setattr(service, "_create_http_bridge_session", fake_create_http_bridge_session) @@ -15936,6 +18023,264 @@ async def test_get_or_create_http_bridge_session_drops_stale_previous_response_m assert alias_key not in service._http_bridge_previous_response_index +@pytest.mark.asyncio +async def test_get_or_create_goal_restart_forces_canonical_durable_takeover( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("session_header", "sid-goal-takeover", None) + created_session = _make_bridge_session(key=key, key_value=key.affinity_key) + claim_durable = AsyncMock() + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(service, "_create_http_bridge_session", AsyncMock(return_value=created_session)) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", claim_durable) + monkeypatch.setattr(proxy_service, "_http_bridge_should_wait_for_registration", AsyncMock(return_value=False)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a", "instance-b"])), + ) + + resolved = await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-session-id": key.affinity_key}, + affinity=proxy_service._AffinityPolicy( + key=key.affinity_key, + kind=proxy_service.StickySessionKind.CODEX_SESSION, + abandon_unavailable_legacy_owner=True, + ), + api_key=None, + request_model="gpt-5.6-sol", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + + assert resolved is created_session + claim_durable.assert_awaited_once_with( + created_session, + allow_takeover=True, + force_owner_epoch_advance=True, + ) + + +@pytest.mark.asyncio +async def test_goal_restart_replacement_generations_remain_bounded_by_session_cap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("session_header", "sid-restart-cap", None) + predecessor = _make_bridge_session(key=key, key_value=key.affinity_key, queued_request_count=1) + replacement = _make_bridge_session(key=key, key_value=key.affinity_key) + service._http_bridge_sessions[key] = predecessor + create_session = AsyncMock(return_value=replacement) + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(service, "_create_http_bridge_session", create_session) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) + monkeypatch.setattr(proxy_service, "_http_bridge_should_wait_for_registration", AsyncMock(return_value=False)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a"])), + ) + affinity = proxy_service._AffinityPolicy( + key=key.affinity_key, + kind=proxy_service.StickySessionKind.CODEX_SESSION, + abandon_unavailable_legacy_owner=True, + ) + + first_scope_token = set_request_scope_id("scope-restart-cap-1") + try: + resolved = await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-session-id": key.affinity_key}, + affinity=affinity, + api_key=None, + request_model="gpt-5.6-sol", + idle_ttl_seconds=120.0, + max_sessions=2, + ) + finally: + reset_request_scope_id(first_scope_token) + + assert resolved is replacement + assert service._http_bridge_detached_sessions[id(predecessor)] is predecessor + + second_scope_token = set_request_scope_id("scope-restart-cap-2") + try: + with pytest.raises(ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-session-id": key.affinity_key}, + affinity=affinity, + api_key=None, + request_model="gpt-5.6-sol", + idle_ttl_seconds=120.0, + max_sessions=2, + ) + finally: + reset_request_scope_id(second_scope_token) + + assert exc_info.value.status_code == 429 + assert exc_info.value.payload["error"]["code"] == "capacity_exhausted_active_sessions" + assert create_session.await_count == 1 + assert set(service._http_bridge_detached_sessions) == {id(predecessor), id(replacement)} + + await service.close_all_http_bridge_sessions() + + +@pytest.mark.asyncio +async def test_goal_restart_idle_predecessor_counts_until_its_slow_close_finishes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("session_header", "sid-idle-restart-cap", None) + predecessor = _make_bridge_session(key=key, key_value=key.affinity_key) + replacement = _make_bridge_session(key=key, key_value=key.affinity_key) + service._http_bridge_sessions[key] = predecessor + create_session = AsyncMock(return_value=replacement) + close_started = asyncio.Event() + release_close = asyncio.Event() + bounded_close = service._close_http_bridge_session_bounded + + async def slow_predecessor_close( + session: proxy_service._HTTPBridgeSession, + *, + reason: str, + ) -> None: + if session is predecessor: + close_started.set() + await release_close.wait() + await bounded_close(session, reason=reason) + + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(service, "_create_http_bridge_session", create_session) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", slow_predecessor_close) + monkeypatch.setattr(proxy_service, "_http_bridge_should_wait_for_registration", AsyncMock(return_value=False)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a"])), + ) + affinity = proxy_service._AffinityPolicy( + key=key.affinity_key, + kind=proxy_service.StickySessionKind.CODEX_SESSION, + abandon_unavailable_legacy_owner=True, + ) + + first_scope_token = set_request_scope_id("scope-idle-restart-cap-1") + try: + resolved = await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-session-id": key.affinity_key}, + affinity=affinity, + api_key=None, + request_model="gpt-5.6-sol", + idle_ttl_seconds=120.0, + max_sessions=2, + ) + finally: + reset_request_scope_id(first_scope_token) + + assert resolved is replacement + await asyncio.wait_for(close_started.wait(), timeout=1.0) + assert predecessor.closed is True + assert service._http_bridge_detached_sessions[id(predecessor)] is predecessor + + second_scope_token = set_request_scope_id("scope-idle-restart-cap-2") + try: + with pytest.raises(ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-session-id": key.affinity_key}, + affinity=affinity, + api_key=None, + request_model="gpt-5.6-sol", + idle_ttl_seconds=120.0, + max_sessions=2, + ) + finally: + reset_request_scope_id(second_scope_token) + + assert exc_info.value.status_code == 429 + assert exc_info.value.payload["error"]["code"] == "capacity_exhausted_active_sessions" + assert create_session.await_count == 1 + assert service._http_bridge_detached_sessions[id(predecessor)] is predecessor + assert set(service._http_bridge_detached_sessions) == {id(predecessor), id(replacement)} + + release_close.set() + await service._drain_http_bridge_background_cleanup_tasks(reason="test") + await service.close_all_http_bridge_sessions() + + +@pytest.mark.asyncio +async def test_goal_restart_closes_idle_predecessor_before_enforcing_one_session_cap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("session_header", "sid-restart-cap-one", None) + predecessor = _make_bridge_session(key=key, key_value=key.affinity_key) + replacement = _make_bridge_session(key=key, key_value=key.affinity_key) + service._http_bridge_sessions[key] = predecessor + events: list[str] = [] + + async def close_http_bridge_session_bounded( + session: proxy_service._HTTPBridgeSession, + *, + reason: str, + ) -> None: + assert session is predecessor + assert reason == "registry_detach" + events.append("close") + service._http_bridge_detached_sessions.pop(id(session), None) + + async def create_http_bridge_session( + _: proxy_service._HTTPBridgeSessionKey, + **__: object, + ) -> proxy_service._HTTPBridgeSession: + assert events == ["close"] + events.append("create") + return replacement + + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(service, "_create_http_bridge_session", create_http_bridge_session) + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", close_http_bridge_session_bounded) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) + monkeypatch.setattr(proxy_service, "_http_bridge_should_wait_for_registration", AsyncMock(return_value=False)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a"])), + ) + + resolved = await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-session-id": key.affinity_key}, + affinity=proxy_service._AffinityPolicy( + key=key.affinity_key, + kind=proxy_service.StickySessionKind.CODEX_SESSION, + abandon_unavailable_legacy_owner=True, + ), + api_key=None, + request_model="gpt-5.6-sol", + idle_ttl_seconds=120.0, + max_sessions=1, + ) + + assert resolved is replacement + assert events == ["close", "create"] + assert id(predecessor) not in service._http_bridge_detached_sessions + assert service._http_bridge_sessions[key] is replacement + + @pytest.mark.asyncio async def test_get_or_create_http_bridge_session_allows_local_rebind_for_previous_response_recovery( monkeypatch: pytest.MonkeyPatch, @@ -16174,13 +18519,81 @@ async def test_get_or_create_http_bridge_session_recovers_locally_when_stale_own resolved = await service._get_or_create_http_bridge_session( key, - headers={"x-codex-session-id": "sid-123"}, - affinity=proxy_service._AffinityPolicy(key="sid-123", kind=proxy_service.StickySessionKind.CODEX_SESSION), + headers={"x-codex-session-id": "sid-123"}, + affinity=proxy_service._AffinityPolicy(key="sid-123", kind=proxy_service.StickySessionKind.CODEX_SESSION), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + allow_forward_to_owner=True, + ) + + assert resolved is created_session + claim_durable.assert_awaited_once() + await_args = claim_durable.await_args + assert await_args is not None + assert await_args.kwargs["allow_takeover"] is True + forward_to_owner.assert_not_awaited() + service._ring_membership.resolve_endpoint.assert_awaited_once_with("instance-old") + + +@pytest.mark.asyncio +async def test_get_or_create_http_bridge_session_recovers_locally_when_owner_endpoint_missing_but_replay_anchor_exists( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_123", None) + created_session = proxy_service._HTTPBridgeSession( + key=key, + headers={"x-codex-turn-state": "http_turn_123"}, + affinity=proxy_service._AffinityPolicy(key="http_turn_123"), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=2.0, + idle_ttl_seconds=120.0, + ) + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(service, "_create_http_bridge_session", AsyncMock(return_value=created_session)) + claim_durable = AsyncMock() + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", claim_durable) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-b")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a", "instance-b"])), + ) + service._ring_membership = cast(Any, SimpleNamespace(resolve_endpoint=AsyncMock(return_value=None))) + + resolved = await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-turn-state": "http_turn_123"}, + affinity=proxy_service._AffinityPolicy(key="http_turn_123"), api_key=None, request_model="gpt-5.4", idle_ttl_seconds=120.0, max_sessions=8, + previous_response_id="resp_prev_1", allow_forward_to_owner=True, + durable_lookup=proxy_service.DurableBridgeLookup( + session_id="durable-1", + canonical_kind="turn_state_header", + canonical_key="http_turn_123", + api_key_scope="__anonymous__", + account_id="acc-1", + owner_instance_id="instance-b", + owner_epoch=2, + lease_expires_at=proxy_service.utcnow() + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_123", + latest_response_id="resp_prev_1", + ), ) assert resolved is created_session @@ -16188,12 +18601,10 @@ async def test_get_or_create_http_bridge_session_recovers_locally_when_stale_own await_args = claim_durable.await_args assert await_args is not None assert await_args.kwargs["allow_takeover"] is True - forward_to_owner.assert_not_awaited() - service._ring_membership.resolve_endpoint.assert_awaited_once_with("instance-old") @pytest.mark.asyncio -async def test_get_or_create_http_bridge_session_recovers_locally_when_owner_endpoint_missing_but_replay_anchor_exists( +async def test_get_or_create_http_bridge_session_does_not_force_takeover_live_draining_lease( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) @@ -16245,7 +18656,7 @@ async def test_get_or_create_http_bridge_session_recovers_locally_when_owner_end owner_instance_id="instance-b", owner_epoch=2, lease_expires_at=proxy_service.utcnow() + timedelta(seconds=60), - state=HttpBridgeSessionState.ACTIVE, + state=HttpBridgeSessionState.DRAINING, latest_turn_state="http_turn_123", latest_response_id="resp_prev_1", ), @@ -16255,7 +18666,103 @@ async def test_get_or_create_http_bridge_session_recovers_locally_when_owner_end claim_durable.assert_awaited_once() await_args = claim_durable.await_args assert await_args is not None - assert await_args.kwargs["allow_takeover"] is True + assert await_args.kwargs["allow_takeover"] is False + + +@pytest.mark.asyncio +async def test_get_or_create_does_not_steal_when_active_lookup_becomes_live_draining( + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + session_factory = async_sessionmaker(engine, expire_on_commit=False) + coordinator = DurableBridgeSessionCoordinator(cast(Callable[[], AsyncSession], session_factory)) + stale_lookup = await coordinator.claim_live_session( + session_key_kind="turn_state_header", + session_key_value="http_turn_race", + api_key_id=None, + instance_id="instance-b", + owner_process_epoch="test-process-b", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state="http_turn_race", + latest_response_id="resp_prev_1", + allow_takeover=True, + ) + assert stale_lookup.state == HttpBridgeSessionState.ACTIVE + + service = proxy_service.ProxyService(cast(Any, nullcontext())) + service._durable_bridge = coordinator + key = proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_race", None) + created_session = proxy_service._HTTPBridgeSession( + key=key, + headers={"x-codex-turn-state": "http_turn_race"}, + affinity=proxy_service._AffinityPolicy(key="http_turn_race"), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=2.0, + idle_ttl_seconds=120.0, + ) + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(service, "_create_http_bridge_session", AsyncMock(return_value=created_session)) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings(http_responses_session_bridge_instance_id="instance-a"), + ) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-b")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a", "instance-b"])), + ) + service._ring_membership = cast(Any, SimpleNamespace(resolve_endpoint=AsyncMock(return_value=None))) + + real_claim = coordinator.claim_live_session + + async def claim_after_drain(*args: Any, **kwargs: Any) -> DurableBridgeLookup: + await coordinator.mark_instance_draining(instance_id="instance-b") + return await real_claim(*args, **kwargs) + + monkeypatch.setattr(coordinator, "claim_live_session", claim_after_drain) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-turn-state": "http_turn_race"}, + affinity=proxy_service._AffinityPolicy(key="http_turn_race"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + previous_response_id="resp_prev_1", + allow_forward_to_owner=True, + durable_lookup=stale_lookup, + ) + + assert exc_info.value.status_code == 409 + assert exc_info.value.payload["error"]["code"] == "bridge_instance_mismatch" + after = await coordinator.lookup_request_targets( + session_key_kind="turn_state_header", + session_key_value="http_turn_race", + api_key_id=None, + turn_state="http_turn_race", + session_header=None, + previous_response_id="resp_prev_1", + ) + assert after is not None + assert after.owner_instance_id == "instance-b" + assert after.state == HttpBridgeSessionState.DRAINING + await engine.dispose() @pytest.mark.asyncio @@ -16568,7 +19075,6 @@ async def test_get_or_create_http_bridge_session_waiter_propagates_terminal_infl service._http_bridge_inflight_sessions[key] = inflight_future monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) - monkeypatch.setattr("app.core.startup._bridge_registration_complete", True) monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) monkeypatch.setattr( @@ -16661,6 +19167,118 @@ async def test_close_all_http_bridge_sessions_fails_inflight_waiters() -> None: assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" +@pytest.mark.asyncio +async def test_close_all_http_bridge_sessions_closes_detached_generations( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("session_header", "sid-shutdown-detached", None) + predecessor = _make_bridge_session(key=key, key_value=key.affinity_key, queued_request_count=1) + replacement = _make_bridge_session(key=key, key_value=key.affinity_key) + predecessor_close = AsyncMock() + replacement_close = AsyncMock() + predecessor.upstream = cast(UpstreamWebSocket, SimpleNamespace(close=predecessor_close)) + replacement.upstream = cast(UpstreamWebSocket, SimpleNamespace(close=replacement_close)) + monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) + service._http_bridge_sessions[key] = predecessor + async with service._http_bridge_lock: + assert ( + service._detach_http_bridge_session_locked( + key, + expected_session=predecessor, + mark_closed=False, + ) + is predecessor + ) + service._http_bridge_sessions[key] = replacement + + snapshot = service.http_bridge_activity_snapshot_nowait() + assert snapshot["http_bridge_live_sessions"] == 2 + assert snapshot["http_bridge_pending_or_queued_requests"] == 1 + assert snapshot["http_bridge_restart_blocking"] is True + + await service.close_all_http_bridge_sessions() + + predecessor_close.assert_awaited_once() + replacement_close.assert_awaited_once() + assert predecessor.closed is True + assert replacement.closed is True + assert service._http_bridge_sessions == {} + assert service._http_bridge_detached_sessions == {} + + +@pytest.mark.asyncio +async def test_close_all_http_bridge_sessions_retains_failed_generation_for_retry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="shutdown-close-retry") + service._http_bridge_sessions[session.key] = session + fail_pending = AsyncMock(side_effect=[RuntimeError("pending settlement failed"), None]) + close_event_batcher = AsyncMock() + service._http_bridge_operation_event_batcher = cast( + Any, + SimpleNamespace(close=close_event_batcher), + ) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) + + with pytest.raises(RuntimeError, match="pending settlement failed"): + await service.close_all_http_bridge_sessions() + + close_event_batcher.assert_awaited_once() + assert service._http_bridge_sessions == {} + assert service._http_bridge_detached_sessions[id(session)] is session + + await service.close_all_http_bridge_sessions() + + assert close_event_batcher.await_count == 2 + assert fail_pending.await_count == 2 + assert service._http_bridge_sessions == {} + assert service._http_bridge_detached_sessions == {} + + +@pytest.mark.asyncio +async def test_close_all_http_bridge_sessions_schedules_every_close_before_propagating_cancellation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + first = _make_bridge_session(key_value="shutdown-cancel-first") + second = _make_bridge_session(key_value="shutdown-cancel-second") + first_close_started = asyncio.Event() + second_close_started = asyncio.Event() + release_first_close = asyncio.Event() + + async def close_first() -> None: + first_close_started.set() + await release_first_close.wait() + + async def close_second() -> None: + second_close_started.set() + + first.upstream = cast(UpstreamWebSocket, SimpleNamespace(close=close_first)) + second.upstream = cast(UpstreamWebSocket, SimpleNamespace(close=close_second)) + service._http_bridge_sessions[first.key] = first + service._http_bridge_sessions[second.key] = second + monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) + + shutdown_task = asyncio.create_task(service.close_all_http_bridge_sessions()) + await asyncio.wait_for(first_close_started.wait(), timeout=1.0) + shutdown_task.cancel() + try: + await asyncio.wait_for(second_close_started.wait(), timeout=1.0) + finally: + release_first_close.set() + + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(shutdown_task, timeout=1.0) + + assert first.closed is True + assert second.closed is True + assert service._http_bridge_sessions == {} + assert service._http_bridge_detached_sessions == {} + + @pytest.mark.asyncio async def test_close_all_http_bridge_sessions_fails_capacity_waiters_instead_of_creating_new_session( monkeypatch: pytest.MonkeyPatch, @@ -16846,6 +19464,56 @@ async def test_get_or_create_http_bridge_session_immediate_capacity_exhaustion_i assert exc_info.value.payload["error"]["code"] == "capacity_exhausted_active_sessions" +@pytest.mark.asyncio +async def test_get_or_create_http_bridge_session_closes_planned_lru_before_capacity_rejection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + first_detached = _make_bridge_session(key_value="sid-capacity-detached-1") + second_detached = _make_bridge_session(key_value="sid-capacity-detached-2") + service._http_bridge_detached_sessions[id(first_detached)] = first_detached + service._http_bridge_detached_sessions[id(second_detached)] = second_detached + existing_key = proxy_service._HTTPBridgeSessionKey("session_header", "sid-capacity-evictable", None) + existing = _make_bridge_session(key=existing_key, key_value=existing_key.affinity_key) + service._http_bridge_sessions[existing_key] = existing + new_key = proxy_service._HTTPBridgeSessionKey("session_header", "sid-capacity-rejected", None) + close_http_bridge_session_bounded = AsyncMock() + create_http_bridge_session = AsyncMock() + + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", close_http_bridge_session_bounded) + monkeypatch.setattr(service, "_create_http_bridge_session", create_http_bridge_session) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_http_bridge_should_wait_for_registration", AsyncMock(return_value=False)) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ("instance-a",))), + ) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + new_key, + headers={"x-codex-session-id": new_key.affinity_key}, + affinity=proxy_service._AffinityPolicy( + key=new_key.affinity_key, + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=2, + ) + + assert exc_info.value.status_code == 429 + assert exc_info.value.payload["error"]["code"] == "capacity_exhausted_active_sessions" + close_http_bridge_session_bounded.assert_awaited_once_with(existing, reason="registry_detach") + assert service._http_bridge_detached_sessions[id(existing)] is existing + assert existing_key not in service._http_bridge_sessions + create_http_bridge_session.assert_not_awaited() + + @pytest.mark.asyncio async def test_get_or_create_http_bridge_session_closes_lru_before_replacement_create( monkeypatch: pytest.MonkeyPatch, @@ -16866,6 +19534,7 @@ async def close_http_bridge_session_bounded( assert reason == "registry_detach" events.append("close") session.closed = True + service._http_bridge_detached_sessions.pop(id(session), None) async def create_http_bridge_session( key: proxy_service._HTTPBridgeSessionKey, @@ -16907,6 +19576,68 @@ async def create_http_bridge_session( assert service._http_bridge_sessions[new_key] is result +@pytest.mark.asyncio +async def test_get_or_create_http_bridge_session_rechecks_capacity_after_timed_out_lru_close( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + existing_key = proxy_service._HTTPBridgeSessionKey("session_header", "sid-lru-timeout-existing", None) + existing = _make_bridge_session(key=existing_key, key_value=existing_key.affinity_key) + service._http_bridge_sessions[existing_key] = existing + new_key = proxy_service._HTTPBridgeSessionKey("session_header", "sid-lru-timeout-new", None) + close_started = asyncio.Event() + release_close = asyncio.Event() + + async def slow_close(session: proxy_service._HTTPBridgeSession) -> None: + assert session is existing + close_started.set() + await release_close.wait() + async with service._http_bridge_lock: + service._http_bridge_detached_sessions.pop(id(session), None) + + create_http_bridge_session = AsyncMock() + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(service, "_close_http_bridge_session", slow_close) + monkeypatch.setattr(service, "_create_http_bridge_session", create_http_bridge_session) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) + monkeypatch.setattr(http_bridge_helpers_module, "_HTTP_BRIDGE_BACKGROUND_CLOSE_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_http_bridge_should_wait_for_registration", AsyncMock(return_value=False)) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ("instance-a",))), + ) + + try: + with pytest.raises(ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + new_key, + headers={"x-codex-session-id": new_key.affinity_key}, + affinity=proxy_service._AffinityPolicy( + key=new_key.affinity_key, + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=1, + ) + + await asyncio.wait_for(close_started.wait(), timeout=1.0) + assert exc_info.value.status_code == 429 + assert exc_info.value.payload["error"]["code"] == "capacity_exhausted_active_sessions" + assert service._http_bridge_detached_sessions[id(existing)] is existing + assert new_key not in service._http_bridge_inflight_sessions + create_http_bridge_session.assert_not_awaited() + finally: + release_close.set() + await service._drain_http_bridge_background_cleanup_tasks(reason="test") + + assert service._http_bridge_detached_sessions == {} + + @pytest.mark.asyncio async def test_get_or_create_http_bridge_session_cancel_during_lru_close_cleans_inflight( monkeypatch: pytest.MonkeyPatch, @@ -17237,7 +19968,7 @@ async def get_session() -> proxy_service._HTTPBridgeSession | proxy_service._HTT assert owner_exc_info.value.status_code == 429 assert owner_exc_info.value.payload["error"]["code"] == "capacity_exhausted_active_sessions" assert key not in service._http_bridge_sessions - close_http_bridge_session.assert_awaited_once_with(created) + close_http_bridge_session.assert_awaited_once_with(created, release_durable_session=True) @pytest.mark.asyncio @@ -17349,7 +20080,7 @@ async def test_recovery_session_is_not_published_when_durable_tables_are_missing assert exc_info.value.status_code == 502 assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" assert key not in service._http_bridge_sessions - close_http_bridge_session.assert_awaited_once_with(created_session) + close_http_bridge_session.assert_awaited_once_with(created_session, release_durable_session=True) @pytest.mark.asyncio @@ -18656,33 +21387,83 @@ async def fake_acquire_admission( @pytest.mark.asyncio -async def test_cleanup_http_bridge_submit_interruption_releases_gate_state_when_gate_already_acquired() -> None: +async def test_submit_http_bridge_request_restores_recovery_claim_when_stream_lease_reacquire_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - admission = cast(Any, SimpleNamespace(release=Mock())) - release_lease = AsyncMock() - lease = proxy_service.AccountLease( - lease_id="lease-held", - account_id="acc-bridge", - kind="stream", - acquired_at=1.0, + send_text = AsyncMock() + session = _make_bridge_session(key_value="recovery-lease-reacquire") + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=AsyncMock()), ) - gate = asyncio.Semaphore(0) request_state = proxy_service._WebSocketRequestState( - request_id="req-submit-leak", - model="gpt-5.4", + request_id="req-recovery-lease-reacquire", + model="gpt-5.5", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.5","input":"retry"}', + transport="http", + skip_request_log=True, + ) + request_state.operation_recovery_claimed = True + service._durable_bridge = cast( + Any, + SimpleNamespace(lookup_retry_circuit=AsyncMock(return_value=None)), + ) + cleanup = AsyncMock() + monkeypatch.setattr(service, "_cleanup_http_bridge_submit_interruption", cleanup) + lease_failure = proxy_service.ProxyResponseError( + 429, + openai_error("account_stream_cap", "stream capacity exhausted"), + ) + monkeypatch.setattr( + service, + "_ensure_http_bridge_session_stream_lease_locked", + AsyncMock(side_effect=lease_failure), + ) + + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + ) + + assert exc_info.value is lease_failure + cleanup.assert_awaited_once() + assert cleanup.await_args is not None + assert cleanup.await_args.kwargs["admission_waiter_registered"] is False + assert cleanup.await_args.kwargs["request_enqueued"] is False + send_text.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_cleanup_http_bridge_submit_interruption_clears_restored_operation_identity() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + mark_operation_unknown = AsyncMock(return_value=True) + service._durable_bridge = cast(Any, SimpleNamespace(mark_operation_unknown=mark_operation_unknown)) + session = _make_bridge_session(key_value="restored-operation-identity") + session.durable_session_id = "durable-restored-operation-identity" + session.durable_owner_epoch = 2 + request_state = proxy_service._WebSocketRequestState( + request_id="req-restored-operation-identity", + model="gpt-5.5", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - response_create_gate=gate, - response_create_gate_acquired=True, - response_create_admission=admission, - account_response_create_lease=lease, - account_response_create_release=release_lease, - awaiting_response_created=True, + operation_id="operation-restored", + operation_fingerprint="fingerprint-restored", + operation_parent_response_id="resp-parent", + operation_registered=True, + operation_recovery_claimed=True, ) - session = _make_bridge_session(key_value="bridge-held-acquire") - session.response_create_gate = gate await service._cleanup_http_bridge_submit_interruption( session, @@ -18692,291 +21473,346 @@ async def test_cleanup_http_bridge_submit_interruption_releases_gate_state_when_ counted_in_queue=False, ) - release_lease.assert_awaited_once_with(lease) - assert admission.release.call_count == 1 - assert request_state.account_response_create_lease is None - assert request_state.account_response_create_release is None - assert request_state.response_create_gate is None - assert request_state.response_create_admission is None - assert request_state.awaiting_response_created is False - assert request_state.response_create_gate_acquired is False - assert gate._value == 1 + mark_operation_unknown.assert_awaited_once() + assert request_state.operation_recovery_claimed is False + assert request_state.operation_id is None + assert request_state.operation_fingerprint is None + assert request_state.operation_parent_response_id is None @pytest.mark.asyncio -async def test_cleanup_http_bridge_submit_interruption_does_not_release_unacquired_gate() -> None: +async def test_http_bridge_capacity_retry_reclaims_unknown_operation_before_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - release_lease = AsyncMock() - lease = proxy_service.AccountLease( - lease_id="lease-held", - account_id="acc-bridge", - kind="stream", - acquired_at=1.0, + send_text = AsyncMock() + session = _make_bridge_session(key_value="unknown-operation-capacity-retry") + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=AsyncMock()), ) - gate = asyncio.Semaphore(1) - await gate.acquire() + session.durable_session_id = "durable-unknown-operation-capacity-retry" + session.durable_owner_epoch = 4 + service._http_bridge_sessions[session.key] = session request_state = proxy_service._WebSocketRequestState( - request_id="req-submit-overload", - model="gpt-5.4", + request_id="req-unknown-operation-capacity-retry", + model="gpt-5.5", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), - response_create_gate=gate, - response_create_gate_acquired=False, - account_response_create_lease=lease, - account_response_create_release=release_lease, + hard_continuity_anchor=True, + previous_response_id="resp-parent", awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","input":"retry"}', + transport="http", + skip_request_log=True, ) - session = _make_bridge_session(key_value="bridge-held-unacquired") - session.response_create_gate = gate - - await service._cleanup_http_bridge_submit_interruption( - session, - request_state=request_state, - gate_acquired=False, - request_enqueued=False, - counted_in_queue=False, + existing_operation = SimpleNamespace( + operation_id="operation-existing-unknown", + session_id=session.durable_session_id, + state="unknown", + created=False, + event_spool_complete=False, + response_id=None, + ) + lookup = AsyncMock(return_value=existing_operation) + claim_unknown = AsyncMock(return_value=True) + restore_unknown = AsyncMock(return_value=True) + service._durable_bridge = cast( + Any, + SimpleNamespace( + get_operation_by_fingerprint=lookup, + get_operation=AsyncMock(return_value=existing_operation), + record_operation=AsyncMock(return_value=existing_operation), + claim_unknown_operation_for_recovery=claim_unknown, + mark_operation_unknown=restore_unknown, + release_live_session=AsyncMock(return_value=None), + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery", + http_responses_session_bridge_instance_id="instance-unknown-operation-capacity-retry", + ), + ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_allowed", AsyncMock(return_value=True)) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=0.0)) + monkeypatch.setattr(service, "_maybe_prewarm_http_bridge_session", AsyncMock()) + capacity_error = ProxyResponseError( + 429, + openai_error( + "account_response_create_cap", + "Account response-create concurrency limit reached", + error_type="rate_limit_error", + ), ) + admission_calls = 0 - release_lease.assert_awaited_once_with(lease) - assert request_state.account_response_create_lease is None - assert request_state.account_response_create_release is None - assert request_state.response_create_gate is None - assert request_state.awaiting_response_created is False - assert request_state.response_create_gate_acquired is False - assert gate.locked() is True - gate.release() + async def acquire_admission( + state: proxy_service._WebSocketRequestState, + *, + response_create_gate: asyncio.Semaphore, + **_kwargs: Any, + ) -> None: + nonlocal admission_calls + admission_calls += 1 + if admission_calls == 1: + raise capacity_error + state.response_create_gate = response_create_gate + await response_create_gate.acquire() + state.response_create_gate_acquired = True + state.awaiting_response_created = True + monkeypatch.setattr(service, "_acquire_request_state_response_create_admission", acquire_admission) + wait_calls = 0 -def test_websocket_admission_rejection_cancels_reservation_heartbeat_before_release() -> None: - source = inspect.getsource(proxy_service.ProxyService.proxy_responses_websocket) - start_index = source.index("except ProxyResponseError as exc:", source.index("not request_state_registered")) - branch = source[start_index : source.index("await proxy._emit_websocket_terminal_error", start_index)] + async def capacity_wait(**_kwargs: object): + nonlocal wait_calls + wait_calls += 1 + if False: + yield "" - assert "proxy._release_websocket_request_state_reservation(response_create_request_state)" in branch - assert "_release_websocket_reservation(request_state.api_key_reservation)" not in source + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 0.001) + monkeypatch.setattr(http_bridge_streaming_module, "_iter_account_capacity_wait_sse", capacity_wait) + async def send_and_finish(_text: str) -> None: + assert claim_unknown.await_count == 2 + event_queue = request_state.event_queue + assert event_queue is not None + await event_queue.put(None) -def test_websocket_request_state_reservation_release_cancels_heartbeat_before_release() -> None: - source = inspect.getsource(proxy_service.ProxyService._release_websocket_request_state_reservation) + send_text.side_effect = send_and_finish - assert source.index("_cancel_request_state_api_key_reservation_heartbeat") < source.index( - "_release_websocket_reservation" - ) + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=4, + propagate_http_errors=True, + downstream_turn_state=None, + request_deadline=time.monotonic() + 10.0, + ): + pass + + assert wait_calls == 1 + assert admission_calls == 2 + assert claim_unknown.await_count == 2 + restore_unknown.assert_awaited_once() + send_text.assert_awaited_once() + assert request_state.operation_id == "operation-existing-unknown" @pytest.mark.asyncio -async def test_recovery_submit_queue_rejection_does_not_publish_turn_alias() -> None: +async def test_submit_hard_turn_rolls_back_new_operation_before_retiring_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = _make_account_neutral_replay_session_key("queue-rejection") - send_text = AsyncMock() - session = _make_bridge_session(key=key, queued_request_count=1) - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace(send_text=send_text, close=AsyncMock()), - ) - session.durable_session_id = "durable-queue-rejection" - session.durable_owner_epoch = 2 - service._http_bridge_sessions[key] = session - register_turn_state = AsyncMock(return_value=DurableBridgeAliasRegistration.REGISTERED) - service._durable_bridge = cast(Any, SimpleNamespace(register_turn_state=register_turn_state)) + session = _make_bridge_session(key_value="hard-turn-unsent-operation") + session.durable_session_id = "durable-hard-turn-unsent-operation" + session.durable_owner_epoch = 3 + session.upstream_control.retire_after_drain = True + session.upstream_close_attempted = True request_state = proxy_service._WebSocketRequestState( - request_id="req-recovery-queue-rejection", - model="gpt-5.6-sol", + request_id="req-hard-turn-unsent-operation", + model="gpt-5.6", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), + hard_continuity_anchor=True, awaiting_response_created=True, event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', + request_text='{"type":"response.create","input":"same"}', transport="http", skip_request_log=True, ) + record_operation = AsyncMock( + return_value=SimpleNamespace( + created=True, + operation_id="operation-unsent", + state="submitted", + response_id=None, + event_spool_complete=False, + ) + ) + rollback_operation = AsyncMock(return_value=True) + service._durable_bridge = cast( + Any, + SimpleNamespace( + get_operation_by_fingerprint=AsyncMock(return_value=None), + get_operation=AsyncMock(return_value=None), + record_operation=record_operation, + rollback_operation_before_dispatch=rollback_operation, + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery", + http_responses_session_bridge_instance_id="instance-hard-turn-unsent-operation", + ), + ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_allowed", AsyncMock(return_value=True)) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=0.0)) with pytest.raises(proxy_service.ProxyResponseError) as exc_info: - await service._submit_http_bridge_request( + await service._submit_http_bridge_request_with_handoff( session, request_state=request_state, text_data=request_state.request_text or "{}", - queue_limit=1, - recovery_turn_state="http_turn_queue_rejection", + queue_limit=8, + request_scope_id="scope-hard-turn-unsent-operation", + owned_unanchored_handoff=False, ) - assert exc_info.value.payload["error"]["code"] == "bridge_queue_full" - register_turn_state.assert_not_awaited() - send_text.assert_not_awaited() - assert session.downstream_turn_state_aliases == set() - assert proxy_service._http_bridge_turn_state_alias_key("http_turn_queue_rejection", None) not in ( - service._http_bridge_turn_state_index + assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" + record_operation.assert_awaited_once() + rollback_operation.assert_awaited_once_with( + operation_id="operation-unsent", + session_id="durable-hard-turn-unsent-operation", + instance_id="instance-hard-turn-unsent-operation", + owner_epoch=3, ) + assert request_state.operation_created is False + assert request_state.operation_registered is False + assert request_state.operation_id is None + assert request_state.operation_fingerprint is None + assert request_state.operation_parent_response_id is None @pytest.mark.asyncio -async def test_recovery_submit_alias_persistence_failure_retires_before_send() -> None: +async def test_cleanup_http_bridge_submit_interruption_releases_gate_state_when_gate_already_acquired() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = _make_account_neutral_replay_session_key("alias-write-failure") - send_text = AsyncMock() - close = AsyncMock() - session = _make_bridge_session(key=key) - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace(send_text=send_text, close=close), - ) - session.durable_session_id = "durable-alias-write-failure" - session.durable_owner_epoch = 2 - service._http_bridge_sessions[key] = session - register_recovery_turn_state = AsyncMock(side_effect=RuntimeError("database unavailable")) - release_live_session = AsyncMock(return_value=None) - service._durable_bridge = cast( - Any, - SimpleNamespace( - register_recovery_turn_state=register_recovery_turn_state, - release_live_session=release_live_session, - ), + admission = cast(Any, SimpleNamespace(release=Mock())) + release_lease = AsyncMock() + lease = proxy_service.AccountLease( + lease_id="lease-held", + account_id="acc-bridge", + kind="stream", + acquired_at=1.0, ) + gate = asyncio.Semaphore(0) request_state = proxy_service._WebSocketRequestState( - request_id="req-recovery-alias-write-failure", - model="gpt-5.6-sol", + request_id="req-submit-leak", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), + response_create_gate=gate, + response_create_gate_acquired=True, + response_create_admission=admission, + account_response_create_lease=lease, + account_response_create_release=release_lease, awaiting_response_created=True, - event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', - transport="http", - skip_request_log=True, ) + session = _make_bridge_session(key_value="bridge-held-acquire") + session.response_create_gate = gate - with pytest.raises(proxy_service.ProxyResponseError) as exc_info: - await service._submit_http_bridge_request( - session, - request_state=request_state, - text_data=request_state.request_text or "{}", - queue_limit=1, - recovery_turn_state="http_turn_alias_write_failure", - ) - - assert exc_info.value.status_code == 502 - assert exc_info.value.payload["error"]["code"] == "bridge_continuity_persistence_failed" - register_recovery_turn_state.assert_awaited_once() - send_text.assert_not_awaited() - close.assert_awaited_once() - release_live_session.assert_awaited_once() - assert session.closed is True - assert session.queued_request_count == 0 - assert session.pending_requests == deque() - assert session.downstream_turn_state_aliases == set() - assert proxy_service._http_bridge_turn_state_alias_key("http_turn_alias_write_failure", None) not in ( - service._http_bridge_turn_state_index + await service._cleanup_http_bridge_submit_interruption( + session, + request_state=request_state, + gate_acquired=False, + request_enqueued=False, + counted_in_queue=False, ) + release_lease.assert_awaited_once_with(lease) + assert admission.release.call_count == 1 + assert request_state.account_response_create_lease is None + assert request_state.account_response_create_release is None + assert request_state.response_create_gate is None + assert request_state.response_create_admission is None + assert request_state.awaiting_response_created is False + assert request_state.response_create_gate_acquired is False + assert gate._value == 1 + @pytest.mark.asyncio -async def test_recovery_submit_owner_fence_rejection_retires_before_send() -> None: +async def test_cleanup_http_bridge_submit_interruption_does_not_release_unacquired_gate() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - send_text = AsyncMock() - close = AsyncMock() - session = _make_bridge_session(key_value="owner-fence-rejection") - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace(send_text=send_text, close=close), - ) - session.durable_session_id = "durable-owner-fence-rejection" - session.durable_owner_epoch = 2 - service._http_bridge_sessions[session.key] = session - record_recovery_attempt = AsyncMock(return_value=None) - service._durable_bridge = cast( - Any, - SimpleNamespace( - lookup_retry_circuit=AsyncMock(return_value=None), - record_recovery_attempt=record_recovery_attempt, - release_live_session=AsyncMock(return_value=None), - ), + release_lease = AsyncMock() + lease = proxy_service.AccountLease( + lease_id="lease-held", + account_id="acc-bridge", + kind="stream", + acquired_at=1.0, ) + gate = asyncio.Semaphore(1) + await gate.acquire() request_state = proxy_service._WebSocketRequestState( - request_id="req-owner-fence-rejection", - model="gpt-5.6-sol", + request_id="req-submit-overload", + model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=time.monotonic(), + response_create_gate=gate, + response_create_gate_acquired=False, + account_response_create_lease=lease, + account_response_create_release=release_lease, awaiting_response_created=True, - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', - fresh_upstream_request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', - fresh_upstream_request_is_retry_safe=True, - transport="http", - skip_request_log=True, ) + session = _make_bridge_session(key_value="bridge-held-unacquired") + session.response_create_gate = gate - with pytest.raises(proxy_service.ProxyResponseError) as exc_info: - await service._submit_http_bridge_request( - session, - request_state=request_state, - text_data=request_state.request_text or "{}", - queue_limit=8, - ) + await service._cleanup_http_bridge_submit_interruption( + session, + request_state=request_state, + gate_acquired=False, + request_enqueued=False, + counted_in_queue=False, + ) - assert exc_info.value.status_code == 502 - assert exc_info.value.payload["error"]["code"] == "bridge_continuity_persistence_failed" - record_recovery_attempt.assert_awaited_once() - send_text.assert_not_awaited() - assert session.closed is True + release_lease.assert_awaited_once_with(lease) + assert request_state.account_response_create_lease is None + assert request_state.account_response_create_release is None + assert request_state.response_create_gate is None + assert request_state.awaiting_response_created is False + assert request_state.response_create_gate_acquired is False + assert gate.locked() is True + gate.release() + + +def test_websocket_admission_rejection_cancels_reservation_heartbeat_before_release() -> None: + source = inspect.getsource(proxy_service.ProxyService.proxy_responses_websocket) + start_index = source.index("except ProxyResponseError as exc:", source.index("not request_state_registered")) + branch = source[start_index : source.index("await proxy._emit_websocket_terminal_error", start_index)] + + assert "proxy._release_websocket_request_state_reservation(response_create_request_state)" in branch + assert "_release_websocket_reservation(request_state.api_key_reservation)" not in source + + +def test_websocket_request_state_reservation_release_cancels_heartbeat_before_release() -> None: + source = inspect.getsource(proxy_service.ProxyService._release_websocket_request_state_reservation) + + assert source.index("_cancel_request_state_api_key_reservation_heartbeat") < source.index( + "_release_websocket_reservation" + ) @pytest.mark.asyncio -async def test_recovery_submit_cancellation_after_alias_commit_restores_previous_owner() -> None: +async def test_recovery_submit_queue_rejection_does_not_publish_turn_alias() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = _make_account_neutral_replay_session_key("alias-commit-cancel") + key = _make_account_neutral_replay_session_key("queue-rejection") send_text = AsyncMock() - close = AsyncMock() - session = _make_bridge_session(key=key) + session = _make_bridge_session(key=key, queued_request_count=1) session.upstream = cast( UpstreamWebSocket, - SimpleNamespace(send_text=send_text, close=close), + SimpleNamespace(send_text=send_text, close=AsyncMock()), ) - session.durable_session_id = "durable-recovery" - session.durable_owner_epoch = 4 + session.durable_session_id = "durable-queue-rejection" + session.durable_owner_epoch = 2 service._http_bridge_sessions[key] = session - alias_owner = {"http_turn_commit_cancel": "durable-predecessor"} - alias_committed = asyncio.Event() - release_registration = asyncio.Event() - - async def register_recovery_turn_state(**_kwargs: Any) -> DurableBridgeAliasRegistrationReceipt: - alias_owner["http_turn_commit_cancel"] = "durable-recovery" - alias_committed.set() - await release_registration.wait() - return DurableBridgeAliasRegistrationReceipt( - status=DurableBridgeAliasRegistration.REGISTERED, - session_id="durable-recovery", - api_key_scope="__anonymous__", - alias_kind="turn_state", - alias_value="http_turn_commit_cancel", - instance_id="test-instance", - owner_epoch=4, - previous_alias_session_id="durable-predecessor", - previous_alias_owner_epoch=1, - previous_alias_account_id="acc-predecessor", - previous_latest_turn_state=None, - ) - - async def rollback_recovery_turn_state_registration(**_kwargs: Any) -> bool: - alias_owner["http_turn_commit_cancel"] = "durable-predecessor" - return True - - release_live_session = AsyncMock(return_value=None) - service._durable_bridge = cast( - Any, - SimpleNamespace( - register_recovery_turn_state=register_recovery_turn_state, - rollback_recovery_turn_state_registration=rollback_recovery_turn_state_registration, - release_live_session=release_live_session, - ), - ) + register_turn_state = AsyncMock(return_value=DurableBridgeAliasRegistration.REGISTERED) + service._durable_bridge = cast(Any, SimpleNamespace(register_turn_state=register_turn_state)) request_state = proxy_service._WebSocketRequestState( - request_id="req-recovery-alias-commit-cancel", + request_id="req-recovery-queue-rejection", model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, @@ -18989,256 +21825,121 @@ async def rollback_recovery_turn_state_registration(**_kwargs: Any) -> bool: skip_request_log=True, ) - submit = asyncio.create_task( - service._submit_http_bridge_request( + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._submit_http_bridge_request( session, request_state=request_state, text_data=request_state.request_text or "{}", queue_limit=1, - recovery_turn_state="http_turn_commit_cancel", - ) - ) - try: - await asyncio.wait_for(alias_committed.wait(), timeout=1.0) - submit.cancel() - await asyncio.sleep(0) - assert not submit.done() - release_registration.set() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(submit, timeout=1.0) - finally: - release_registration.set() - if not submit.done(): - submit.cancel() - await asyncio.gather(submit, return_exceptions=True) - - assert alias_owner["http_turn_commit_cancel"] == "durable-predecessor" - send_text.assert_not_awaited() - close.assert_awaited_once() - release_live_session.assert_awaited_once() - assert session.closed is True - assert session.queued_request_count == 0 - assert session.pending_requests == deque() - - -@pytest.mark.asyncio -async def test_recovery_send_cancellation_retires_before_admitted_waiter_can_reconnect() -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = _make_account_neutral_replay_session_key("ambiguous-send-waiter") - send_started = asyncio.Event() - - async def send_text_once(_text: str) -> None: - send_started.set() - await asyncio.Future() - - send_text = AsyncMock(side_effect=send_text_once) - close = AsyncMock() - session = _make_bridge_session(key=key) - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace(send_text=send_text, close=close), - ) - session.durable_session_id = "durable-ambiguous-send" - session.durable_owner_epoch = 5 - service._http_bridge_sessions[key] = session - receipt = DurableBridgeAliasRegistrationReceipt( - status=DurableBridgeAliasRegistration.REGISTERED, - session_id="durable-ambiguous-send", - api_key_scope="__anonymous__", - alias_kind="turn_state", - alias_value="http_turn_ambiguous_send", - instance_id="test-instance", - owner_epoch=5, - previous_alias_session_id="durable-predecessor", - previous_alias_owner_epoch=1, - previous_alias_account_id="acc-predecessor", - previous_latest_turn_state=None, - ) - register_recovery_turn_state = AsyncMock(return_value=receipt) - rollback_registration = AsyncMock(return_value=True) - release_live_session = AsyncMock(return_value=None) - service._durable_bridge = cast( - Any, - SimpleNamespace( - register_recovery_turn_state=register_recovery_turn_state, - rollback_recovery_turn_state_registration=rollback_registration, - release_live_session=release_live_session, - ), - ) - reconnect = AsyncMock(return_value=True) - service._retry_http_bridge_request_on_fresh_upstream = reconnect # type: ignore[method-assign] - - def make_request(request_id: str) -> proxy_service._WebSocketRequestState: - return proxy_service._WebSocketRequestState( - request_id=request_id, - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, - event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', - transport="http", - skip_request_log=True, - ) - - first = asyncio.create_task( - service._submit_http_bridge_request( - session, - request_state=make_request("req-ambiguous-send"), - text_data='{"type":"response.create","model":"gpt-5.6-sol","input":"first"}', - queue_limit=2, - recovery_turn_state="http_turn_ambiguous_send", - ) - ) - second = None - try: - await asyncio.wait_for(send_started.wait(), timeout=1.0) - second = asyncio.create_task( - service._submit_http_bridge_request( - session, - request_state=make_request("req-admitted-waiter"), - text_data='{"type":"response.create","model":"gpt-5.6-sol","input":"second"}', - queue_limit=2, - recovery_turn_state="http_turn_ambiguous_send", - ) + recovery_turn_state="http_turn_queue_rejection", ) - for _ in range(20): - if session.admission_waiter_count == 1: - break - await asyncio.sleep(0) - assert session.admission_waiter_count == 1 - - first.cancel() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(first, timeout=1.0) - with pytest.raises(proxy_service.ProxyResponseError): - await asyncio.wait_for(second, timeout=1.0) - finally: - for task in (first, second): - if task is not None and not task.done(): - task.cancel() - await asyncio.gather(*(task for task in (first, second) if task is not None), return_exceptions=True) - assert send_text.await_count == 1 - reconnect.assert_not_awaited() - rollback_registration.assert_not_awaited() - close.assert_awaited_once() - release_live_session.assert_awaited_once() - assert session.closed is True - assert session.queued_request_count == 0 - assert session.pending_requests == deque() + assert exc_info.value.payload["error"]["code"] == "bridge_queue_full" + register_turn_state.assert_not_awaited() + send_text.assert_not_awaited() + assert session.downstream_turn_state_aliases == set() + assert proxy_service._http_bridge_turn_state_alias_key("http_turn_queue_rejection", None) not in ( + service._http_bridge_turn_state_index + ) @pytest.mark.asyncio -async def test_submit_http_bridge_request_rejects_retiring_session() -> None: +async def test_recovery_submit_alias_persistence_failure_retires_before_send() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = _make_account_neutral_replay_session_key("alias-write-failure") send_text = AsyncMock() close = AsyncMock() - pending_request_state = proxy_service._WebSocketRequestState( - request_id="req-pending-retire", - model="gpt-5.5", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - response_id="resp_pending_retire", - awaiting_response_created=False, - event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.5","input":"pending"}', - transport="http", - skip_request_log=True, + session = _make_bridge_session(key=key) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=close), ) - new_request_state = proxy_service._WebSocketRequestState( - request_id="req-new-retire", - model="gpt-5.5", + session.durable_session_id = "durable-alias-write-failure" + session.durable_owner_epoch = 2 + service._http_bridge_sessions[key] = session + register_recovery_turn_state = AsyncMock(side_effect=RuntimeError("database unavailable")) + release_live_session = AsyncMock(return_value=None) + service._durable_bridge = cast( + Any, + SimpleNamespace( + register_recovery_turn_state=register_recovery_turn_state, + release_live_session=release_live_session, + ), + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-recovery-alias-write-failure", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=2.0, + started_at=time.monotonic(), awaiting_response_created=True, event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', transport="http", skip_request_log=True, ) - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_retiring", None), - headers={"x-codex-turn-state": "http_turn_retiring"}, - affinity=proxy_service._AffinityPolicy( - key="http_turn_retiring", - kind=proxy_service.StickySessionKind.CODEX_SESSION, - ), - request_model="gpt-5.5", - account=cast(Any, SimpleNamespace(id="acc-limited", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=send_text, close=close)), - upstream_control=proxy_service._WebSocketUpstreamControl( - reconnect_requested=True, - retire_after_drain=True, - ), - pending_requests=deque([pending_request_state]), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=1, - last_used_at=1.0, - idle_ttl_seconds=120.0, - ) with pytest.raises(proxy_service.ProxyResponseError) as exc_info: await service._submit_http_bridge_request( session, - request_state=new_request_state, - text_data=new_request_state.request_text or "{}", - queue_limit=8, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=1, + recovery_turn_state="http_turn_alias_write_failure", ) assert exc_info.value.status_code == 502 - assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" - assert session.pending_requests == deque([pending_request_state]) - assert session.queued_request_count == 1 - assert session.closed is False + assert exc_info.value.payload["error"]["code"] == "bridge_continuity_persistence_failed" + register_recovery_turn_state.assert_awaited_once() send_text.assert_not_awaited() - close.assert_not_awaited() + close.assert_awaited_once() + release_live_session.assert_awaited_once() + assert session.closed is True + assert session.queued_request_count == 0 + assert session.pending_requests == deque() + assert session.downstream_turn_state_aliases == set() + assert proxy_service._http_bridge_turn_state_alias_key("http_turn_alias_write_failure", None) not in ( + service._http_bridge_turn_state_index + ) @pytest.mark.asyncio -async def test_submit_http_bridge_request_rejects_unregistered_session_after_admission() -> None: +async def test_recovery_submit_owner_fence_rejection_retires_before_send() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) send_text = AsyncMock() + close = AsyncMock() + session = _make_bridge_session(key_value="owner-fence-rejection") + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=close), + ) + session.durable_session_id = "durable-owner-fence-rejection" + session.durable_owner_epoch = 2 + service._http_bridge_sessions[session.key] = session + record_recovery_attempt = AsyncMock(return_value=None) + service._durable_bridge = cast( + Any, + SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + record_recovery_attempt=record_recovery_attempt, + release_live_session=AsyncMock(return_value=None), + ), + ) request_state = proxy_service._WebSocketRequestState( - request_id="req-unregistered-submit", - model="gpt-5.5", + request_id="req-owner-fence-rejection", + model="gpt-5.6-sol", service_tier=None, reasoning_effort=None, api_key_reservation=None, - # started_at is monotonic in production; the budget clamp on bridge - # gate waits treats stale values as an exhausted request budget. started_at=time.monotonic(), awaiting_response_created=True, - event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', + fresh_upstream_request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', + fresh_upstream_request_is_retry_safe=True, transport="http", skip_request_log=True, ) - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_unregistered", None), - headers={"x-codex-turn-state": "http_turn_unregistered"}, - affinity=proxy_service._AffinityPolicy( - key="http_turn_unregistered", - kind=proxy_service.StickySessionKind.CODEX_SESSION, - ), - request_model="gpt-5.5", - account=cast(Any, SimpleNamespace(id="acc-unregistered", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=send_text, close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, - ) with pytest.raises(proxy_service.ProxyResponseError) as exc_info: await service._submit_http_bridge_request( @@ -19249,153 +21950,243 @@ async def test_submit_http_bridge_request_rejects_unregistered_session_after_adm ) assert exc_info.value.status_code == 502 - assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" - assert session.pending_requests == deque() - assert session.queued_request_count == 0 - assert request_state.response_create_gate is None - assert request_state.response_create_gate_acquired is False - assert session.response_create_gate.locked() is False + assert exc_info.value.payload["error"]["code"] == "bridge_continuity_persistence_failed" + record_recovery_attempt.assert_awaited_once() send_text.assert_not_awaited() + assert session.closed is True @pytest.mark.asyncio -async def test_submit_http_bridge_request_rejects_unregistered_closed_session_without_reconnect( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_recovery_submit_cancellation_after_alias_commit_restores_previous_owner() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = _make_account_neutral_replay_session_key("alias-commit-cancel") send_text = AsyncMock() - retry_fresh = AsyncMock(return_value=True) - monkeypatch.setattr(service, "_retry_http_bridge_request_on_fresh_upstream", retry_fresh) - request_state = proxy_service._WebSocketRequestState( - request_id="req-unregistered-closed-submit", - model="gpt-5.5", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - awaiting_response_created=True, - event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', - transport="http", - skip_request_log=True, + close = AsyncMock() + session = _make_bridge_session(key=key) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=close), ) - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_unregistered_closed", None), - headers={"x-codex-turn-state": "http_turn_unregistered_closed"}, - affinity=proxy_service._AffinityPolicy( - key="http_turn_unregistered_closed", - kind=proxy_service.StickySessionKind.CODEX_SESSION, + session.durable_session_id = "durable-recovery" + session.durable_owner_epoch = 4 + service._http_bridge_sessions[key] = session + alias_owner = {"http_turn_commit_cancel": "durable-predecessor"} + alias_committed = asyncio.Event() + release_registration = asyncio.Event() + + async def register_recovery_turn_state(**_kwargs: Any) -> DurableBridgeAliasRegistrationReceipt: + alias_owner["http_turn_commit_cancel"] = "durable-recovery" + alias_committed.set() + await release_registration.wait() + return DurableBridgeAliasRegistrationReceipt( + status=DurableBridgeAliasRegistration.REGISTERED, + session_id="durable-recovery", + api_key_scope="__anonymous__", + alias_kind="turn_state", + alias_value="http_turn_commit_cancel", + instance_id="test-instance", + owner_epoch=4, + previous_alias_session_id="durable-predecessor", + previous_alias_owner_epoch=1, + previous_alias_account_id="acc-predecessor", + previous_latest_turn_state=None, + ) + + async def rollback_recovery_turn_state_registration(**_kwargs: Any) -> bool: + alias_owner["http_turn_commit_cancel"] = "durable-predecessor" + return True + + release_live_session = AsyncMock(return_value=None) + service._durable_bridge = cast( + Any, + SimpleNamespace( + register_recovery_turn_state=register_recovery_turn_state, + rollback_recovery_turn_state_registration=rollback_recovery_turn_state_registration, + release_live_session=release_live_session, ), - request_model="gpt-5.5", - account=cast(Any, SimpleNamespace(id="acc-unregistered", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=send_text, close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, - closed=True, + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-recovery-alias-commit-cancel", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', + transport="http", + skip_request_log=True, ) - with pytest.raises(proxy_service.ProxyResponseError) as exc_info: - await service._submit_http_bridge_request( + submit = asyncio.create_task( + service._submit_http_bridge_request( session, request_state=request_state, text_data=request_state.request_text or "{}", - queue_limit=8, + queue_limit=1, + recovery_turn_state="http_turn_commit_cancel", ) + ) + try: + await asyncio.wait_for(alias_committed.wait(), timeout=1.0) + submit.cancel() + await asyncio.sleep(0) + assert not submit.done() + release_registration.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(submit, timeout=1.0) + finally: + release_registration.set() + if not submit.done(): + submit.cancel() + await asyncio.gather(submit, return_exceptions=True) - assert exc_info.value.status_code == 502 - assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" - retry_fresh.assert_not_awaited() + assert alias_owner["http_turn_commit_cancel"] == "durable-predecessor" send_text.assert_not_awaited() + close.assert_awaited_once() + release_live_session.assert_awaited_once() + assert session.closed is True + assert session.queued_request_count == 0 + assert session.pending_requests == deque() @pytest.mark.asyncio -async def test_submit_http_bridge_request_waits_for_closed_session_retirement_before_reconnect( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_recovery_send_cancellation_retires_before_admitted_waiter_can_reconnect() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - send_text = AsyncMock() - retry_started = asyncio.Event() + key = _make_account_neutral_replay_session_key("ambiguous-send-waiter") + send_started = asyncio.Event() - async def retry_fresh(*_args: object, **_kwargs: object) -> bool: - retry_started.set() - return True + async def send_text_once(_text: str) -> None: + send_started.set() + await asyncio.Future() - monkeypatch.setattr(service, "_retry_http_bridge_request_on_fresh_upstream", retry_fresh) - request_state = proxy_service._WebSocketRequestState( - request_id="req-closed-retiring-submit", - model="gpt-5.5", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - awaiting_response_created=True, - event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', - transport="http", - skip_request_log=True, + send_text = AsyncMock(side_effect=send_text_once) + close = AsyncMock() + session = _make_bridge_session(key=key) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=close), ) - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_closed_retiring", None), - headers={"x-codex-turn-state": "http_turn_closed_retiring"}, - affinity=proxy_service._AffinityPolicy( - key="http_turn_closed_retiring", - kind=proxy_service.StickySessionKind.CODEX_SESSION, + session.durable_session_id = "durable-ambiguous-send" + session.durable_owner_epoch = 5 + service._http_bridge_sessions[key] = session + receipt = DurableBridgeAliasRegistrationReceipt( + status=DurableBridgeAliasRegistration.REGISTERED, + session_id="durable-ambiguous-send", + api_key_scope="__anonymous__", + alias_kind="turn_state", + alias_value="http_turn_ambiguous_send", + instance_id="test-instance", + owner_epoch=5, + previous_alias_session_id="durable-predecessor", + previous_alias_owner_epoch=1, + previous_alias_account_id="acc-predecessor", + previous_latest_turn_state=None, + ) + register_recovery_turn_state = AsyncMock(return_value=receipt) + rollback_registration = AsyncMock(return_value=True) + release_live_session = AsyncMock(return_value=None) + service._durable_bridge = cast( + Any, + SimpleNamespace( + register_recovery_turn_state=register_recovery_turn_state, + rollback_recovery_turn_state_registration=rollback_registration, + release_live_session=release_live_session, ), - request_model="gpt-5.5", - account=cast(Any, SimpleNamespace(id="acc-retiring", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=send_text, close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, - closed=True, ) - service._http_bridge_sessions[session.key] = session + reconnect = AsyncMock(return_value=True) + service._retry_http_bridge_request_on_fresh_upstream = reconnect # type: ignore[method-assign] - async with session.lifecycle_lock: - submit_task = asyncio.create_task( + def make_request(request_id: str) -> proxy_service._WebSocketRequestState: + return proxy_service._WebSocketRequestState( + request_id=request_id, + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hi"}', + transport="http", + skip_request_log=True, + ) + + first = asyncio.create_task( + service._submit_http_bridge_request( + session, + request_state=make_request("req-ambiguous-send"), + text_data='{"type":"response.create","model":"gpt-5.6-sol","input":"first"}', + queue_limit=2, + recovery_turn_state="http_turn_ambiguous_send", + ) + ) + second = None + try: + await asyncio.wait_for(send_started.wait(), timeout=1.0) + second = asyncio.create_task( service._submit_http_bridge_request( session, - request_state=request_state, - text_data=request_state.request_text or "{}", - queue_limit=8, + request_state=make_request("req-admitted-waiter"), + text_data='{"type":"response.create","model":"gpt-5.6-sol","input":"second"}', + queue_limit=2, + recovery_turn_state="http_turn_ambiguous_send", ) ) - try: - with pytest.raises(asyncio.TimeoutError): - await asyncio.wait_for(retry_started.wait(), timeout=0.01) - service._http_bridge_sessions.pop(session.key, None) - finally: - if submit_task.done(): - await submit_task + for _ in range(20): + if session.admission_waiter_count == 1: + break + await asyncio.sleep(0) + assert session.admission_waiter_count == 1 - with pytest.raises(proxy_service.ProxyResponseError) as exc_info: - await submit_task + first.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(first, timeout=1.0) + with pytest.raises(proxy_service.ProxyResponseError): + await asyncio.wait_for(second, timeout=1.0) + finally: + for task in (first, second): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather(*(task for task in (first, second) if task is not None), return_exceptions=True) - assert exc_info.value.status_code == 502 - assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" - assert retry_started.is_set() is False - send_text.assert_not_awaited() + assert send_text.await_count == 1 + reconnect.assert_not_awaited() + rollback_registration.assert_not_awaited() + close.assert_awaited_once() + release_live_session.assert_awaited_once() + assert session.closed is True + assert session.queued_request_count == 0 + assert session.pending_requests == deque() @pytest.mark.asyncio -async def test_submit_http_bridge_request_does_not_send_after_retirement_between_validation_and_send() -> None: +async def test_submit_http_bridge_request_rejects_retiring_session() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - request_state = proxy_service._WebSocketRequestState( - request_id="req-retired-after-submit-validation", + send_text = AsyncMock() + close = AsyncMock() + pending_request_state = proxy_service._WebSocketRequestState( + request_id="req-pending-retire", model="gpt-5.5", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=1.0, + response_id="resp_pending_retire", + awaiting_response_created=False, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.5","input":"pending"}', + transport="http", + skip_request_log=True, + ) + new_request_state = proxy_service._WebSocketRequestState( + request_id="req-new-retire", + model="gpt-5.5", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=2.0, awaiting_response_created=True, event_queue=asyncio.Queue(), request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', @@ -19403,88 +22194,72 @@ async def test_submit_http_bridge_request_does_not_send_after_retirement_between skip_request_log=True, ) session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_retire_gap", None), - headers={"x-codex-turn-state": "http_turn_retire_gap"}, + key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_retiring", None), + headers={"x-codex-turn-state": "http_turn_retiring"}, affinity=proxy_service._AffinityPolicy( - key="http_turn_retire_gap", + key="http_turn_retiring", kind=proxy_service.StickySessionKind.CODEX_SESSION, ), request_model="gpt-5.5", - account=cast(Any, SimpleNamespace(id="acc-retire-gap", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=AsyncMock(), close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), + account=cast(Any, SimpleNamespace(id="acc-limited", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=send_text, close=close)), + upstream_control=proxy_service._WebSocketUpstreamControl( + reconnect_requested=True, + retire_after_drain=True, + ), + pending_requests=deque([pending_request_state]), pending_lock=anyio.Lock(), response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, + queued_request_count=1, last_used_at=1.0, idle_ttl_seconds=120.0, ) - service._http_bridge_sessions[session.key] = session - stale_send_seen = False - - async def send_text(_text: str) -> None: - nonlocal stale_send_seen - stale_send_seen = service._http_bridge_sessions.get(session.key) is not session or session.closed - - cast(Any, session.upstream).send_text.side_effect = send_text - class RetireAfterValidationLock: - async def __aenter__(self) -> None: - return None - - async def __aexit__(self, *_exc: object) -> None: - if request_state.response_create_gate_acquired and request_state not in session.pending_requests: - session.closed = True - service._http_bridge_sessions.pop(session.key, None) - return None - - service._http_bridge_lock = cast(Any, RetireAfterValidationLock()) - - try: + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: await service._submit_http_bridge_request( session, - request_state=request_state, - text_data=request_state.request_text or "{}", + request_state=new_request_state, + text_data=new_request_state.request_text or "{}", queue_limit=8, ) - except proxy_service.ProxyResponseError: - pass - finally: - if request_state.response_create_gate_acquired: - await proxy_service._release_websocket_response_create_gate(request_state, session.response_create_gate) - assert stale_send_seen is False + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" + assert session.pending_requests == deque([pending_request_state]) + assert session.queued_request_count == 1 + assert session.closed is False + send_text.assert_not_awaited() + close.assert_not_awaited() @pytest.mark.asyncio -async def test_submit_http_bridge_request_rejects_state_after_response_event() -> None: +async def test_submit_http_bridge_request_rejects_unregistered_session_after_admission() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) send_text = AsyncMock() request_state = proxy_service._WebSocketRequestState( - request_id="req-visible-submit", + request_id="req-unregistered-submit", model="gpt-5.5", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, - response_id="resp_visible_submit", - response_event_count=1, - awaiting_response_created=False, + # started_at is monotonic in production; the budget clamp on bridge + # gate waits treats stale values as an exhausted request budget. + started_at=time.monotonic(), + awaiting_response_created=True, event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.5","input":"visible"}', + request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', transport="http", skip_request_log=True, ) session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_visible_submit", None), - headers={"x-codex-turn-state": "http_turn_visible_submit"}, + key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_unregistered", None), + headers={"x-codex-turn-state": "http_turn_unregistered"}, affinity=proxy_service._AffinityPolicy( - key="http_turn_visible_submit", + key="http_turn_unregistered", kind=proxy_service.StickySessionKind.CODEX_SESSION, ), request_model="gpt-5.5", - account=cast(Any, SimpleNamespace(id="acc-visible-submit", status=AccountStatus.ACTIVE)), + account=cast(Any, SimpleNamespace(id="acc-unregistered", status=AccountStatus.ACTIVE)), upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=send_text, close=AsyncMock())), upstream_control=proxy_service._WebSocketUpstreamControl(), pending_requests=deque(), @@ -19507,540 +22282,342 @@ async def test_submit_http_bridge_request_rejects_state_after_response_event() - assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" assert session.pending_requests == deque() assert session.queued_request_count == 0 + assert request_state.response_create_gate is None + assert request_state.response_create_gate_acquired is False + assert session.response_create_gate.locked() is False send_text.assert_not_awaited() @pytest.mark.asyncio -async def test_submit_http_bridge_request_recovers_once_after_proven_pre_dispatch_close( +async def test_submit_http_bridge_request_rejects_unregistered_closed_session_without_reconnect( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - closed_send = AsyncMock( - side_effect=UpstreamWebSocketTransportError( - "already closed", - error_code=UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - ) - ) - replacement_send = AsyncMock() + send_text = AsyncMock() + retry_fresh = AsyncMock(return_value=True) + monkeypatch.setattr(service, "_retry_http_bridge_request_on_fresh_upstream", retry_fresh) request_state = proxy_service._WebSocketRequestState( - request_id="req-pre-dispatch-recovery", + request_id="req-unregistered-closed-submit", model="gpt-5.5", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), + started_at=1.0, awaiting_response_created=True, event_queue=asyncio.Queue(), request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', transport="http", skip_request_log=True, ) - session = _make_bridge_session(key_value="pre-dispatch-recovery") - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace(send_text=closed_send, close=AsyncMock()), - ) - service._http_bridge_sessions[session.key] = session - monkeypatch.setattr( - service, - "_http_bridge_precreated_retry_decision", - AsyncMock(return_value=http_bridge_retry_circuit_module._HTTPBridgeRetryCircuitDecision(allowed=True)), - ) - - async def retry_on_replacement( - target_session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, - send_request: bool, - require_same_account: bool, - ) -> bool: - assert target_session is session - assert send_request is True - assert require_same_account is True - request_state.replay_count += 1 - await replacement_send(text_data) - return True - - retry = AsyncMock(side_effect=retry_on_replacement) - monkeypatch.setattr(service, "_retry_http_bridge_request_on_fresh_upstream", retry) - - await service._submit_http_bridge_request( - session, - request_state=request_state, - text_data=request_state.request_text or "{}", - queue_limit=8, + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_unregistered_closed", None), + headers={"x-codex-turn-state": "http_turn_unregistered_closed"}, + affinity=proxy_service._AffinityPolicy( + key="http_turn_unregistered_closed", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + request_model="gpt-5.5", + account=cast(Any, SimpleNamespace(id="acc-unregistered", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=send_text, close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + closed=True, ) - closed_send.assert_awaited_once() - replacement_send.assert_awaited_once_with(request_state.request_text) - retry.assert_awaited_once() - assert request_state.replay_count == 1 - assert request_state.recovery_attempt_dispatched is True - await service._detach_http_bridge_request(session, request_state=request_state) - - -@pytest.mark.asyncio -async def test_submit_http_bridge_request_keeps_uploaded_file_recovery_on_owner_account( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - closed_send = AsyncMock( - side_effect=UpstreamWebSocketTransportError( - "already closed", - error_code=UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - ) - ) - request_state = proxy_service._WebSocketRequestState( - request_id="req-file-owner-pre-dispatch-recovery", - model="gpt-5.5", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, - event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.5","input":"use file"}', - transport="http", - skip_request_log=True, - file_required_preferred_account=True, - ) - session = _make_bridge_session( - key=proxy_service._HTTPBridgeSessionKey( - "prompt_cache", - "file-owner-pre-dispatch-recovery", - None, + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, ) - ) - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace(send_text=closed_send, close=AsyncMock()), - ) - service._http_bridge_sessions[session.key] = session - monkeypatch.setattr( - service, - "_http_bridge_precreated_retry_decision", - AsyncMock(return_value=http_bridge_retry_circuit_module._HTTPBridgeRetryCircuitDecision(allowed=True)), - ) - retry = AsyncMock(return_value=True) - monkeypatch.setattr(service, "_retry_http_bridge_request_on_fresh_upstream", retry) - - await service._submit_http_bridge_request( - session, - request_state=request_state, - text_data=request_state.request_text or "{}", - queue_limit=8, - ) - retry.assert_awaited_once_with( - session, - request_state=request_state, - text_data=request_state.request_text, - send_request=True, - require_same_account=True, - ) - assert request_state.recovery_attempt_dispatched is True - await service._detach_http_bridge_request(session, request_state=request_state) + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" + retry_fresh.assert_not_awaited() + send_text.assert_not_awaited() @pytest.mark.asyncio -@pytest.mark.parametrize( - ("key_kind", "account_neutral"), - [ - ("session_header", False), - ("session_header", True), - ("prompt_cache", False), - ("prompt_cache", True), - ], -) -async def test_submit_http_bridge_request_keeps_current_logical_key_on_owner( +async def test_submit_http_bridge_request_waits_for_closed_session_retirement_before_reconnect( monkeypatch: pytest.MonkeyPatch, - key_kind: str, - account_neutral: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - closed_send = AsyncMock( - side_effect=UpstreamWebSocketTransportError( - "already closed", - error_code=UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - ) - ) - fresh_body = ( - '{"type":"response.create","model":"gpt-5.5","input":"full resend"}' - if account_neutral - else '{"type":"response.create","model":"gpt-5.5","conversation":"conv_owner","input":"full resend"}' - ) - request_state = proxy_service._WebSocketRequestState( - request_id=f"req-account-neutral-{account_neutral}", - model="gpt-5.5", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, - event_queue=asyncio.Queue(), - previous_response_id="resp_owner", - request_text=( - '{"type":"response.create","model":"gpt-5.5","previous_response_id":"resp_owner","input":"delta"}' - ), - fresh_upstream_request_text=fresh_body, - fresh_upstream_request_is_retry_safe=True, - fresh_upstream_request_is_account_neutral=account_neutral, - transport="http", - skip_request_log=True, - ) - session = _make_bridge_session( - key=proxy_service._HTTPBridgeSessionKey( - key_kind, - f"fresh-body-account-neutral-{account_neutral}", - None, - ) - ) - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace(send_text=closed_send, close=AsyncMock()), - ) - service._http_bridge_sessions[session.key] = session - monkeypatch.setattr( - service, - "_http_bridge_precreated_retry_decision", - AsyncMock(return_value=http_bridge_retry_circuit_module._HTTPBridgeRetryCircuitDecision(allowed=True)), - ) - retry = AsyncMock(return_value=True) - monkeypatch.setattr(service, "_retry_http_bridge_request_on_fresh_upstream", retry) - - await service._submit_http_bridge_request( - session, - request_state=request_state, - text_data=request_state.request_text or "{}", - queue_limit=8, - ) - - retry.assert_awaited_once_with( - session, - request_state=request_state, - text_data=request_state.request_text, - send_request=True, - require_same_account=True, - ) - await service._detach_http_bridge_request(session, request_state=request_state) + send_text = AsyncMock() + retry_started = asyncio.Event() + async def retry_fresh(*_args: object, **_kwargs: object) -> bool: + retry_started.set() + return True -@pytest.mark.asyncio -async def test_submit_http_bridge_request_does_not_loop_after_replacement_is_also_closed( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - closed_error = UpstreamWebSocketTransportError( - "already closed", - error_code=UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - ) - closed_send = AsyncMock(side_effect=closed_error) + monkeypatch.setattr(service, "_retry_http_bridge_request_on_fresh_upstream", retry_fresh) request_state = proxy_service._WebSocketRequestState( - request_id="req-pre-dispatch-recovery-closed-twice", + request_id="req-closed-retiring-submit", model="gpt-5.5", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), + started_at=1.0, awaiting_response_created=True, event_queue=asyncio.Queue(), request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', transport="http", skip_request_log=True, ) - key = _make_account_neutral_replay_session_key("pre-dispatch-recovery-closed-twice") - session = _make_bridge_session(key=key) - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace(send_text=closed_send, close=AsyncMock()), - ) - session.durable_session_id = "durable-pre-dispatch-recovery-closed-twice" - session.durable_owner_epoch = 7 - service._http_bridge_sessions[session.key] = session - receipt = DurableBridgeAliasRegistrationReceipt( - status=DurableBridgeAliasRegistration.REGISTERED, - session_id=session.durable_session_id, - api_key_scope="__anonymous__", - alias_kind="turn_state", - alias_value="turn-pre-dispatch-recovery-closed-twice", - instance_id="test-instance", - owner_epoch=session.durable_owner_epoch, - previous_alias_session_id="durable-predecessor", - previous_alias_owner_epoch=3, - previous_alias_account_id="acc-predecessor", - previous_latest_turn_state=None, - ) - register_recovery_turn_state = AsyncMock(return_value=receipt) - rollback_registration = AsyncMock(return_value=True) - release_live_session = AsyncMock(return_value=None) - service._durable_bridge = cast( - Any, - SimpleNamespace( - register_recovery_turn_state=register_recovery_turn_state, - rollback_recovery_turn_state_registration=rollback_registration, - release_live_session=release_live_session, + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_closed_retiring", None), + headers={"x-codex-turn-state": "http_turn_closed_retiring"}, + affinity=proxy_service._AffinityPolicy( + key="http_turn_closed_retiring", + kind=proxy_service.StickySessionKind.CODEX_SESSION, ), + request_model="gpt-5.5", + account=cast(Any, SimpleNamespace(id="acc-retiring", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=send_text, close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + closed=True, ) - monkeypatch.setattr( - service, - "_http_bridge_precreated_retry_decision", - AsyncMock(return_value=http_bridge_retry_circuit_module._HTTPBridgeRetryCircuitDecision(allowed=True)), - ) - - async def retry_on_closed_replacement( - _session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, - send_request: bool, - require_same_account: bool, - ) -> bool: - del text_data, send_request, require_same_account - request_state.replay_count += 1 - raise closed_error - - retry = AsyncMock(side_effect=retry_on_closed_replacement) - monkeypatch.setattr(service, "_retry_http_bridge_request_on_fresh_upstream", retry) + service._http_bridge_sessions[session.key] = session - with pytest.raises(ProxyResponseError) as exc_info: - await service._submit_http_bridge_request( - session, - request_state=request_state, - text_data=request_state.request_text or "{}", - queue_limit=8, - recovery_turn_state="turn-pre-dispatch-recovery-closed-twice", + async with session.lifecycle_lock: + submit_task = asyncio.create_task( + service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + ) ) + try: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(retry_started.wait(), timeout=0.01) + service._http_bridge_sessions.pop(session.key, None) + finally: + if submit_task.done(): + await submit_task + + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await submit_task assert exc_info.value.status_code == 502 - assert exc_info.value.payload["error"]["code"] == UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE - closed_send.assert_awaited_once() - retry.assert_awaited_once() - register_recovery_turn_state.assert_awaited_once() - rollback_registration.assert_awaited_once_with(receipt=receipt) - assert request_state.replay_count == 1 - assert request_state.recovery_attempt_dispatched is False - assert session.closed is True - assert session.upstream_control.reconnect_requested is True - assert session.upstream_control.retire_after_drain is True + assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" + assert retry_started.is_set() is False + send_text.assert_not_awaited() @pytest.mark.asyncio -async def test_submit_http_bridge_request_cancellation_during_replacement_setup_rolls_back_alias( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_submit_http_bridge_request_does_not_send_after_retirement_between_validation_and_send() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - closed_send = AsyncMock( - side_effect=UpstreamWebSocketTransportError( - "already closed", - error_code=UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - ) - ) - replacement_setup_started = asyncio.Event() - replacement_setup_release = asyncio.Event() request_state = proxy_service._WebSocketRequestState( - request_id="req-pre-dispatch-recovery-cancelled-setup", + request_id="req-retired-after-submit-validation", model="gpt-5.5", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), + started_at=1.0, awaiting_response_created=True, event_queue=asyncio.Queue(), request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', transport="http", skip_request_log=True, ) - key = _make_account_neutral_replay_session_key("pre-dispatch-recovery-cancelled-setup") - close = AsyncMock() - session = _make_bridge_session(key=key) - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace(send_text=closed_send, close=close), - ) - session.durable_session_id = "durable-pre-dispatch-recovery-cancelled-setup" - session.durable_owner_epoch = 11 - service._http_bridge_sessions[session.key] = session - receipt = DurableBridgeAliasRegistrationReceipt( - status=DurableBridgeAliasRegistration.REGISTERED, - session_id=session.durable_session_id, - api_key_scope="__anonymous__", - alias_kind="turn_state", - alias_value="turn-pre-dispatch-recovery-cancelled-setup", - instance_id="test-instance", - owner_epoch=session.durable_owner_epoch, - previous_alias_session_id="durable-predecessor", - previous_alias_owner_epoch=4, - previous_alias_account_id="acc-predecessor", - previous_latest_turn_state=None, - ) - register_recovery_turn_state = AsyncMock(return_value=receipt) - rollback_registration = AsyncMock(return_value=True) - release_live_session = AsyncMock(return_value=None) - service._durable_bridge = cast( - Any, - SimpleNamespace( - register_recovery_turn_state=register_recovery_turn_state, - rollback_recovery_turn_state_registration=rollback_registration, - release_live_session=release_live_session, + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_retire_gap", None), + headers={"x-codex-turn-state": "http_turn_retire_gap"}, + affinity=proxy_service._AffinityPolicy( + key="http_turn_retire_gap", + kind=proxy_service.StickySessionKind.CODEX_SESSION, ), + request_model="gpt-5.5", + account=cast(Any, SimpleNamespace(id="acc-retire-gap", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=AsyncMock(), close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, ) - monkeypatch.setattr( - service, - "_http_bridge_precreated_retry_decision", - AsyncMock(return_value=http_bridge_retry_circuit_module._HTTPBridgeRetryCircuitDecision(allowed=True)), - ) + service._http_bridge_sessions[session.key] = session + stale_send_seen = False - async def wait_during_replacement_setup( - _session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - restart_reader: bool, - require_same_account: bool, - ) -> None: - del _session, request_state, restart_reader, require_same_account - replacement_setup_started.set() - await replacement_setup_release.wait() + async def send_text(_text: str) -> None: + nonlocal stale_send_seen + stale_send_seen = service._http_bridge_sessions.get(session.key) is not session or session.closed - reconnect = AsyncMock(side_effect=wait_during_replacement_setup) - monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + cast(Any, session.upstream).send_text.side_effect = send_text - submit = asyncio.create_task( - service._submit_http_bridge_request( + class RetireAfterValidationLock: + async def __aenter__(self) -> None: + return None + + async def __aexit__(self, *_exc: object) -> None: + if request_state.response_create_gate_acquired and request_state not in session.pending_requests: + session.closed = True + service._http_bridge_sessions.pop(session.key, None) + return None + + service._http_bridge_lock = cast(Any, RetireAfterValidationLock()) + + try: + await service._submit_http_bridge_request( session, request_state=request_state, text_data=request_state.request_text or "{}", queue_limit=8, - recovery_turn_state="turn-pre-dispatch-recovery-cancelled-setup", ) - ) - try: - await asyncio.wait_for(replacement_setup_started.wait(), timeout=1.0) - submit.cancel() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(submit, timeout=1.0) + except proxy_service.ProxyResponseError: + pass finally: - replacement_setup_release.set() - if not submit.done(): - submit.cancel() - await asyncio.gather(submit, return_exceptions=True) + if request_state.response_create_gate_acquired: + await proxy_service._release_websocket_response_create_gate(request_state, session.response_create_gate) - closed_send.assert_awaited_once() - reconnect.assert_awaited_once() - register_recovery_turn_state.assert_awaited_once() - rollback_registration.assert_awaited_once_with(receipt=receipt) - assert request_state.replay_count == 1 - assert request_state.fresh_upstream_send_primitive_reached is False - assert request_state.recovery_attempt_dispatched is False - assert session.closed is True - assert session.upstream_control.reconnect_requested is True - assert session.upstream_control.retire_after_drain is True - assert session.queued_request_count == 0 - assert session.pending_requests == deque() + assert stale_send_seen is False @pytest.mark.asyncio -async def test_submit_http_bridge_request_setup_failure_before_replacement_send_rolls_back_alias( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_submit_http_bridge_request_rejects_state_after_response_event() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - closed_send = AsyncMock( - side_effect=UpstreamWebSocketTransportError( - "already closed", - error_code=UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - ) - ) + send_text = AsyncMock() request_state = proxy_service._WebSocketRequestState( - request_id="req-pre-dispatch-recovery-setup-failed", + request_id="req-visible-submit", model="gpt-5.5", service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, + started_at=1.0, + response_id="resp_visible_submit", + response_event_count=1, + awaiting_response_created=False, event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', + request_text='{"type":"response.create","model":"gpt-5.5","input":"visible"}', transport="http", skip_request_log=True, ) - key = _make_account_neutral_replay_session_key("pre-dispatch-recovery-setup-failed") - session = _make_bridge_session(key=key) - session.upstream = cast( - UpstreamWebSocket, - SimpleNamespace(send_text=closed_send, close=AsyncMock()), - ) - session.durable_session_id = "durable-pre-dispatch-recovery-setup-failed" - session.durable_owner_epoch = 13 - service._http_bridge_sessions[session.key] = session - receipt = DurableBridgeAliasRegistrationReceipt( - status=DurableBridgeAliasRegistration.REGISTERED, - session_id=session.durable_session_id, - api_key_scope="__anonymous__", - alias_kind="turn_state", - alias_value="turn-pre-dispatch-recovery-setup-failed", - instance_id="test-instance", - owner_epoch=session.durable_owner_epoch, - previous_alias_session_id="durable-predecessor", - previous_alias_owner_epoch=5, - previous_alias_account_id="acc-predecessor", - previous_latest_turn_state=None, - ) - register_recovery_turn_state = AsyncMock(return_value=receipt) - rollback_registration = AsyncMock(return_value=True) - service._durable_bridge = cast( - Any, - SimpleNamespace( - register_recovery_turn_state=register_recovery_turn_state, - rollback_recovery_turn_state_registration=rollback_registration, - release_live_session=AsyncMock(return_value=None), + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_visible_submit", None), + headers={"x-codex-turn-state": "http_turn_visible_submit"}, + affinity=proxy_service._AffinityPolicy( + key="http_turn_visible_submit", + kind=proxy_service.StickySessionKind.CODEX_SESSION, ), + request_model="gpt-5.5", + account=cast(Any, SimpleNamespace(id="acc-visible-submit", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=send_text, close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, ) - monkeypatch.setattr( - service, - "_http_bridge_precreated_retry_decision", - AsyncMock(return_value=http_bridge_retry_circuit_module._HTTPBridgeRetryCircuitDecision(allowed=True)), - ) - reconnect = AsyncMock(side_effect=RuntimeError("replacement setup failed")) - monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) - with pytest.raises(ProxyResponseError) as exc_info: + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: await service._submit_http_bridge_request( session, request_state=request_state, text_data=request_state.request_text or "{}", queue_limit=8, - recovery_turn_state="turn-pre-dispatch-recovery-setup-failed", ) assert exc_info.value.status_code == 502 - assert exc_info.value.payload["error"]["code"] == UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE - reconnect.assert_awaited_once() - register_recovery_turn_state.assert_awaited_once() - rollback_registration.assert_awaited_once_with(receipt=receipt) + assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" + assert session.pending_requests == deque() + assert session.queued_request_count == 0 + send_text.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_retry_http_bridge_request_on_fresh_upstream_reconnects_without_resending_previous_response_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + send_text = AsyncMock() + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), + headers={"x-codex-session-id": "sid-123"}, + affinity=proxy_service._AffinityPolicy( + key="sid-123", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=send_text, close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-1", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + previous_response_id="resp_prev_1", + transport="http", + error_code_override="upstream_unavailable", + error_message_override="Proxy request budget exhausted", + error_http_status_override=502, + ) + reconnect = AsyncMock() + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + + recovered = await service._retry_http_bridge_request_on_fresh_upstream( + session=session, + request_state=request_state, + text_data='{"type":"response.create","previous_response_id":"resp_prev_1"}', + send_request=False, + ) + + assert recovered is True assert request_state.replay_count == 1 - assert request_state.fresh_upstream_send_primitive_reached is False - assert request_state.recovery_attempt_dispatched is False - assert session.closed is True - assert session.upstream_control.reconnect_requested is True - assert session.upstream_control.retire_after_drain is True + assert request_state.error_code_override is None + assert request_state.error_message_override is None + assert request_state.error_http_status_override is None + reconnect.assert_awaited_once_with( + session, + request_state=request_state, + restart_reader=True, + require_same_account=False, + require_preferred_account=False, + ) + send_text.assert_not_awaited() @pytest.mark.asyncio -async def test_retry_http_bridge_request_on_fresh_upstream_reconnects_without_resending_previous_response_id( +async def test_retry_http_bridge_request_on_fresh_upstream_requires_file_pin_owner( monkeypatch: pytest.MonkeyPatch, ) -> None: + # given service = proxy_service.ProxyService(cast(Any, nullcontext())) - send_text = AsyncMock() session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), - headers={"x-codex-session-id": "sid-123"}, + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-file", None), + headers={}, affinity=proxy_service._AffinityPolicy( - key="sid-123", - kind=proxy_service.StickySessionKind.CODEX_SESSION, + key="sid-file", + kind=proxy_service.StickySessionKind.PROMPT_CACHE, ), request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id="acc-1", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=send_text, close=AsyncMock())), + account=cast(Any, SimpleNamespace(id="acc-file", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=AsyncMock(), close=AsyncMock())), upstream_control=proxy_service._WebSocketUpstreamControl(), pending_requests=deque(), pending_lock=anyio.Lock(), @@ -20050,41 +22627,160 @@ async def test_retry_http_bridge_request_on_fresh_upstream_reconnects_without_re idle_ttl_seconds=120.0, ) request_state = proxy_service._WebSocketRequestState( - request_id="req-1", + request_id="req-file", model="gpt-5.4", service_tier=None, reasoning_effort=None, api_key_reservation=None, started_at=1.0, - previous_response_id="resp_prev_1", + preferred_account_id="acc-file", + file_required_preferred_account=True, transport="http", - error_code_override="upstream_unavailable", - error_message_override="Proxy request budget exhausted", - error_http_status_override=502, ) reconnect = AsyncMock() monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + # when recovered = await service._retry_http_bridge_request_on_fresh_upstream( session=session, request_state=request_state, - text_data='{"type":"response.create","previous_response_id":"resp_prev_1"}', + text_data='{"type":"response.create"}', send_request=False, ) + # then assert recovered is True - assert request_state.replay_count == 1 - assert request_state.fresh_upstream_send_primitive_reached is False - assert request_state.error_code_override is None - assert request_state.error_message_override is None - assert request_state.error_http_status_override is None reconnect.assert_awaited_once_with( session, request_state=request_state, restart_reader=True, - require_same_account=True, + require_same_account=False, + require_preferred_account=True, ) - send_text.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_retry_http_bridge_request_on_fresh_upstream_propagates_file_owner_unavailable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-file-unavail", None), + headers={}, + affinity=proxy_service._AffinityPolicy( + key="sid-file-unavail", + kind=proxy_service.StickySessionKind.PROMPT_CACHE, + ), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-file", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=AsyncMock(), close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-file-unavail", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + preferred_account_id="acc-file", + file_required_preferred_account=True, + transport="http", + ) + owner_unavailable = http_bridge_helpers_module._http_bridge_previous_response_owner_unavailable_error() + monkeypatch.setattr(service, "_reconnect_http_bridge_session", AsyncMock(side_effect=owner_unavailable)) + + # when / then + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._retry_http_bridge_request_on_fresh_upstream( + session=session, + request_state=request_state, + text_data='{"type":"response.create"}', + send_request=False, + ) + + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + + +@pytest.mark.asyncio +async def test_submit_http_bridge_request_emits_owner_unavailable_when_file_pin_reconnect_connect_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-submit-file-1011", None), + key_value="sid-submit-file-1011", + ) + session.closed = True + session.last_upstream_close_code = 1011 + request_state = proxy_service._WebSocketRequestState( + request_id="req-submit-file-1011", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.4","input":"file"}', + transport="http", + preferred_account_id="acc-bridge", + file_required_preferred_account=True, + skip_request_log=True, + ) + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + async def open_upstream(*_args: object, **_kwargs: object) -> Any: + raise proxy_service.ProxyResponseError( + 503, + proxy_service.openai_error("upstream_proxy_unavailable", "Upstream proxy unavailable"), + ) + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", open_upstream) + service._http_bridge_sessions[session.key] = session + + # when + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + ) + + # then + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert exc_info.value.payload["error"]["type"] == "server_error" @pytest.mark.asyncio @@ -20611,6 +23307,82 @@ async def test_process_http_bridge_upstream_text_marks_text_delta_downstream_vis assert forwarded_payload["delta"] == "I started" +@pytest.mark.asyncio +async def test_http_bridge_recovery_releases_origin_after_terminal_event_following_created( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-recovery-origin-terminal", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.4","input":"hello"}', + transport="http", + skip_request_log=True, + recovery_attempt_fingerprint="recovery-origin-terminal-fingerprint", + recovery_attempt_session_id="durable-recovery-origin", + recovery_attempt_owner_epoch=4, + ) + session = _make_bridge_session( + key_value="recovery-origin-terminal", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.durable_session_id = "durable-replacement" + session.durable_owner_epoch = 9 + mark_replayed = AsyncMock(return_value=True) + release_origin = AsyncMock() + service._durable_bridge = cast( + Any, + SimpleNamespace( + mark_recovery_attempt_replayed=mark_replayed, + release_live_session=release_origin, + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings(http_responses_session_bridge_instance_id="replacement-instance"), + ) + + await service._process_http_bridge_upstream_text( + session, + json.dumps( + {"type": "response.created", "response": {"id": "resp-recovery-origin", "status": "in_progress"}}, + separators=(",", ":"), + ), + ) + release_origin.assert_not_awaited() + + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "response.incomplete", + "response": { + "id": "resp-recovery-origin", + "status": "incomplete", + "incomplete_details": {"reason": "max_output_tokens"}, + }, + }, + separators=(",", ":"), + ), + ) + + assert mark_replayed.await_count == 2 + release_origin.assert_awaited_once_with( + session_id="durable-recovery-origin", + instance_id="replacement-instance", + owner_epoch=4, + draining=False, + ) + + @pytest.mark.asyncio async def test_retry_http_bridge_request_on_fresh_upstream_refuses_to_resend_previous_response_id( monkeypatch: pytest.MonkeyPatch, @@ -20662,19 +23434,8 @@ async def test_retry_http_bridge_request_on_fresh_upstream_refuses_to_resend_pre @pytest.mark.asyncio -@pytest.mark.parametrize( - ("key_kind", "account_neutral", "expected_require_same_account"), - [ - ("session_header", True, True), - ("prompt_cache", True, True), - ("prompt_cache", False, True), - ], -) async def test_retry_http_bridge_request_on_fresh_upstream_replays_retry_safe_injection_without_anchor( monkeypatch: pytest.MonkeyPatch, - key_kind: str, - account_neutral: bool, - expected_require_same_account: bool, ) -> None: """Durable-anchor injections opt in to fresh-turn replay on send failure. @@ -20687,7 +23448,7 @@ async def test_retry_http_bridge_request_on_fresh_upstream_replays_retry_safe_in service = proxy_service.ProxyService(cast(Any, nullcontext())) send_text = AsyncMock() session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey(key_kind, "sid-safe", None), + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-safe", None), headers={"x-codex-session-id": "sid-safe"}, affinity=proxy_service._AffinityPolicy( key="sid-safe", @@ -20718,24 +23479,19 @@ async def test_retry_http_bridge_request_on_fresh_upstream_replays_retry_safe_in '"client_metadata":{"x-codex-installation-id":"installation-a"}}' ), fresh_upstream_request_is_retry_safe=True, - fresh_upstream_request_is_account_neutral=account_neutral, transport="http", ) async def reconnect(*args: object, **kwargs: object) -> None: - assert args == (session,) - assert kwargs["request_state"] is request_state - assert kwargs["restart_reader"] is True - assert kwargs["require_same_account"] is expected_require_same_account - if not expected_require_same_account: - session.account = cast( - Any, - SimpleNamespace( - id="acc-2", - status=AccountStatus.ACTIVE, - codex_installation_id="installation-b", - ), - ) + del args, kwargs + session.account = cast( + Any, + SimpleNamespace( + id="acc-2", + status=AccountStatus.ACTIVE, + codex_installation_id="installation-b", + ), + ) monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) @@ -20748,7 +23504,6 @@ async def reconnect(*args: object, **kwargs: object) -> None: assert recovered is True assert request_state.replay_count == 1 - assert request_state.fresh_upstream_send_primitive_reached is True # Replaying should have dropped the anchor metadata so the request # executes as a fresh turn using the captured unanchored payload. assert request_state.previous_response_id is None @@ -21311,7 +24066,9 @@ async def test_create_http_bridge_session_does_not_classify_post_selection_failu @pytest.mark.asyncio async def test_stream_via_http_bridge_fails_closed_before_file_affinity_when_previous_response_owner_misses( monkeypatch: pytest.MonkeyPatch, + db_setup: bool, ) -> None: + del db_setup service = proxy_service.ProxyService(cast(Any, nullcontext())) await service._pin_file_account("file_from_other_account", "acc-file") payload = proxy_service.ResponsesRequest.model_validate( @@ -21549,7 +24306,6 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe latest_response_id="resp_completed_anchor", latest_input_item_count=len(historical_input), latest_input_full_fingerprint=proxy_service._fingerprint_input_items(historical_input), - latest_pending_tool_calls={}, model=stored_model, ) owner_unavailable = ProxyResponseError( @@ -22512,6 +25268,119 @@ async def close(self) -> None: session=session, ) assert "http_bridge_event event=missing_response_created_timeout" in caplog.text + if leading_telemetry: + assert session.key not in service._http_bridge_retry_circuits + else: + assert service._http_bridge_retry_circuits[session.key].consecutive_failures == 1 + + +@pytest.mark.asyncio +async def test_http_bridge_stream_and_reader_count_one_eventless_send_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + upstream = _SilentEventlessUpstream() + session = _make_bridge_session(key_value="eventless-observer-race") + session.upstream = cast(UpstreamWebSocket, upstream) + service._http_bridge_sessions[session.key] = session + settings = _make_app_settings( + sse_keepalive_interval_seconds=0.02, + stream_idle_timeout_seconds=0.3, + http_responses_session_bridge_request_budget_seconds=1.0, + http_responses_session_bridge_stuck_gate_retire_after_seconds=0.005, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(http_bridge_streaming_module, "_stream_keepalive_max_count", lambda: 1) + monkeypatch.setattr(proxy_service, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.001) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) + monkeypatch.setattr(service, "_write_request_log", AsyncMock()) + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = cast( + Any, + SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=persist_retry_circuit, + ), + ) + + async def submit( + target_session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + ) -> None: + del queue_limit + gate = target_session.response_create_gate + await gate.acquire() + request_state.response_create_gate = gate + request_state.response_create_gate_acquired = True + request_state.awaiting_response_created = True + request_state.request_text = text_data + async with target_session.pending_lock: + target_session.pending_requests.append(request_state) + target_session.queued_request_count = 1 + await http_bridge_request_submit_module._send_http_bridge_request_text_with_archive_id( + target_session, + request_state, + text_data, + ) + + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + request_state = _make_eventless_http_bridge_owner(request_id="req-eventless-observer-race", sent_at=0.0) + request_state.started_at = time.monotonic() + request_state.response_create_sent_at = None + request_state.response_create_gate = None + request_state.response_create_gate_acquired = False + reader_retry_started = asyncio.Event() + release_reader_retry = asyncio.Event() + + async def retry_precreated( + _session: proxy_service._HTTPBridgeSession, + *, + restart_reader: bool = False, + ) -> bool: + if restart_reader: + await asyncio.wait_for(reader_retry_started.wait(), timeout=2.0) + return False + request_state.response_create_sent_at = None + reader_retry_started.set() + await asyncio.wait_for(release_reader_retry.wait(), timeout=2.0) + return False + + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) + + reader_task = asyncio.create_task(service._relay_http_bridge_upstream_messages(session)) + await asyncio.wait_for(upstream.first_receive_started.wait(), timeout=0.5) + stream = service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + queue_limit=8, + propagate_http_errors=False, + downstream_turn_state=None, + ) + try: + terminal_task = asyncio.create_task(anext(stream)) + await asyncio.wait_for(reader_retry_started.wait(), timeout=1.0) + terminal = await asyncio.wait_for(terminal_task, timeout=1.0) + assert '"code":"stream_idle_timeout"' in terminal + release_reader_retry.set() + await asyncio.wait_for(reader_task, timeout=1.0) + + state = cast(Any, service)._http_bridge_retry_circuits[session.key] + assert state.consecutive_failures == 1 + assert state.cooldown_until == 0.0 + assert request_state.response_create_attempt is not None + assert request_state.response_create_attempt.retry_circuit_failure_recorded is True + assert persist_retry_circuit.await_count == 1 + finally: + release_reader_retry.set() + if not reader_task.done(): + reader_task.cancel() + with pytest.raises(asyncio.CancelledError): + await reader_task + await stream.aclose() @pytest.mark.asyncio @@ -22804,10 +25673,9 @@ async def test_clear_durable_http_bridge_response_anchor_logs_only_on_confirmed_ service = SimpleNamespace(_durable_bridge=durable_bridge) caplog.set_level(logging.INFO, logger="app.modules.proxy.service") - result = await http_bridge_upstream_events_module._clear_durable_http_bridge_response_anchor(service, session) + await http_bridge_upstream_events_module._clear_durable_http_bridge_response_anchor(service, session) assert ("event=durable_anchor_invalidated" in caplog.text) is expect_logged - assert result is (lookup if expect_logged else None) @pytest.mark.asyncio @@ -23286,6 +26154,8 @@ async def test_http_bridge_liveness_timeout_is_neutral_not_replayed_and_forces_r session, detail=UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE, response_events_seen=0, + retired_request_count=1, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), ) assert session.queued_request_count == 0 assert session.closed is True @@ -23432,11 +26302,17 @@ async def controlled_fail_reader( assert session.queued_request_count == 0 assert session.closed is True assert session.liveness_settlement_owner == "send" - retire.assert_awaited_once_with( - session, - detail=UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE, - response_events_seen=0, - ) + retire.assert_awaited_once() + retire_call = retire.await_args + assert retire_call is not None + assert retire_call.args == (session,) + assert retire_call.kwargs["detail"] == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE + assert retire_call.kwargs["response_events_seen"] == 0 + assert retire_call.kwargs["retired_request_count"] == 2 + assert request_state.response_create_attempt is not None + retry_circuit_attempt_selection = retire_call.kwargs["retry_circuit_attempt_selection"] + assert retry_circuit_attempt_selection.attempt is request_state.response_create_attempt + assert request_state.response_create_attempt.disarmed is True @pytest.mark.asyncio @@ -23558,6 +26434,8 @@ def pending_sibling(request_id: str) -> proxy_service._WebSocketRequestState: session, detail=UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE, response_events_seen=0, + retired_request_count=2, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), ) @@ -23577,6 +26455,10 @@ async def test_http_bridge_retry_send_network_failure_is_neutral_and_not_replaye request_text='{"type":"response.create","model":"gpt-5.4","input":"hello"}', transport="http", ) + first_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + request_state.response_create_attempt_count = 1 + request_state.response_create_attempt = first_attempt + request_state.response_create_sent_at = time.monotonic() session = _make_bridge_session( key_value="bridge-retry-send-network", pending_requests=deque([request_state]), @@ -23615,15 +26497,22 @@ async def fail_reader( await service._relay_http_bridge_upstream_messages(session) - assert failure_calls == [ - { - "error_code": "proxy_network_unavailable", - "error_message": "Codex upstream websocket send failed: OSError", - "penalize_account": False, - } - ] + assert len(failure_calls) == 1 + failure_call = failure_calls[0] + assert failure_call["error_code"] == "proxy_network_unavailable" + assert failure_call["error_message"] == "Codex upstream websocket send failed: OSError" + assert failure_call["penalize_account"] is False + retry_circuit_attempt_selection = failure_call["retry_circuit_attempt_selection"] + assert isinstance( + retry_circuit_attempt_selection, + proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, + ) + assert retry_circuit_attempt_selection.attempt is first_attempt assert request_state.replay_count == 1 retry_send.assert_awaited_once() + assert request_state.response_create_attempt is not first_attempt + assert request_state.response_create_attempt is not None + assert request_state.response_create_attempt.disarmed is True assert session.closed is True @@ -23677,7 +26566,145 @@ async def test_http_bridge_clean_close_before_response_does_not_penalize_account assert fail_pending.await_args is not None assert fail_pending.await_args.kwargs["penalize_account"] is False - retire.assert_awaited_once_with(session, detail="stream_incomplete", response_events_seen=0) + retire.assert_awaited_once_with( + session, + detail="stream_incomplete", + response_events_seen=0, + retired_request_count=0, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), + ) + + +@pytest.mark.asyncio +async def test_http_bridge_clean_close_retry_failure_preserves_pre_recovery_attempt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = _make_eventless_http_bridge_owner(request_id="req-clean-close-retry-attempt") + original_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + replacement_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=2) + request_state.response_create_attempt = original_attempt + session = _make_bridge_session( + key_value="bridge-clean-close-retry-attempt", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock(return_value=UpstreamWebSocketMessage(kind="close", close_code=1000)), + close=AsyncMock(), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + + async def retry_precreated(target_session: Any, **_kwargs: object) -> bool: + assert target_session is session + request_state.response_create_attempt = replacement_attempt + request_state.response_create_sent_at = None + return False + + failure_calls: list[dict[str, object]] = [] + + async def fail_reader(target_session: Any, **kwargs: object) -> bool: + assert target_session is session + failure_calls.append(dict(kwargs)) + target_session.closed = True + return True + + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) + monkeypatch.setattr(service, "_fail_http_bridge_reader_and_maybe_retire", fail_reader) + + await service._relay_http_bridge_upstream_messages(session) + + assert len(failure_calls) == 1 + retry_circuit_attempt_selection = failure_calls[0]["retry_circuit_attempt_selection"] + assert isinstance( + retry_circuit_attempt_selection, + proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, + ) + assert retry_circuit_attempt_selection.attempt is original_attempt + + +@pytest.mark.asyncio +async def test_http_bridge_reader_exception_captures_attempt_before_lifecycle_wait( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = _make_eventless_http_bridge_owner(request_id="req-reader-exception-attempt") + request_state.started_at = time.monotonic() + request_state.response_create_sent_at = None + original_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + replacement_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=2) + request_state.response_create_attempt = original_attempt + session = _make_bridge_session( + key_value="bridge-reader-exception-attempt", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock( + return_value=UpstreamWebSocketMessage( + kind="text", + text='{"type":"response.created","response":{"id":"resp_reader_exception"}}', + ) + ), + close=AsyncMock(), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + service, + "_process_http_bridge_upstream_text", + AsyncMock(side_effect=RuntimeError("processing failed")), + ) + attempt_captured = asyncio.Event() + original_selector = ( + http_bridge_upstream_events_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests + ) + + def capture_attempt_before_lifecycle_wait( + request_states: tuple[proxy_service._WebSocketRequestState, ...], + ) -> proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection: + selection = original_selector(request_states) + attempt_captured.set() + return selection + + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_http_bridge_retry_circuit_attempt_selection_for_pending_requests", + capture_attempt_before_lifecycle_wait, + ) + failure_calls: list[dict[str, object]] = [] + + async def fail_reader( + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> bool: + assert target_session is session + failure_calls.append(dict(kwargs)) + target_session.closed = True + return True + + monkeypatch.setattr(service, "_fail_http_bridge_reader_and_maybe_retire", fail_reader) + + async with session.lifecycle_lock: + relay_task = asyncio.create_task(service._relay_http_bridge_upstream_messages(session)) + await asyncio.wait_for(attempt_captured.wait(), timeout=1.0) + request_state.response_create_attempt = replacement_attempt + + await asyncio.wait_for(relay_task, timeout=1.0) + + assert len(failure_calls) == 1 + retry_circuit_attempt_selection = failure_calls[0]["retry_circuit_attempt_selection"] + assert isinstance( + retry_circuit_attempt_selection, + proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, + ) + assert retry_circuit_attempt_selection.attempt is original_attempt + assert replacement_attempt.retry_circuit_failure_recorded is False @pytest.mark.asyncio @@ -23728,170 +26755,334 @@ async def fail_reader( assert session.last_upstream_close_code == (None if routed else 1011) assert len(failure_calls) == 1 assert failure_calls[0]["error_code"] == "stream_incomplete" - assert failure_calls[0]["penalize_account"] is True assert failure_calls[0]["response_events_seen"] == 0 if routed: + # Intentional flip for issue #1754: an abrupt drop with no close frame + # and zero response events must stay account-neutral instead of + # feeding error backoff and stranding continuity-bound follow-ups. + assert failure_calls[0]["penalize_account"] is False + assert failure_calls[0]["account_neutral_transport_drop"] is True assert failure_calls[0]["upstream_close_code"] is None assert failure_calls[0]["transport_classification"] == "websocket_transport_error" else: + # A close frame — even a non-clean 1011 — is upstream-authored + # evidence and keeps the existing account penalty. + assert failure_calls[0]["penalize_account"] is True + assert failure_calls[0]["account_neutral_transport_drop"] is False assert failure_calls[0]["upstream_close_code"] == 1011 assert failure_calls[0]["transport_classification"] == "websocket_close_transient" @pytest.mark.asyncio -async def test_http_bridge_idle_generic_receive_error_retires_without_retry_or_circuit_and_cleans_aliases( +async def test_http_bridge_abrupt_eventless_drop_stays_account_neutral_and_records_drop_signal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Issue #1754: an abrupt upstream drop with no close frame and zero + response events must not feed per-drop account error backoff (which 502s + continuity-bound follow-ups), but repeated eventless drops must still feed + the windowed account drain signal so genuine account faults surface.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-abrupt-drop", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + ) + session = _make_bridge_session( + key_value="bridge-abrupt-drop", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock( + return_value=UpstreamWebSocketMessage( + kind="error", + error="Upstream websocket closed before response.completed: no close frame received or sent", + ) + ), + close=AsyncMock(), + ), + ) + fail_pending = AsyncMock(return_value=True) + retire = AsyncMock() + drop_signals: list[dict[str, object]] = [] + + async def record_signal( + target_service: object, + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> None: + assert target_session is session + drop_signals.append(dict(kwargs)) + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_record_http_bridge_account_timeout_signal", + record_signal, + ) + + await service._relay_http_bridge_upstream_messages(session) + + assert fail_pending.await_args is not None + assert fail_pending.await_args.kwargs["error_code"] == "stream_incomplete" + assert fail_pending.await_args.kwargs["penalize_account"] is False + assert drop_signals == [{"detail": "eventless_transport_drop"}] + retire.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_http_bridge_abrupt_drop_after_response_events_still_penalizes_account( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A drop after the upstream already streamed response events remains + account-attributable: only the eventless no-close-frame case is neutral.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-mid-stream-drop", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + ) + request_state.response_event_count = 8 + session = _make_bridge_session( + key_value="bridge-mid-stream-drop", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock(return_value=UpstreamWebSocketMessage(kind="error", error="upstream reset")), + close=AsyncMock(), + ), + ) + fail_pending = AsyncMock(return_value=True) + retire = AsyncMock() + drop_signals: list[dict[str, object]] = [] + + async def record_signal( + target_service: object, + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> None: + drop_signals.append(dict(kwargs)) + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_record_http_bridge_account_timeout_signal", + record_signal, + ) + + await service._relay_http_bridge_upstream_messages(session) + + assert fail_pending.await_args is not None + assert fail_pending.await_args.kwargs["penalize_account"] is True + assert drop_signals == [] + + +@pytest.mark.asyncio +async def test_http_bridge_non_clean_close_frame_before_response_still_penalizes_account( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An upstream-authored close frame (e.g. policy 1008) with zero response + events keeps the existing account penalty; only the frame-less drop is + account-neutral.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-policy-close") + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock(return_value=UpstreamWebSocketMessage(kind="close", close_code=1008)), + close=AsyncMock(), + ), + ) + fail_pending = AsyncMock(return_value=True) + retire = AsyncMock() + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + + await service._relay_http_bridge_upstream_messages(session) + + assert fail_pending.await_args is not None + assert fail_pending.await_args.kwargs["penalize_account"] is True + + +@pytest.mark.asyncio +async def test_http_bridge_synthetic_1006_close_stays_account_neutral( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """aiohttp reports an abnormal socket loss as CLOSED with the synthesized + reserved code 1006 (never sent in an actual close frame): it is the same + frame-less eventless drop and must stay account-neutral (issue #1754).""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-1006-drop", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + ) + session = _make_bridge_session( + key_value="bridge-1006-drop", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock(return_value=UpstreamWebSocketMessage(kind="close", close_code=1006)), + close=AsyncMock(), + ), + ) + fail_pending = AsyncMock(return_value=True) + retire = AsyncMock() + drop_signals: list[dict[str, object]] = [] + + async def record_signal( + target_service: object, + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> None: + drop_signals.append(dict(kwargs)) + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_record_http_bridge_account_timeout_signal", + record_signal, + ) + + await service._relay_http_bridge_upstream_messages(session) + + assert fail_pending.await_args is not None + assert fail_pending.await_args.kwargs["penalize_account"] is False + assert drop_signals == [{"detail": "eventless_transport_drop"}] + + +@pytest.mark.asyncio +async def test_http_bridge_abrupt_drop_after_buffered_reasoning_prelude_still_penalizes_account( monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, ) -> None: + """A buffered reasoning prelude is deliberately excluded from + response_event_count, but it is application-layer output: a following + frame-less drop is not eventless and keeps the account penalty.""" service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="bridge-idle-generic-receive-error") - session.last_used_at = time.monotonic() - 65.0 - session.previous_response_ids.add("resp-idle-generic-receive-error") - session.downstream_turn_state_aliases.add("turn-idle-generic-receive-error") - service._http_bridge_sessions[session.key] = session - service._http_bridge_previous_response_index[("resp-idle-generic-receive-error", None)] = session.key - service._http_bridge_turn_state_index[("turn-idle-generic-receive-error", None)] = session.key + request_state = proxy_service._WebSocketRequestState( + request_id="req-reasoning-prelude-drop", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + ) + request_state.upstream_model_output_seen = True + assert request_state.response_event_count == 0 + session = _make_bridge_session( + key_value="bridge-reasoning-prelude-drop", + pending_requests=deque([request_state]), + queued_request_count=1, + ) session.upstream = cast( UpstreamWebSocket, SimpleNamespace( - receive=AsyncMock( - side_effect=[ - UpstreamWebSocketMessage( - kind="error", - error="generic receive failure with no stable transport code", - error_code=None, - ), - UpstreamWebSocketMessage( - kind="error", - error="generic receive failure with no stable transport code", - error_code=None, - ), - ] - ), + receive=AsyncMock(return_value=UpstreamWebSocketMessage(kind="error", error="upstream reset")), close=AsyncMock(), ), ) - retry_precreated = AsyncMock(side_effect=[True, False]) - record_retry_circuit_failure = AsyncMock() - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_retry_circuit_failure) - - with caplog.at_level(logging.INFO): - await service._relay_http_bridge_upstream_messages(session) - - retry_precreated.assert_not_awaited() - record_retry_circuit_failure.assert_not_awaited() - assert session.closed is True - assert session.key not in service._http_bridge_sessions - assert ("resp-idle-generic-receive-error", None) not in service._http_bridge_previous_response_index - assert ("turn-idle-generic-receive-error", None) not in service._http_bridge_turn_state_index - assert "event=idle_transport_retire" in caplog.text - assert "pending=0" in caplog.text - assert "admission_waiters=0" in caplog.text - assert "idle_age_bucket=60s_to_5m" in caplog.text - assert "retry_action=not_attempted" in caplog.text - assert "circuit_action=not_recorded" in caplog.text - assert "event=reader_failure" not in caplog.text - assert "detail=stream_incomplete" not in caplog.text + fail_pending = AsyncMock(return_value=True) + retire = AsyncMock() + drop_signals: list[dict[str, object]] = [] + async def record_signal( + target_service: object, + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> None: + drop_signals.append(dict(kwargs)) -@pytest.mark.asyncio -async def test_http_bridge_idle_transport_classification_falls_back_when_admission_waiter_arrives( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session(key_value="bridge-idle-classification-race") - session.admission_waiter_count = 1 - fail_pending = AsyncMock() + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_record_http_bridge_account_timeout_signal", + record_signal, + ) - with caplog.at_level(logging.INFO): - retired = await service._fail_http_bridge_reader_and_maybe_retire( - session, - error_code="stream_incomplete", - error_message="Upstream websocket receive failed before response.completed", - penalize_account=True, - idle_transport_retire=True, - ) + await service._relay_http_bridge_upstream_messages(session) - assert retired is False - fail_pending.assert_awaited_once() - assert "event=reader_failure" in caplog.text - assert "event=idle_transport_retire" not in caplog.text + assert fail_pending.await_args is not None + assert fail_pending.await_args.kwargs["penalize_account"] is True + assert drop_signals == [] @pytest.mark.asyncio -async def test_http_bridge_active_generic_receive_records_preclear_hard_key_failure_and_second_opens_circuit( +async def test_http_bridge_protocol_invalid_binary_frame_still_penalizes_account( monkeypatch: pytest.MonkeyPatch, ) -> None: + """A non-terminal protocol-invalid frame (binary payload) also carries no + close code, but the socket did not end: it must keep the existing account + penalty instead of being classified as an abrupt transport drop.""" service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = proxy_service._HTTPBridgeSessionKey( - "session_header", - "bridge-active-generic-circuit", - None, + session = _make_bridge_session(key_value="bridge-binary-frame") + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock(return_value=UpstreamWebSocketMessage(kind="binary", data=b"\x00\x01")), + close=AsyncMock(), + ), ) - monkeypatch.setattr(service, "_load_http_bridge_retry_circuit", AsyncMock(return_value=True)) - monkeypatch.setattr(service, "_persist_http_bridge_retry_circuit", AsyncMock()) - record_failure = AsyncMock(wraps=service._record_http_bridge_retry_circuit_failure) - monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) + fail_pending = AsyncMock(return_value=True) + retire = AsyncMock() + drop_signals: list[dict[str, object]] = [] - async def clear_pending_requests(**kwargs: Any) -> None: - async with kwargs["pending_lock"]: - kwargs["pending_requests"].clear() + async def record_signal( + target_service: object, + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> None: + drop_signals.append(dict(kwargs)) - monkeypatch.setattr(service, "_fail_pending_websocket_requests", clear_pending_requests) - retire = AsyncMock() + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_record_http_bridge_account_timeout_signal", + record_signal, + ) - sessions: list[proxy_service._HTTPBridgeSession] = [] - snapshots: list[Any] = [] - for failure_number in (1, 2): - session = _make_bridge_session( - key=key, - pending_requests=deque( - [ - _make_eventless_http_bridge_owner( - request_id=f"req-active-generic-circuit-{failure_number}", - sent_at=time.monotonic(), - ) - ] - ), - queued_request_count=1, - ) - sessions.append(session) + await service._relay_http_bridge_upstream_messages(session) - retired = await service._fail_http_bridge_reader_and_maybe_retire( - session, - error_code="stream_incomplete", - error_message="Upstream websocket receive failed before response.completed", - penalize_account=True, - retry_action="suppressed_ambiguous_accept", - circuit_action="record_stream_incomplete", - ) - - assert retired is True - snapshots.append(await service._http_bridge_retry_circuit_snapshot(session)) - - assert record_failure.await_count == 2 - for failure_call, session in zip(record_failure.await_args_list, sessions, strict=True): - assert failure_call.args == (session,) - assert failure_call.kwargs == {"detail": "stream_incomplete"} - assert retire.await_count == 2 - for retire_call, session in zip(retire.await_args_list, sessions, strict=True): - assert retire_call.args == (session,) - assert retire_call.kwargs == { - "detail": "stream_incomplete", - "response_events_seen": 0, - "retry_circuit_already_recorded": True, - } - assert snapshots[0].allowed is True - assert snapshots[0].consecutive_failures == 1 - assert snapshots[0].retry_after_seconds == 0 - assert snapshots[1].allowed is False - assert snapshots[1].consecutive_failures == 2 - assert snapshots[1].retry_after_seconds > 0 + assert fail_pending.await_args is not None + assert fail_pending.await_args.kwargs["penalize_account"] is True + assert drop_signals == [] @pytest.mark.asyncio @@ -24006,118 +27197,137 @@ async def test_retire_stale_pending_http_bridge_session_unregisters_aliases_and_ @pytest.mark.asyncio -async def test_http_bridge_repeated_zero_event_stream_incompletes_poison_anchor_with_waiter( +async def test_stale_retirement_tracks_detached_generation_until_bounded_close_finishes( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - session = _make_bridge_session( - key_value="bridge-anchor-poison-stream-incomplete", - pending_requests=deque([_make_eventless_http_bridge_owner()]), - queued_request_count=1, - ) - session.admission_waiter_count = 1 - session.durable_session_id = "durable-anchor-poison-stream-incomplete" - session.durable_owner_epoch = 3 - durable_bridge = SimpleNamespace( - lookup_retry_circuit=AsyncMock(return_value=None), - persist_retry_circuit=AsyncMock(), - rebind_session_account=AsyncMock(return_value=True), - ) - service._durable_bridge = cast(Any, durable_bridge) - monkeypatch.setattr(service, "_fail_pending_websocket_requests", AsyncMock()) - retire = AsyncMock() - monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + session = _make_bridge_session(key_value="stale-retirement-detached") + close_started = asyncio.Event() + release_close = asyncio.Event() - for failure_number in range(1, 8): - retired = await service._fail_http_bridge_reader_and_maybe_retire( + async def slow_close() -> None: + close_started.set() + await release_close.wait() + + session.upstream = cast(UpstreamWebSocket, SimpleNamespace(close=slow_close)) + service._http_bridge_sessions[session.key] = session + monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) + + retirement_task = asyncio.create_task( + service._retire_stale_pending_http_bridge_session( session, - error_code="stream_incomplete", - error_message="Upstream websocket closed before response.completed", + detail="stream_incomplete", + response_events_seen=0, ) - assert retired is (failure_number == 7) - - durable_bridge.rebind_session_account.assert_awaited_once_with( - session_id="durable-anchor-poison-stream-incomplete", - api_key_id=None, - instance_id=proxy_service.get_settings().http_responses_session_bridge_instance_id, - owner_epoch=3, - account_id="acc-bridge", - clear_continuity=True, ) - retire.assert_awaited_once_with( + try: + await asyncio.wait_for(close_started.wait(), timeout=1.0) + assert service._http_bridge_sessions == {} + assert service._http_bridge_detached_sessions[id(session)] is session + assert http_bridge_helpers_module._http_bridge_capacity_generation_count(service) == 1 + finally: + release_close.set() + + await asyncio.wait_for(retirement_task, timeout=1.0) + + assert service._http_bridge_detached_sessions == {} + + +@pytest.mark.asyncio +async def test_http_bridge_idle_retirement_does_not_record_retry_circuit_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-idle-retire") + record_failure = AsyncMock() + close = AsyncMock() + monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", close) + + await service._retire_stale_pending_http_bridge_session( session, - detail="repeated_zero_event_stream_incomplete", + detail="stream_incomplete", response_events_seen=0, ) + record_failure.assert_not_awaited() + close.assert_awaited_once_with(session, reason="retire_stale_pending") + @pytest.mark.asyncio -async def test_http_bridge_retire_stale_pending_poisons_already_recorded_eventless_failure( +async def test_http_bridge_eventless_pending_retirement_records_one_retry_circuit_failure( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - durable_bridge = SimpleNamespace( - lookup_retry_circuit=AsyncMock(return_value=None), - persist_retry_circuit=AsyncMock(), - rebind_session_account=AsyncMock(return_value=True), + owner = _make_eventless_http_bridge_owner(request_id="req-eventless-retire") + session = _make_bridge_session( + key_value="bridge-eventless-retire", + pending_requests=deque([owner]), + queued_request_count=1, ) - service._durable_bridge = cast(Any, durable_bridge) + record_failure = AsyncMock(return_value=None) + monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) - for _ in range(7): - session = _make_bridge_session( - key_value="bridge-anchor-poison-retire", - pending_requests=deque([_make_eventless_http_bridge_owner()]), - queued_request_count=1, - ) - session.durable_session_id = "durable-anchor-poison-retire" - session.durable_owner_epoch = 5 - await service._record_http_bridge_retry_circuit_failure(session, detail="stream_incomplete") - await service._retire_stale_pending_http_bridge_session( - session, - detail="stream_incomplete", - retry_circuit_already_recorded=True, - response_events_seen=0, - ) + await service._retire_stale_pending_http_bridge_session( + session, + detail="missing_response_created_timeout", + response_events_seen=0, + ) - durable_bridge.rebind_session_account.assert_awaited_once_with( - session_id="durable-anchor-poison-retire", - api_key_id=None, - instance_id=proxy_service.get_settings().http_responses_session_bridge_instance_id, - owner_epoch=5, - account_id="acc-bridge", - clear_continuity=True, + record_failure.assert_awaited_once_with(session, detail="missing_response_created_timeout") + + +@pytest.mark.asyncio +async def test_http_bridge_direct_retirement_derives_observed_response_events( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + owner = _make_eventless_http_bridge_owner(request_id="req-eventful-direct-retire") + owner.response_event_count = 1 + session = _make_bridge_session( + key_value="bridge-eventful-direct-retire", + pending_requests=deque([owner]), + queued_request_count=1, + ) + record_failure = AsyncMock() + monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) + + await service._retire_stale_pending_http_bridge_session( + session, + detail="stuck_response_create_gate", ) + record_failure.assert_not_awaited() + @pytest.mark.asyncio -async def test_http_bridge_retire_stale_pending_clean_close_never_poisons_anchor( +async def test_http_bridge_reader_failure_preserves_pre_drain_request_for_retry_circuit( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - durable_bridge = SimpleNamespace( - lookup_retry_circuit=AsyncMock(return_value=None), - persist_retry_circuit=AsyncMock(), - rebind_session_account=AsyncMock(return_value=True), + owner = _make_eventless_http_bridge_owner(request_id="req-reader-failure-retire") + session = _make_bridge_session( + key_value="bridge-reader-failure-retire", + pending_requests=deque([owner]), + queued_request_count=1, ) - service._durable_bridge = cast(Any, durable_bridge) + record_failure = AsyncMock(return_value=None) + monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) - for _ in range(7): - session = _make_bridge_session( - key_value="bridge-anchor-clean-close", - pending_requests=deque([_make_eventless_http_bridge_owner()]), - queued_request_count=1, - ) - session.durable_session_id = "durable-anchor-clean-close" - session.durable_owner_epoch = 6 - await service._retire_stale_pending_http_bridge_session( - session, - detail="stream_incomplete", - retry_circuit_detail="clean_close", - ) + retired = await service._fail_http_bridge_reader_and_maybe_retire( + session, + error_code="stream_incomplete", + error_message="upstream closed before response.completed", + penalize_account=False, + response_events_seen=0, + ) - durable_bridge.rebind_session_account.assert_not_awaited() + assert retired is True + assert not session.pending_requests + record_failure.assert_awaited_once_with(session, detail="stream_incomplete") @pytest.mark.asyncio @@ -24174,65 +27384,15 @@ async def test_http_bridge_retry_circuit_allows_only_one_half_open_probe() -> No ) service._durable_bridge = SimpleNamespace(lookup_retry_circuit=AsyncMock(return_value=None)) - assert await service._http_bridge_precreated_retry_allowed(hard_session) is True - assert await service._http_bridge_precreated_retry_allowed(hard_session) is False - assert ( - await service._http_bridge_precreated_retry_allowed( - hard_session, - allow_proof_gated_continuity_replay=True, - ) - is True - ) - - -@pytest.mark.asyncio -async def test_http_bridge_half_open_probe_does_not_suppress_its_own_stream() -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - hard_session = _make_bridge_session(key_value="bridge-circuit-half-open-stream") - now = time.monotonic() - cast(Any, service)._http_bridge_retry_circuits[hard_session.key] = ( - http_bridge_retry_circuit_module._HTTPBridgeRetryCircuitState( - consecutive_failures=2, - cooldown_until=now - 1.0, - last_detail="stream_incomplete", - last_touched_monotonic=now, - ) - ) - service._durable_bridge = SimpleNamespace(lookup_retry_circuit=AsyncMock(return_value=None)) - - decision = await service._http_bridge_precreated_retry_decision(hard_session) - snapshot = await service._http_bridge_retry_circuit_snapshot(hard_session) - - assert decision.allowed is True - assert snapshot.allowed is True - assert snapshot.retry_after_seconds == 0 + assert await service._http_bridge_precreated_retry_allowed(hard_session) is True assert await service._http_bridge_precreated_retry_allowed(hard_session) is False - - -@pytest.mark.asyncio -@pytest.mark.parametrize("consecutive_failures", [0, 1]) -async def test_http_bridge_retry_circuit_ignores_stale_cooldown_below_open_threshold( - consecutive_failures: int, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - hard_session = _make_bridge_session(key_value=f"bridge-circuit-stale-{consecutive_failures}") - now = time.monotonic() - state = http_bridge_retry_circuit_module._HTTPBridgeRetryCircuitState( - consecutive_failures=consecutive_failures, - cooldown_until=now + 60.0, - last_detail="stream_incomplete" if consecutive_failures else None, - last_touched_monotonic=now, + assert ( + await service._http_bridge_precreated_retry_allowed( + hard_session, + allow_proof_gated_continuity_replay=True, + ) + is True ) - cast(Any, service)._http_bridge_retry_circuits[hard_session.key] = state - service._durable_bridge = SimpleNamespace(lookup_retry_circuit=AsyncMock(return_value=None)) - - decision = await service._http_bridge_precreated_retry_decision(hard_session) - snapshot = await service._http_bridge_retry_circuit_snapshot(hard_session) - - assert decision.allowed is True - assert snapshot.allowed is True - assert snapshot.retry_after_seconds == 0 - assert state.half_open_until == 0 @pytest.mark.asyncio @@ -24318,28 +27478,6 @@ async def test_http_bridge_clean_close_retry_circuit_is_bounded() -> None: assert retry_circuits[session.key].last_detail == "clean_close" -@pytest.mark.parametrize( - ("detail", "cause"), - [ - ("stream_incomplete", "repeated incomplete upstream WebSocket streams"), - ("clean_close", "repeated clean upstream WebSocket closes"), - ("stream_idle_timeout", "repeated upstream response timeouts"), - ("unclassified", "repeated upstream transport failures"), - ], -) -def test_http_bridge_retry_circuit_message_reports_the_actual_cause( - detail: str, - cause: str, -) -> None: - assert ( - http_bridge_retry_circuit_module._http_bridge_retry_circuit_error_message( - detail, - retry_after_seconds=37, - ) - == f"HTTP responses session bridge is cooling down after {cause}; retry after 37 seconds." - ) - - @pytest.mark.asyncio async def test_http_bridge_retry_circuit_restores_persisted_cooldown() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) @@ -24564,51 +27702,6 @@ async def test_http_bridge_retry_circuit_clear_retries_after_lookup_failure() -> assert hard_session.key not in cast(Any, service)._http_bridge_retry_circuit_persisted_keys -@pytest.mark.asyncio -async def test_http_bridge_completed_response_clears_retry_circuit( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - request_state = proxy_service._WebSocketRequestState( - request_id="req-circuit-terminal-success", - response_id="resp_circuit_terminal_success", - model="gpt-5.6-sol", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - event_queue=asyncio.Queue(), - transport="http", - ) - session = _make_bridge_session( - key_value="bridge-circuit-terminal-success", - pending_requests=deque([request_state]), - queued_request_count=1, - ) - clear_circuit = AsyncMock() - monkeypatch.setattr(service, "_clear_http_bridge_retry_circuit", clear_circuit) - monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", AsyncMock(return_value=True)) - monkeypatch.setattr(service, "_finalize_websocket_request_state", AsyncMock()) - - await service._process_http_bridge_upstream_text( - session, - json.dumps( - { - "type": "response.completed", - "response": { - "id": "resp_circuit_terminal_success", - "object": "response", - "status": "completed", - "output": [], - }, - }, - separators=(",", ":"), - ), - ) - - clear_circuit.assert_awaited_once_with(session) - - @pytest.mark.asyncio async def test_http_bridge_retry_circuit_replaces_local_state_after_newer_reset() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) @@ -24689,9 +27782,473 @@ async def test_http_bridge_retry_circuit_counts_stream_idle_timeout() -> None: persist_retry_circuit=AsyncMock(), ) - await service._record_http_bridge_retry_circuit_failure(hard_session, detail="stream_idle_timeout") - - assert cast(Any, service)._http_bridge_retry_circuits[hard_session.key].consecutive_failures == 1 + await service._record_http_bridge_retry_circuit_failure(hard_session, detail="stream_idle_timeout") + + assert cast(Any, service)._http_bridge_retry_circuits[hard_session.key].consecutive_failures == 1 + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_claims_one_attempt_once_across_concurrent_observers() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-concurrent") + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + both_lookups_started = asyncio.Event() + lookup_count = 0 + + async def lookup_retry_circuit(**_kwargs: object) -> None: + nonlocal lookup_count + lookup_count += 1 + if lookup_count == 2: + both_lookups_started.set() + await asyncio.wait_for(both_lookups_started.wait(), timeout=0.5) + + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(side_effect=lookup_retry_circuit), + persist_retry_circuit=persist_retry_circuit, + ) + + results = await asyncio.gather( + service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ), + service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ), + ) + + state = cast(Any, service)._http_bridge_retry_circuits[session.key] + assert results == [1, 1] + assert state.consecutive_failures == 1 + assert state.cooldown_until == 0.0 + assert attempt.retry_circuit_failure_recorded is True + assert persist_retry_circuit.await_count == 1 + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_duplicate_waits_for_persisted_merge( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-persist-merge") + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + persist_started = asyncio.Event() + allow_persist = asyncio.Event() + duplicate_wait_started = asyncio.Event() + + async def persist_retry_circuit(**_kwargs: object) -> SimpleNamespace: + persist_started.set() + await asyncio.wait_for(allow_persist.wait(), timeout=0.5) + return SimpleNamespace( + consecutive_failures=2, + cooldown_until_epoch=time.time() + 60.0, + last_detail="stream_idle_timeout", + updated_at_epoch=time.time(), + ) + + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=AsyncMock(side_effect=persist_retry_circuit), + ) + await_attempt_settlement = service._await_http_bridge_retry_circuit_attempt_settlement + + async def track_attempt_settlement( + target_session: proxy_service._HTTPBridgeSession, + *, + attempt: proxy_support_module._HTTPBridgeResponseCreateAttempt, + detail: str, + ) -> int: + duplicate_wait_started.set() + return await await_attempt_settlement( + target_session, + attempt=attempt, + detail=detail, + ) + + monkeypatch.setattr( + service, + "_await_http_bridge_retry_circuit_attempt_settlement", + track_attempt_settlement, + ) + + first_task = asyncio.create_task( + service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + ) + await asyncio.wait_for(persist_started.wait(), timeout=0.5) + duplicate_task = asyncio.create_task( + service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + ) + await asyncio.wait_for(duplicate_wait_started.wait(), timeout=0.5) + assert duplicate_task.done() is False + + allow_persist.set() + + assert await asyncio.wait_for(first_task, timeout=0.5) == 2 + assert await asyncio.wait_for(duplicate_task, timeout=0.5) == 2 + state = cast(Any, service)._http_bridge_retry_circuits[session.key] + assert state.consecutive_failures == 2 + assert attempt.retry_circuit_failure_settled is not None + assert attempt.retry_circuit_failure_settled.is_set() is True + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_duplicate_does_not_retry_failed_persistence( + caplog: pytest.LogCaptureFixture, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-persist-failure") + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + persist_retry_circuit = AsyncMock(side_effect=RuntimeError("durable write unavailable")) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=persist_retry_circuit, + ) + + with caplog.at_level(logging.WARNING): + first_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + duplicate_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + + assert (first_count, duplicate_count) == (1, 1) + assert cast(Any, service)._http_bridge_retry_circuits[session.key].consecutive_failures == 1 + assert persist_retry_circuit.await_count == 1 + assert "Failed to persist HTTP bridge retry circuit" in caplog.text + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_counts_distinct_send_attempts_and_not_delayed_old_observer() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-generations") + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=persist_retry_circuit, + ) + first_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + second_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=2) + + first_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=first_attempt, + ) + second_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=second_attempt, + ) + delayed_first_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=first_attempt, + ) + + state = cast(Any, service)._http_bridge_retry_circuits[session.key] + assert (first_count, second_count, delayed_first_count) == (1, 2, 2) + assert state.consecutive_failures == 2 + assert state.cooldown_until > time.monotonic() + assert first_attempt.retry_circuit_failure_recorded is True + assert second_attempt.retry_circuit_failure_recorded is True + assert persist_retry_circuit.await_count == 2 + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_does_not_claim_attempt_when_response_wins_lookup_race() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-response-race") + request_state = _make_eventless_http_bridge_owner() + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + request_state.response_create_attempt = attempt + lookup_started = asyncio.Event() + release_lookup = asyncio.Event() + + async def lookup_retry_circuit(**_kwargs: object) -> None: + lookup_started.set() + await asyncio.wait_for(release_lookup.wait(), timeout=0.5) + + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(side_effect=lookup_retry_circuit), + persist_retry_circuit=persist_retry_circuit, + ) + record_task = asyncio.create_task( + service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + ) + await asyncio.wait_for(lookup_started.wait(), timeout=0.5) + + proxy_support_module._record_response_event(request_state, "response.created") + release_lookup.set() + + assert await asyncio.wait_for(record_task, timeout=0.5) is None + assert attempt.response_observed is True + assert attempt.retry_circuit_failure_recorded is False + assert session.key not in cast(Any, service)._http_bridge_retry_circuits + persist_retry_circuit.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_ignores_disarmed_send_attempt() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-disarmed") + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1, disarmed=True) + lookup_retry_circuit = AsyncMock(return_value=None) + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=lookup_retry_circuit, + persist_retry_circuit=persist_retry_circuit, + ) + + result = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + + assert result is None + assert attempt.retry_circuit_failure_recorded is False + assert session.key not in cast(Any, service)._http_bridge_retry_circuits + lookup_retry_circuit.assert_not_awaited() + persist_retry_circuit.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_delayed_duplicate_does_not_recreate_cleared_state() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-reset-race") + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + lookup_retry_circuit = AsyncMock(return_value=None) + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=lookup_retry_circuit, + persist_retry_circuit=persist_retry_circuit, + ) + + assert ( + await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + == 1 + ) + await service._clear_http_bridge_retry_circuit(session) + lookup_count_after_clear = lookup_retry_circuit.await_count + + assert ( + await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + == 0 + ) + assert session.key not in cast(Any, service)._http_bridge_retry_circuits + assert lookup_retry_circuit.await_count == lookup_count_after_clear + assert persist_retry_circuit.await_count == 1 + + +def test_http_bridge_retry_circuit_attempt_selection_prefers_one_eventless_owner() -> None: + eventless = _make_eventless_http_bridge_owner(request_id="req-attempt-eventless") + eventless_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + eventless.response_create_attempt = eventless_attempt + responded = _make_eventless_http_bridge_owner(request_id="req-attempt-responded") + responded_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt( + ordinal=1, + response_observed=True, + ) + responded.response_create_attempt = responded_attempt + responded.response_event_count = 1 + responded.response_id = "resp-attempt-responded" + + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (responded, eventless) + ) + assert selection.kind == "eligible" + assert selection.attempt is eventless_attempt + + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (responded,) + ) + assert selection.kind == "settled" + assert selection.attempt is responded_attempt + + reconnecting = _make_eventless_http_bridge_owner(request_id="req-attempt-reconnecting") + reconnecting_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + reconnecting.response_create_attempt = reconnecting_attempt + reconnecting.response_create_sent_at = None + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (reconnecting,) + ) + assert selection.kind == "eligible" + assert selection.attempt is reconnecting_attempt + + other_eventless = _make_eventless_http_bridge_owner(request_id="req-attempt-other-eventless") + other_eventless_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + other_eventless.response_create_attempt = other_eventless_attempt + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (eventless, other_eventless) + ) + assert selection.kind == "eligible" + assert selection.ambiguous is True + assert selection.attempt is None + assert selection.attempts == (eventless_attempt, other_eventless_attempt) + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_ambiguous_attempts_never_fall_back_to_unscoped_failure() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-ambiguous") + first_request = _make_eventless_http_bridge_owner(request_id="req-attempt-ambiguous-first") + second_request = _make_eventless_http_bridge_owner(request_id="req-attempt-ambiguous-second") + first_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + second_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + first_request.response_create_attempt = first_attempt + second_request.response_create_attempt = second_attempt + lookup_retry_circuit = AsyncMock(return_value=None) + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=lookup_retry_circuit, + persist_retry_circuit=persist_retry_circuit, + ) + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (first_request, second_request) + ) + + assert selection.kind == "eligible" + assert selection.ambiguous is True + assert ( + await service._record_http_bridge_retry_circuit_failure_for_attempt_selection( + session, + detail="stream_idle_timeout", + selection=selection, + ) + is None + ) + assert session.key not in cast(Any, service)._http_bridge_retry_circuits + assert first_attempt.retry_circuit_failure_recorded is False + assert second_attempt.retry_circuit_failure_recorded is False + lookup_retry_circuit.assert_not_awaited() + persist_retry_circuit.assert_not_awaited() + + first_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=first_attempt, + ) + second_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=second_attempt, + ) + + assert (first_count, second_count) == (1, 2) + assert persist_retry_circuit.await_count == 2 + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_ineligible_attempt_never_falls_back_to_unscoped_failure() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-ineligible") + request_state = _make_eventless_http_bridge_owner(request_id="req-attempt-ineligible") + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + request_state.response_create_attempt = attempt + request_state.response_event_count = 1 + lookup_retry_circuit = AsyncMock(return_value=None) + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=lookup_retry_circuit, + persist_retry_circuit=persist_retry_circuit, + ) + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (request_state,) + ) + + assert selection.kind == "ineligible" + assert selection.attempt is None + assert ( + await service._record_http_bridge_retry_circuit_failure_for_attempt_selection( + session, + detail="stream_idle_timeout", + selection=selection, + ) + is None + ) + assert session.key not in cast(Any, service)._http_bridge_retry_circuits + assert attempt.retry_circuit_failure_recorded is False + lookup_retry_circuit.assert_not_awaited() + persist_retry_circuit.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_multiple_recorded_attempts_report_live_count_without_increment() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-recorded-selection") + first_request = _make_eventless_http_bridge_owner(request_id="req-attempt-recorded-first") + second_request = _make_eventless_http_bridge_owner(request_id="req-attempt-recorded-second") + first_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + second_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + first_request.response_create_attempt = first_attempt + second_request.response_create_attempt = second_attempt + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=persist_retry_circuit, + ) + assert ( + await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=first_attempt, + ) + == 1 + ) + assert ( + await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=second_attempt, + ) + == 2 + ) + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (first_request, second_request) + ) + + assert selection.kind == "recorded" + assert selection.ambiguous is True + assert ( + await service._record_http_bridge_retry_circuit_failure_for_attempt_selection( + session, + detail="stream_idle_timeout", + selection=selection, + ) + == 2 + ) + assert cast(Any, service)._http_bridge_retry_circuits[session.key].consecutive_failures == 2 + assert persist_retry_circuit.await_count == 2 @pytest.mark.parametrize( @@ -24768,7 +28325,7 @@ async def test_http_bridge_submit_suppresses_hard_key_during_retry_cooldown() -> http_bridge_retry_circuit_module._HTTPBridgeRetryCircuitState( consecutive_failures=2, cooldown_until=now + 60.0, - last_detail="stream_incomplete", + last_detail="stream_idle_timeout", last_touched_monotonic=now, ) ) @@ -24792,15 +28349,13 @@ async def test_http_bridge_submit_suppresses_hard_key_during_retry_cooldown() -> text_data=request_state.request_text or "", queue_limit=8, request_scope_id="scope-submit-cooldown", + owned_unanchored_handoff=False, ) assert exc_info.value.status_code == 503 assert exc_info.value.payload["error"]["code"] == "upstream_request_timeout" - assert exc_info.value.payload["error"]["message"] == ( - "HTTP responses session bridge is cooling down after repeated incomplete upstream WebSocket streams; " - "retry after 60 seconds." - ) - assert exc_info.value.retry_after_seconds == 60 + assert exc_info.value.retry_after_seconds is not None + assert exc_info.value.retry_after_seconds >= 60 assert hard_session.queued_request_count == 0 @@ -25029,6 +28584,61 @@ async def fail_replay(target_session: proxy_service._HTTPBridgeSession) -> bool: await submit_task +@pytest.mark.asyncio +async def test_http_bridge_reader_failure_classifies_each_operation_from_its_own_events( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + eventless = proxy_service._WebSocketRequestState( + request_id="req-eventless-sibling", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + operation_id="op-eventless-sibling", + response_event_count=0, + ) + streamed = proxy_service._WebSocketRequestState( + request_id="req-streamed-sibling", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + operation_id="op-streamed-sibling", + response_event_count=1, + ) + session = _make_bridge_session( + key_value="per-operation-close-classification", + pending_requests=deque([eventless, streamed]), + queued_request_count=2, + ) + event_order: list[str] = [] + update_operation = AsyncMock() + + async def discard_operation(*, operation_id: str) -> None: + event_order.append(f"discard:{operation_id}") + + update_operation.side_effect = lambda *args, **kwargs: event_order.append(f"update:{kwargs['state']}") + service._http_bridge_operation_event_batcher = cast(Any, SimpleNamespace(discard_operation=discard_operation)) + monkeypatch.setattr(http_bridge_upstream_events_module, "_update_http_bridge_operation_state", update_operation) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", AsyncMock()) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", AsyncMock()) + + await service._fail_http_bridge_reader_and_maybe_retire( + session, + error_code="stream_incomplete", + error_message="closed", + ) + + assert [call.kwargs["state"] for call in update_operation.await_args_list] == ["unknown", "acknowledged"] + assert event_order[:2] == ["discard:op-eventless-sibling", "discard:op-streamed-sibling"] + assert event_order[2:] == ["update:unknown", "update:acknowledged"] + + @pytest.mark.asyncio async def test_http_bridge_reader_failure_keeps_waiter_count_when_draining_request_is_present( monkeypatch: pytest.MonkeyPatch, @@ -25102,7 +28712,263 @@ async def test_http_bridge_repeated_zero_event_idle_timeouts_poison_anchor_with_ session, detail="repeated_zero_event_idle_timeout", response_events_seen=0, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), + ) + + +@pytest.mark.asyncio +async def test_http_bridge_repeated_zero_event_stream_incompletes_poison_anchor_with_waiter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Issue #1830: consecutive eventless ``stream_incomplete`` failures on the + # same anchor must count toward anchor poison exactly like idle timeouts, + # or a poisoned anchor wedges the session behind the retry circuit forever. + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key_value="bridge-anchor-poison-stream-incomplete", + pending_requests=deque([_make_eventless_http_bridge_owner()]), + queued_request_count=1, + ) + session.admission_waiter_count = 1 + session.durable_session_id = "durable-anchor-poison-stream-incomplete" + session.durable_owner_epoch = 3 + durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=AsyncMock(), + rebind_session_account=AsyncMock(return_value=True), + ) + service._durable_bridge = durable_bridge + fail_pending = AsyncMock() + retire = AsyncMock() + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + + for failure_number in range(1, 8): + retired = await service._fail_http_bridge_reader_and_maybe_retire( + session, + error_code="stream_incomplete", + error_message="Upstream websocket closed before response.completed", + ) + assert retired is (failure_number == 7) + + durable_bridge.rebind_session_account.assert_awaited_once_with( + session_id="durable-anchor-poison-stream-incomplete", + api_key_id=None, + instance_id=proxy_service.get_settings().http_responses_session_bridge_instance_id, + owner_epoch=3, + account_id="acc-bridge", + clear_continuity=True, + ) + retire.assert_awaited_once_with( + session, + detail="repeated_zero_event_stream_incomplete", + response_events_seen=0, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), + ) + + +@pytest.mark.asyncio +async def test_http_bridge_retire_stale_pending_poisons_anchor_after_repeated_eventless_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Issue #1830: the shared retirement boundary is the only strike recorder + # when a wedged anchored session fails without admission waiters, so it + # must clear the poisoned durable anchor once the threshold is reached. + service = proxy_service.ProxyService(cast(Any, nullcontext())) + durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=AsyncMock(), + rebind_session_account=AsyncMock(return_value=True), + ) + service._durable_bridge = durable_bridge + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) + + for _failure_number in range(7): + session = _make_bridge_session( + key_value="bridge-anchor-poison-retire", + pending_requests=deque([_make_eventless_http_bridge_owner()]), + queued_request_count=1, + ) + session.durable_session_id = "durable-anchor-poison-retire" + session.durable_owner_epoch = 5 + await service._retire_stale_pending_http_bridge_session( + session, + detail="stream_incomplete", + ) + + durable_bridge.rebind_session_account.assert_awaited_once_with( + session_id="durable-anchor-poison-retire", + api_key_id=None, + instance_id=proxy_service.get_settings().http_responses_session_bridge_instance_id, + owner_epoch=5, + account_id="acc-bridge", + clear_continuity=True, + ) + + +@pytest.mark.asyncio +async def test_http_bridge_retire_stale_pending_reattempts_failed_poison_clear( + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A clear that cannot be confirmed must not lose the self-heal: the next + # eligible eventless failure at or above the threshold re-attempts it, and + # each failed clear stays visible in the poison-clear telemetry. + service = proxy_service.ProxyService(cast(Any, nullcontext())) + durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=AsyncMock(), + rebind_session_account=AsyncMock(return_value=False), + ) + service._durable_bridge = durable_bridge + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) + + with caplog.at_level(logging.INFO): + for _failure_number in range(8): + session = _make_bridge_session( + key_value="bridge-anchor-poison-clear-retry", + pending_requests=deque([_make_eventless_http_bridge_owner()]), + queued_request_count=1, + ) + session.durable_session_id = "durable-anchor-poison-clear-retry" + session.durable_owner_epoch = 5 + await service._retire_stale_pending_http_bridge_session( + session, + detail="stream_incomplete", + ) + + assert durable_bridge.rebind_session_account.await_count == 2 + assert caplog.text.count("event=durable_anchor_poison_clear_failed") == 2 + + +@pytest.mark.asyncio +async def test_stream_via_http_bridge_recovers_terse_previous_response_rejection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Issue #1830 product path: an anchored bridge request fails with the terse + # parameterless previous-response rejection (classifiable only after code + # normalization). The bridge must enter local previous-response recovery + # instead of surfacing the failure into the ambiguous-transport class. + service = proxy_service.ProxyService(cast(Any, nullcontext())) + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "previous_response_id": "resp_stale_anchor", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "continue"}]}], + } + ) + session = _make_bridge_session(key_value="sid-terse-recovery") + terse_rejection = ProxyResponseError( + 400, + { + "error": { + "type": "invalid_request_error", + "message": "Invalid `previous_response_id`.", + } + }, + ) + get_or_create = AsyncMock(side_effect=[session, session]) + stream_attempts: list[str | None] = [] + + async def fake_stream_events( + _session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + propagate_http_errors: bool, + downstream_turn_state: str | None, + request_deadline: float | None = None, + ): + del queue_limit, propagate_http_errors, downstream_turn_state, request_deadline + stream_attempts.append(request_state.previous_response_id) + del text_data + if len(stream_attempts) == 1: + raise terse_rejection + yield 'data: {"type":"response.completed"}\n\n' + + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_http_bridge_local_owner_account_id", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-owner")) + monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_http_bridge_can_forward_to_active_owner", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_reset_http_bridge_session_after_local_terminal_error", AsyncMock()) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"session_id": "sid-terse-recovery"}, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] + + assert chunks == ['data: {"type":"response.completed"}\n\n'] + assert get_or_create.await_count == 2 + recovery_call = get_or_create.await_args_list[1] + assert recovery_call.kwargs["allow_previous_response_recovery_rebind"] is True + assert recovery_call.kwargs["request_stage"] == "reattach" + assert stream_attempts == ["resp_stale_anchor", "resp_stale_anchor"] + + +@pytest.mark.asyncio +async def test_http_bridge_retire_stale_pending_clean_close_never_poisons_anchor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=AsyncMock(), + rebind_session_account=AsyncMock(return_value=True), ) + service._durable_bridge = durable_bridge + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) + + for _failure_number in range(7): + session = _make_bridge_session( + key_value="bridge-anchor-clean-close", + pending_requests=deque([_make_eventless_http_bridge_owner()]), + queued_request_count=1, + ) + session.durable_session_id = "durable-anchor-clean-close" + session.durable_owner_epoch = 6 + await service._retire_stale_pending_http_bridge_session( + session, + detail="stream_incomplete", + retry_circuit_detail="clean_close", + ) + + durable_bridge.rebind_session_account.assert_not_awaited() @pytest.mark.asyncio @@ -25181,7 +29047,13 @@ async def test_http_bridge_eventless_timeout_force_retires_with_admission_waiter assert retired is True assert session.closed is True - retire.assert_awaited_once_with(session, detail="missing_response_created_timeout", response_events_seen=0) + retire.assert_awaited_once_with( + session, + detail="missing_response_created_timeout", + response_events_seen=0, + retired_request_count=0, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), + ) fail_pending_await_args = fail_pending.await_args assert fail_pending_await_args is not None assert fail_pending_await_args.kwargs["penalize_account"] is False @@ -25205,7 +29077,13 @@ async def test_http_bridge_reader_failure_retires_without_waiters_when_notificat error_message="closed", ) - retire.assert_awaited_once_with(session, detail="stream_incomplete", response_events_seen=0) + retire.assert_awaited_once_with( + session, + detail="stream_incomplete", + response_events_seen=0, + retired_request_count=0, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), + ) @pytest.mark.asyncio @@ -26727,6 +30605,67 @@ async def test_fail_stale_http_bridge_pending_requests_quarantines_wedged_gate_h record_failure.assert_not_awaited() +@pytest.mark.asyncio +async def test_fail_stale_http_bridge_pending_requests_captures_attempt_before_pending_lock_wait( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = _make_eventless_http_bridge_owner(request_id="req-stale-attempt-snapshot") + original_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + replacement_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=2) + request_state.response_create_attempt = original_attempt + session = _make_bridge_session( + key_value="stale-attempt-snapshot", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + attempt_captured = asyncio.Event() + original_selector = ( + http_bridge_request_submit_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests + ) + + def capture_attempt_before_lock( + request_states: list[proxy_service._WebSocketRequestState], + ) -> proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection: + selection = original_selector(request_states) + attempt_captured.set() + return selection + + monkeypatch.setattr( + http_bridge_request_submit_module, + "_http_bridge_retry_circuit_attempt_selection_for_pending_requests", + capture_attempt_before_lock, + ) + record_failure = AsyncMock(return_value=1) + monkeypatch.setattr( + service, + "_record_http_bridge_retry_circuit_failure_for_attempt_selection", + record_failure, + ) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", AsyncMock()) + + async with session.pending_lock: + fail_task = asyncio.create_task( + service._fail_stale_http_bridge_pending_requests( + session, + [request_state], + detail="response_create_gate_timeout_stuck_pending", + ) + ) + await asyncio.wait_for(attempt_captured.wait(), timeout=0.5) + request_state.response_create_attempt = replacement_attempt + + await asyncio.wait_for(fail_task, timeout=0.5) + + record_failure.assert_awaited_once() + record_failure_call = record_failure.await_args + assert record_failure_call is not None + selection = record_failure_call.kwargs["selection"] + assert selection.kind == "eligible" + assert selection.attempt is original_attempt + assert replacement_attempt.retry_circuit_failure_recorded is False + + @pytest.mark.asyncio async def test_http_bridge_missing_created_timeout_records_eventless_quarantine_strike( monkeypatch: pytest.MonkeyPatch, @@ -26816,15 +30755,14 @@ async def test_retire_stale_pending_http_bridge_session_quarantines_wedged_reatt @pytest.mark.asyncio -async def test_stream_http_bridge_quarantined_unproved_full_resend_keeps_anchor_when_reattach_gate_already_false( +async def test_stream_http_bridge_quarantined_full_resend_stays_unanchored_when_reattach_gate_already_false( monkeypatch: pytest.MonkeyPatch, ) -> None: - """A live alias never turns quarantine into an unanchored replay grant. - - This payload is multi-item/full-resend-shaped but omits the prior output, - so it cannot satisfy the durable completeness proof. Even when the normal - fresh-reattach eligibility gate is already false, the anchor must remain. - """ + """The quarantine suppression must be evaluated independently of the + fresh-reattach eligibility gate: when that gate is already false (here a + live alias session), a quarantined full-resend must still dispatch + unanchored instead of restoring the wedged durable anchor through session + hydration and session-level injection.""" service = proxy_service.ProxyService(cast(Any, nullcontext())) historical_input = [ {"role": "user", "content": [{"type": "input_text", "text": "leading question"}]}, @@ -26979,14 +30917,12 @@ async def fake_stream_events(*args: object, **kwargs: object): assert chunks == ['data: {"type":"response.completed"}\n\n'] assert len(dispatched_text) == 1 dispatched_payload = json.loads(dispatched_text[0]) - # The durable prefix is trimmed only because the exact anchor remains. - # A stale upstream anchor therefore fails closed instead of executing the - # incomplete suffix as an unanchored fresh conversation. - assert dispatched_payload["previous_response_id"] == "resp_wedged_anchor" - assert dispatched_payload["input"] == [ - {"role": "user", "content": [{"type": "input_text", "text": "follow-up without prior output"}]} - ] - assert fresh_session.last_completed_response_id == "resp_wedged_anchor" + # Genuinely unanchored: the suppressed durable anchor did not come back + # through session hydration or session-level injection, and the client's + # payload was not prefix-trimmed against the durable stored context. + assert "previous_response_id" not in dispatched_payload + assert len(dispatched_payload["input"]) == len(historical_input) + 1 + assert fresh_session.last_completed_response_id is None @pytest.mark.asyncio @@ -27049,3 +30985,542 @@ async def test_http_bridge_has_live_local_session_treats_quarantined_as_absent() ) is True ) + + +@pytest.mark.asyncio +async def test_http_bridge_eventless_timeout_signal_drains_after_repeated_sessions() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + account = SimpleNamespace(id="acc-eventless-timeouts") + session = cast(proxy_service._HTTPBridgeSession, SimpleNamespace(account=account)) + record_errors = AsyncMock() + service._load_balancer.record_errors = record_errors + + for _ in range(2): + await http_bridge_upstream_events_module._record_http_bridge_account_timeout_signal(service, session) + record_errors.assert_not_awaited() + + await http_bridge_upstream_events_module._record_http_bridge_account_timeout_signal(service, session) + record_errors.assert_awaited_once_with(account, 2) + + # The evidence window resets after one penalty; another isolated failure + # must not immediately apply a second account health penalty. + await http_bridge_upstream_events_module._record_http_bridge_account_timeout_signal(service, session) + await http_bridge_upstream_events_module._record_http_bridge_account_timeout_signal(service, session) + record_errors.assert_awaited_once_with(account, 2) + + +@pytest.mark.asyncio +async def test_prune_idle_http_bridge_sessions_evicts_without_request_traffic() -> None: + """The idle sweep is otherwise only reached from the request path, so a + replica that stops taking bridge requests would keep idle sessions' + upstream WebSockets open until restart (issue #1354).""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + idle = _make_bridge_session(key_value="idle-no-traffic") + idle.last_used_at = time.monotonic() - (idle.idle_ttl_seconds + 60.0) + fresh = _make_bridge_session(key_value="fresh-no-traffic") + fresh.last_used_at = time.monotonic() + service._http_bridge_sessions[idle.key] = idle + service._http_bridge_sessions[fresh.key] = fresh + + pruned_count = await service.prune_idle_http_bridge_sessions() + await asyncio.gather(*service._background_cleanup_tasks) + + assert pruned_count == 1 + assert idle.key not in service._http_bridge_sessions + assert fresh.key in service._http_bridge_sessions + assert fresh.closed is False + + +@pytest.mark.asyncio +async def test_prune_idle_http_bridge_sessions_spares_sessions_with_pending_work() -> None: + """The sweep reuses _prune_http_bridge_sessions_locked, so a session with + in-flight work keeps its own lifecycle even past the idle TTL.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + busy = _make_bridge_session(key_value="busy-no-traffic", queued_request_count=1) + busy.last_used_at = time.monotonic() - (busy.idle_ttl_seconds + 60.0) + service._http_bridge_sessions[busy.key] = busy + + assert await service.prune_idle_http_bridge_sessions() == 0 + assert busy.key in service._http_bridge_sessions + assert busy.closed is False + + +@pytest.mark.asyncio +async def test_prune_idle_http_bridge_sessions_is_a_noop_on_an_empty_registry() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + + assert await service.prune_idle_http_bridge_sessions() == 0 + assert not service._background_cleanup_tasks + + +@pytest.mark.asyncio +async def test_heartbeat_maintenance_runs_both_bridge_passes() -> None: + """The ring heartbeat is what makes these request-independent. Asserting the + sweep only through a direct call would still pass if the wiring were + removed, leaving the quiet-replica leak (issue #1354).""" + from app.main import run_http_bridge_heartbeat_maintenance + + proxy_service_double = SimpleNamespace( + reconcile_durable_http_bridge_ownership=AsyncMock(return_value=0), + prune_idle_http_bridge_sessions=AsyncMock(return_value=0), + ) + + await run_http_bridge_heartbeat_maintenance(proxy_service_double) + + proxy_service_double.reconcile_durable_http_bridge_ownership.assert_awaited_once() + proxy_service_double.prune_idle_http_bridge_sessions.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_heartbeat_maintenance_isolates_a_failing_pass() -> None: + """A failing reconcile must not skip the sweep, and neither may stop the + heartbeat loop.""" + from app.main import run_http_bridge_heartbeat_maintenance + + proxy_service_double = SimpleNamespace( + reconcile_durable_http_bridge_ownership=AsyncMock(side_effect=RuntimeError("durable read failed")), + prune_idle_http_bridge_sessions=AsyncMock(return_value=0), + ) + + await run_http_bridge_heartbeat_maintenance(proxy_service_double) + + proxy_service_double.prune_idle_http_bridge_sessions.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_heartbeat_maintenance_tolerates_a_missing_service_or_pass() -> None: + from app.main import run_http_bridge_heartbeat_maintenance + + await run_http_bridge_heartbeat_maintenance(None) + await run_http_bridge_heartbeat_maintenance(SimpleNamespace()) + + +@pytest.mark.asyncio +async def test_rejected_creator_does_not_release_the_registered_winners_durable_row( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Issue #1695: after an inflight evict, a replacement can register while + the stale creator is still claiming. The stale creator claims LAST, so its + epoch is current and its fenced release would succeed — closing the durable + row out from under the session that actually won the registry. A rejected + creator must not release a row that now belongs to someone else.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("turn_state_header", "sid-rejected-creator", None) + settings = _make_app_settings() + settings.proxy_admission_wait_timeout_seconds = 0.01 + stale_creator_session = _make_bridge_session(key_value="sid-rejected-creator") + stale_creator_session.key = key + registered_winner = _make_bridge_session(key_value="sid-rejected-creator-winner") + registered_winner.key = key + create_started = asyncio.Event() + finish_create = asyncio.Event() + + async def create_session(*_: object, **__: object) -> proxy_service._HTTPBridgeSession: + create_started.set() + await finish_create.wait() + return stale_creator_session + + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(service, "_create_http_bridge_session", create_session) + claim_durable_session = AsyncMock() + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", claim_durable_session) + close_http_bridge_session = AsyncMock() + monkeypatch.setattr(service, "_close_http_bridge_session", close_http_bridge_session) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "_http_bridge_should_wait_for_registration", AsyncMock(return_value=False)) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ("instance-a",))), + ) + + async def get_session() -> proxy_service._HTTPBridgeSession | proxy_service._HTTPBridgeOwnerForward: + return await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-turn-state": "sid-rejected-creator"}, + affinity=proxy_service._AffinityPolicy( + key="sid-rejected-creator", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + + owner_task = asyncio.create_task(get_session()) + await asyncio.wait_for(create_started.wait(), timeout=1.0) + + # The waiter times out and evicts the inflight future; a replacement then + # wins the registry slot while the stale creator is still in flight. + with pytest.raises(ProxyResponseError): + await asyncio.wait_for(get_session(), timeout=1.0) + async with service._http_bridge_lock: + service._http_bridge_sessions[key] = registered_winner + + finish_create.set() + with pytest.raises(ProxyResponseError): + await asyncio.wait_for(owner_task, timeout=1.0) + + # The winner keeps the registry slot, and the rejected creator closed its + # own session WITHOUT releasing the durable row the winner now owns. + assert service._http_bridge_sessions[key] is registered_winner + close_http_bridge_session.assert_awaited_once_with(stale_creator_session, release_durable_session=False) + # It never claimed at all: claiming would have advanced the durable epoch + # past the registered winner, fencing the winner's own renewals out of a + # row it legitimately owns. + claim_durable_session.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_superseded_creator_hands_its_claimed_epoch_to_the_registered_winner() -> None: + """Eviction can land DURING the claim, so a creator may advance the shared + row's epoch past the session that won the registry slot — fencing the + winner's own renewals out of a row it owns. Both sessions belong to this + instance and point at the same row, so the epoch is handed over rather + than stranded (issue #1695).""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("turn_state_header", "sid-handover", None) + winner = _make_bridge_session(key_value="sid-handover-winner") + winner.key = key + winner.durable_session_id = "durable-shared" + winner.durable_owner_epoch = 4 + superseded_creator = _make_bridge_session(key_value="sid-handover-stale") + superseded_creator.key = key + superseded_creator.durable_session_id = "durable-shared" + superseded_creator.durable_owner_epoch = 5 + service._http_bridge_sessions[key] = winner + + superseded = await http_bridge_helpers_module._settle_failed_http_bridge_creation( + service, + key, + inflight_future=None, + created_session=superseded_creator, + exc=RuntimeError("registration lost"), + ) + + assert superseded is True + assert winner.durable_owner_epoch == 5 + + +@pytest.mark.asyncio +async def test_settle_failed_creation_leaves_an_unrelated_winner_epoch_alone() -> None: + """The handover applies only when both sessions point at the same durable + row and the creator's epoch is genuinely newer.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("turn_state_header", "sid-handover-other", None) + winner = _make_bridge_session(key_value="sid-handover-other-winner") + winner.key = key + winner.durable_session_id = "durable-winner-row" + winner.durable_owner_epoch = 9 + creator = _make_bridge_session(key_value="sid-handover-other-stale") + creator.key = key + creator.durable_session_id = "durable-different-row" + creator.durable_owner_epoch = 12 + service._http_bridge_sessions[key] = winner + + assert ( + await http_bridge_helpers_module._settle_failed_http_bridge_creation( + service, + key, + inflight_future=None, + created_session=creator, + exc=RuntimeError("registration lost"), + ) + is True + ) + assert winner.durable_owner_epoch == 9 + + +@pytest.mark.asyncio +async def test_renew_adopts_a_same_instance_epoch_advance_for_the_registered_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A creator superseded mid-claim advances the row's epoch under this + instance. The session still holding the registry slot legitimately owns the + key, so its renewal must adopt the epoch — evicting would 409 the owner + (issue #1695).""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + registered = _make_bridge_session(key_value="sid-adopt") + registered.durable_session_id = "durable-adopt" + registered.durable_owner_epoch = 3 + service._http_bridge_sessions[registered.key] = registered + renew_live_session = AsyncMock( + return_value=SimpleNamespace( + owner_instance_id=_make_app_settings().http_responses_session_bridge_instance_id, + owner_process_epoch=http_bridge_owner_process_epoch(), + owner_epoch=4, + account_id="acc-bridge", + session_id="durable-adopt", + ) + ) + monkeypatch.setattr(service, "_durable_bridge", SimpleNamespace(renew_live_session=renew_live_session)) + schedule_closes = Mock() + monkeypatch.setattr(service, "_schedule_http_bridge_session_closes", schedule_closes) + + await http_bridge_helpers_module._renew_durable_http_bridge_lease(service, registered) + + assert registered.durable_owner_epoch == 4 + assert registered.closed is False + assert service._http_bridge_sessions[registered.key] is registered + schedule_closes.assert_not_called() + + +@pytest.mark.asyncio +async def test_renew_still_evicts_when_another_local_session_holds_the_slot( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A session that lost the registry slot to a different local session is a + genuine ownership loss and must still be evicted with the 409 contract.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + loser = _make_bridge_session(key_value="sid-evict") + loser.durable_session_id = "durable-evict" + loser.durable_owner_epoch = 3 + winner = _make_bridge_session(key_value="sid-evict-winner") + winner.key = loser.key + service._http_bridge_sessions[loser.key] = winner + renew_live_session = AsyncMock( + return_value=SimpleNamespace( + owner_instance_id=_make_app_settings().http_responses_session_bridge_instance_id, + owner_process_epoch=http_bridge_owner_process_epoch(), + owner_epoch=4, + account_id="acc-bridge", + session_id="durable-evict", + ) + ) + monkeypatch.setattr(service, "_durable_bridge", SimpleNamespace(renew_live_session=renew_live_session)) + monkeypatch.setattr(service, "_schedule_http_bridge_session_closes", Mock()) + + with pytest.raises(ProxyResponseError) as exc_info: + await http_bridge_helpers_module._renew_durable_http_bridge_lease(service, loser) + + assert exc_info.value.status_code == 409 + assert loser.durable_owner_epoch == 3 + assert loser.closed is True + + +@pytest.mark.asyncio +async def test_settle_failed_creation_spares_a_replacement_that_has_not_registered_yet() -> None: + """A replacement that has claimed but not yet published its session is as + much the winner as a registered one: releasing here would close the row + beneath it, and it would then register with an older epoch and be fenced + out on its first renewal (issue #1695).""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("turn_state_header", "sid-inflight-winner", None) + stale_creator = _make_bridge_session(key_value="sid-inflight-stale") + stale_creator.key = key + stale_creator.durable_session_id = "durable-inflight" + stale_creator.durable_owner_epoch = 6 + replacement_future: asyncio.Future[Any] = asyncio.get_running_loop().create_future() + service._http_bridge_inflight_sessions[key] = replacement_future + + superseded = await http_bridge_helpers_module._settle_failed_http_bridge_creation( + service, + key, + inflight_future=None, + created_session=stale_creator, + exc=RuntimeError("registration lost"), + ) + + # No session is registered yet, but the replacement owns the inflight slot. + assert service._http_bridge_sessions.get(key) is None + assert superseded is True + # The replacement's future is left intact for it to resolve. + assert service._http_bridge_inflight_sessions.get(key) is replacement_future + replacement_future.cancel() + + +@pytest.mark.asyncio +async def test_settle_failed_creation_releases_when_nothing_replaced_it() -> None: + """With no registered session and no in-flight replacement, the creator + still owns the row and must release it rather than leak it.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("turn_state_header", "sid-sole-creator", None) + sole_creator = _make_bridge_session(key_value="sid-sole-creator") + sole_creator.key = key + sole_creator.durable_session_id = "durable-sole" + sole_creator.durable_owner_epoch = 2 + + superseded = await http_bridge_helpers_module._settle_failed_http_bridge_creation( + service, + key, + inflight_future=None, + created_session=sole_creator, + exc=RuntimeError("upstream failed"), + ) + + assert superseded is False + + +@pytest.mark.asyncio +async def test_renew_evicts_when_a_newer_process_incarnation_advanced_the_epoch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Two incarnations can share a configured instance ID across a graceful + restart. The successor process's claim must still fence the predecessor + out, so adoption requires a matching process epoch, not just the instance + ID (issue #1695).""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + old_process_session = _make_bridge_session(key_value="sid-restart") + old_process_session.durable_session_id = "durable-restart" + old_process_session.durable_owner_epoch = 3 + service._http_bridge_sessions[old_process_session.key] = old_process_session + renew_live_session = AsyncMock( + return_value=SimpleNamespace( + owner_instance_id=_make_app_settings().http_responses_session_bridge_instance_id, + # Same instance ID, different incarnation. + owner_process_epoch="successor-process-epoch", + owner_epoch=4, + account_id="acc-bridge", + session_id="durable-restart", + ) + ) + monkeypatch.setattr(service, "_durable_bridge", SimpleNamespace(renew_live_session=renew_live_session)) + monkeypatch.setattr(service, "_schedule_http_bridge_session_closes", Mock()) + + with pytest.raises(ProxyResponseError) as exc_info: + await http_bridge_helpers_module._renew_durable_http_bridge_lease(service, old_process_session) + + assert exc_info.value.status_code == 409 + assert old_process_session.durable_owner_epoch == 3 + assert old_process_session.closed is True + + +@pytest.mark.asyncio +async def test_renew_does_not_adopt_an_epoch_advance_that_moved_the_row_to_another_account( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A claim rewrites the row's account_id and clears continuity aliases, so + an advance that moved the row to another account is a real ownership change + for this session — adopting would keep it dispatching on a row bound to a + different account (issue #1695).""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + session = _make_bridge_session(key_value="sid-account-moved") + session.durable_session_id = "durable-account-moved" + session.durable_owner_epoch = 3 + service._http_bridge_sessions[session.key] = session + renew_live_session = AsyncMock( + return_value=SimpleNamespace( + owner_instance_id=_make_app_settings().http_responses_session_bridge_instance_id, + owner_process_epoch=http_bridge_owner_process_epoch(), + owner_epoch=4, + account_id="acc-other", + session_id="durable-account-moved", + ) + ) + monkeypatch.setattr(service, "_durable_bridge", SimpleNamespace(renew_live_session=renew_live_session)) + monkeypatch.setattr(service, "_schedule_http_bridge_session_closes", Mock()) + + with pytest.raises(ProxyResponseError) as exc_info: + await http_bridge_helpers_module._renew_durable_http_bridge_lease(service, session) + + assert exc_info.value.status_code == 409 + assert session.durable_owner_epoch == 3 + assert session.closed is True + + +@pytest.mark.asyncio +async def test_settle_failed_creation_releases_a_row_rebound_away_from_the_winner() -> None: + """A winner on a different account no longer shares this row: the claim + already rewrote its account binding and cleared its continuity aliases. + The epoch is not handed over, and the row IS released so the winner is + fenced promptly instead of dispatching against a row bound elsewhere.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("turn_state_header", "sid-account-handover", None) + winner = _make_bridge_session(key_value="sid-account-handover-winner") + winner.key = key + winner.durable_session_id = "durable-account-handover" + winner.durable_owner_epoch = 4 + winner.account = cast(Any, SimpleNamespace(id="acc-winner", status=AccountStatus.ACTIVE, plan_type="plus")) + stale_creator = _make_bridge_session(key_value="sid-account-handover-stale") + stale_creator.key = key + stale_creator.durable_session_id = "durable-account-handover" + stale_creator.durable_owner_epoch = 5 + stale_creator.account = cast(Any, SimpleNamespace(id="acc-stale", status=AccountStatus.ACTIVE, plan_type="plus")) + service._http_bridge_sessions[key] = winner + + superseded = await http_bridge_helpers_module._settle_failed_http_bridge_creation( + service, + key, + inflight_future=None, + created_session=stale_creator, + exc=RuntimeError("registration lost"), + ) + + # Not treated as superseded: the row is ours to release, and the winner + # will be fenced on its next renewal and retry cleanly. + assert superseded is False + assert winner.durable_owner_epoch == 4 + + +@pytest.mark.asyncio +async def test_admission_waiters_do_not_accumulate_callbacks_on_shared_inflight_future( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regression for the 2026-08-20 event-loop livelock: admission waiters + piling onto the shared inflight future must not attach per-waiter + callbacks. The old ``wait_for(asyncio.shield(...))`` pattern left + O(waiters) callbacks on the registry future (Python 3.14 shield never + removes ``_clear_awaited_by_callback`` on waiter cancellation) and paid + O(n) removal scans per timeout, so a mass timeout ground the event loop + at O(n^2).""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + # prompt_cache_key keeps the canonical key: session_header requests + # without turn state are rewritten to per-request parallel fork keys and + # never share the inflight future. + key = proxy_service._HTTPBridgeSessionKey("prompt_cache_key", "bridge-waiter-pileup", None) + inflight: asyncio.Future[Any] = asyncio.get_running_loop().create_future() + setattr( + inflight, + http_bridge_mixin_module._HTTP_BRIDGE_INFLIGHT_STARTED_AT_ATTR, + time.monotonic(), + ) + service._http_bridge_inflight_sessions[key] = inflight + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_proxy_admission_wait_timeout_seconds", lambda settings=None: 0.2) + + async def _single_instance_ring(settings: Any, ring_membership: Any = None) -> tuple[str, tuple[str, ...]]: + return "local-instance", ("local-instance",) + + monkeypatch.setattr(proxy_service, "_active_http_bridge_instance_ring", _single_instance_ring) + + async def _wait_once() -> Any: + return await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-waiter-pileup"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + + waiters = [asyncio.create_task(_wait_once()) for _ in range(50)] + await asyncio.sleep(0.05) + assert not inflight.done() + callbacks = getattr(inflight, "_callbacks", None) + assert callbacks is not None and len(callbacks) == 1, ( + f"admission waiters must share one fan-out callback on the inflight future, found " + f"{None if callbacks is None else len(callbacks)}" + ) + + # Client-disconnect storm: cancelling waiters must leave the shared future + # pending (the owner's creation continues) and leak no callbacks. + for waiter in waiters[:25]: + waiter.cancel() + cancelled = await asyncio.gather(*waiters[:25], return_exceptions=True) + assert all(isinstance(result, asyncio.CancelledError) for result in cancelled) + assert not inflight.done() + callbacks = getattr(inflight, "_callbacks", None) + assert callbacks is not None and len(callbacks) == 1 + + # The surviving waiters time out: the first to fire fails the shared + # future for the rest with the local-overload contract error. + remaining = await asyncio.gather(*waiters[25:], return_exceptions=True) + assert all(isinstance(result, ProxyResponseError) and result.status_code == 429 for result in remaining) + assert key not in service._http_bridge_inflight_sessions diff --git a/tests/unit/test_proxy_load_balancer_refresh.py b/tests/unit/test_proxy_load_balancer_refresh.py index f4a63bc274..6166c2e824 100644 --- a/tests/unit/test_proxy_load_balancer_refresh.py +++ b/tests/unit/test_proxy_load_balancer_refresh.py @@ -7,7 +7,7 @@ from dataclasses import replace from datetime import datetime, timedelta, timezone from types import SimpleNamespace -from typing import Any, cast +from typing import Any, Literal, cast import pytest @@ -202,7 +202,9 @@ async def get_account_id( *, kind: StickySessionKind, max_age_seconds: int | None = None, + continuity_source: Literal["session_header", "thread_header", "turn_state"] | None = None, ) -> str | None: + del continuity_source return None async def get_account_id_and_abandonment( @@ -211,11 +213,17 @@ async def get_account_id_and_abandonment( *, kind: StickySessionKind, max_age_seconds: int | None = None, + continuity_source: Literal["session_header", "thread_header", "turn_state"] | None = None, ) -> StickyOwnerLookup: # Delegates to get_account_id (rather than duplicating its logic) so # a test that only overrides get_account_id — the common pattern in # this file — is still observed here. - account_id = await self.get_account_id(key, kind=kind, max_age_seconds=max_age_seconds) + account_id = await self.get_account_id( + key, + kind=kind, + max_age_seconds=max_age_seconds, + continuity_source=continuity_source, + ) return StickyOwnerLookup(account_id=account_id, continuity_abandoned=False) async def upsert(self, key: str, account_id: str, *, kind: StickySessionKind) -> StickySession: @@ -1229,26 +1237,26 @@ async def test_select_account_filters_requested_service_tier_plans(monkeypatch) @pytest.mark.asyncio async def test_select_account_filters_requested_service_tier_accounts(monkeypatch) -> None: - no_fast = _make_account("acc-tier-pro-default", "tier-pro-default@example.com") - no_fast.plan_type = "pro" - fast = _make_account("acc-tier-pro-fast", "tier-pro-fast@example.com") - fast.plan_type = "pro" + no_ultrafast = _make_account("acc-tier-pro-default", "tier-pro-default@example.com") + no_ultrafast.plan_type = "pro" + ultrafast = _make_account("acc-tier-pro-ultrafast", "tier-pro-ultrafast@example.com") + ultrafast.plan_type = "pro" now = utcnow() now_epoch = int(now.replace(tzinfo=timezone.utc).timestamp()) usage_repo = StubUsageRepository( primary={ - no_fast.id: UsageHistory( + no_ultrafast.id: UsageHistory( id=63, - account_id=no_fast.id, + account_id=no_ultrafast.id, recorded_at=now, window="primary", used_percent=1.0, reset_at=now_epoch + 300, window_minutes=5, ), - fast.id: UsageHistory( + ultrafast.id: UsageHistory( id=64, - account_id=fast.id, + account_id=ultrafast.id, recorded_at=now, window="primary", used_percent=2.0, @@ -1264,7 +1272,7 @@ async def test_select_account_filters_requested_service_tier_accounts(monkeypatc lambda: SimpleNamespace( plan_types_for_model=lambda _model: frozenset({"pro"}), account_ids_for_model_service_tier=lambda _model, tier: ( - frozenset({fast.id}) if tier == "priority" else None + frozenset({ultrafast.id}) if tier == "ultrafast" else None ), plan_types_for_model_service_tier=lambda _model, _tier: frozenset({"pro"}), ), @@ -1272,15 +1280,15 @@ async def test_select_account_filters_requested_service_tier_accounts(monkeypatc balancer = LoadBalancer( lambda: _repo_factory( - StubAccountsRepository([no_fast, fast]), + StubAccountsRepository([no_ultrafast, ultrafast]), usage_repo, StubStickySessionsRepository(), ) ) - selection = await balancer.select_account(model="gpt-5.5", service_tier="priority") + selection = await balancer.select_account(model="gpt-5.6-sol", service_tier="ultrafast") assert selection.account is not None - assert selection.account.id == fast.id + assert selection.account.id == ultrafast.id @pytest.mark.asyncio @@ -2264,8 +2272,9 @@ async def pinned_account_id( *, kind: StickySessionKind, max_age_seconds: int | None = None, + continuity_source: Literal["session_header", "thread_header", "turn_state"] | None = None, ) -> str | None: - del key, kind, max_age_seconds + del key, kind, max_age_seconds, continuity_source return account.id original_persist_selection_state = balancer._persist_selection_state @@ -2350,8 +2359,9 @@ async def pinned_account_id( *, kind: StickySessionKind, max_age_seconds: int | None = None, + continuity_source: Literal["session_header", "thread_header", "turn_state"] | None = None, ) -> str | None: - del key, kind, max_age_seconds + del key, kind, max_age_seconds, continuity_source return account.id async def always_stale_selected_persist( @@ -2435,8 +2445,9 @@ async def pinned_account_id( *, kind: StickySessionKind, max_age_seconds: int | None = None, + continuity_source: Literal["session_header", "thread_header", "turn_state"] | None = None, ) -> str | None: - del key, kind, max_age_seconds + del key, kind, max_age_seconds, continuity_source return paused_team.id monkeypatch.setattr(sticky_repo, "get_account_id", pinned_account_id) @@ -4136,7 +4147,7 @@ def test_enforced_service_tier_provenance_treats_default_aliases_as_omitted( service_tier_was_enforced = apply_api_key_enforcement( payload, _service_tier_enforcement_key("priority"), - ) + ).service_tier_was_enforced assert service_tier_was_enforced is True assert payload.service_tier == "priority" @@ -4175,7 +4186,7 @@ async def test_select_account_ignores_enforced_service_tier_the_model_never_adve service_tier_was_enforced = apply_api_key_enforcement( payload, _service_tier_enforcement_key("priority"), - ) + ).service_tier_was_enforced assert service_tier_was_enforced is True assert apply_enforced_service_tier_model_fallback( payload, @@ -4204,7 +4215,7 @@ async def test_select_account_ignores_enforced_service_tier_the_model_never_adve explicitly_requested = apply_api_key_enforcement( explicit_payload, _service_tier_enforcement_key("priority"), - ) + ).service_tier_was_enforced assert explicitly_requested is False assert not apply_enforced_service_tier_model_fallback( explicit_payload, @@ -4303,7 +4314,7 @@ async def test_api_key_enforced_priority_tier_still_routes_a_model_without_prior last_used_at=None, ) payload = ResponsesRequest(model=model, instructions="ping", input=[]) - service_tier_was_enforced = apply_api_key_enforcement(payload, api_key) + service_tier_was_enforced = apply_api_key_enforcement(payload, api_key).service_tier_was_enforced assert payload.service_tier == "priority" assert service_tier_was_enforced is True assert apply_enforced_service_tier_model_fallback( diff --git a/tests/unit/test_proxy_security_work.py b/tests/unit/test_proxy_security_work.py index d419686792..b2f61fd45f 100644 --- a/tests/unit/test_proxy_security_work.py +++ b/tests/unit/test_proxy_security_work.py @@ -88,6 +88,77 @@ async def test_process_websocket_security_retry_releases_response_create_gate() gate.release() +@pytest.mark.asyncio +async def test_websocket_security_cleanup_finishes_after_cancellation() -> None: + """The externally exercised WebSocket path cannot orphan a response lease.""" + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_security_gate_cancel") + gate = asyncio.Semaphore(1) + await gate.acquire() + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_security_gate_cancel", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + awaiting_response_created=True, + transport="websocket", + request_text='{"type":"response.create","model":"gpt-5.1","input":[]}', + ) + request_state.response_create_gate = gate + request_state.response_create_gate_acquired = True + lease = proxy_service.AccountLease("lease-ws-security-gate-cancel", account.id, "response_create", 1.0) + release_started = asyncio.Event() + release_finished = asyncio.Event() + + async def release_account_lease(value): + assert value is lease + release_started.set() + await release_finished.wait() + + request_state.account_response_create_lease = lease + request_state.account_response_create_release = release_account_lease + pending_requests = deque([request_state]) + upstream_control = proxy_service._WebSocketUpstreamControl() + text = json.dumps( + { + "type": "response.failed", + "response": { + "id": "resp_ws_security_gate_cancel", + "status": "failed", + "error": {"code": "invalid_request_error", "type": "invalid_request_error", "message": "cancel"}, + }, + }, + separators=(",", ":"), + ) + + task = asyncio.create_task( + service._process_upstream_websocket_text( + text, + account=account, + account_id_value=account.id, + pending_requests=pending_requests, + pending_lock=anyio.Lock(), + api_key=None, + upstream_control=upstream_control, + response_create_gate=gate, + ) + ) + await release_started.wait() + task.cancel() + release_finished.set() + with pytest.raises(asyncio.CancelledError): + await task + + assert request_state.account_response_create_lease is None + assert request_state.account_response_create_release is None + assert request_state.response_create_gate_acquired is False + assert request_state.response_create_gate is None + await asyncio.wait_for(gate.acquire(), timeout=0.1) + gate.release() + + def test_http_bridge_deferred_reasoning_blocks_previsible_replay() -> None: request_state = proxy_service._WebSocketRequestState( request_id="http_security_deferred_reasoning", diff --git a/tests/unit/test_proxy_tool_call_dedupe.py b/tests/unit/test_proxy_tool_call_dedupe.py index 71e48b21a0..f9ad9f0749 100644 --- a/tests/unit/test_proxy_tool_call_dedupe.py +++ b/tests/unit/test_proxy_tool_call_dedupe.py @@ -2178,3 +2178,59 @@ def test_rewrite_parallel_tool_call_payload_removes_duplicate_goal_side_effects( "functions.update_plan", "functions.request_user_input", ] + + +def test_rewrite_parallel_tool_call_text_does_not_revalidate_unchanged_frames( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Hot streaming paths validate only lifecycle frames; the unchanged path + # must not re-run pydantic validation when the caller passed event=None. + calls: list[JsonValue | None] = [] + + def counting_validate(payload: JsonValue | None) -> None: + calls.append(payload) + return None + + monkeypatch.setattr(tool_call_dedupe, "parse_sse_event_payload", counting_validate) + payload: dict[str, JsonValue] = {"type": "response.output_text.delta", "delta": "hello"} + text = json.dumps(payload, separators=(",", ":")) + + rewritten_text, rewritten_payload, event, event_type, event_block = ( + tool_call_dedupe.rewrite_parallel_tool_call_text( + text, + payload, + event_block=f"data: {text}\n\n", + ) + ) + + assert calls == [] + assert event is None + assert event_type == "response.output_text.delta" + assert rewritten_text == text + assert rewritten_payload is payload + assert event_block == f"data: {text}\n\n" + + +def test_rewrite_parallel_tool_call_sse_line_does_not_revalidate_unchanged_frames( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[JsonValue | None] = [] + + def counting_validate(payload: JsonValue | None) -> None: + calls.append(payload) + return None + + monkeypatch.setattr(tool_call_dedupe, "parse_sse_event_payload", counting_validate) + payload: dict[str, JsonValue] = {"type": "response.reasoning_text.delta", "delta": "r"} + line = format_sse_event(payload) + + rewritten_line, rewritten_payload, event, event_type = tool_call_dedupe.rewrite_parallel_tool_call_sse_line( + line, + payload, + ) + + assert calls == [] + assert event is None + assert event_type == "response.reasoning_text.delta" + assert rewritten_line == line + assert rewritten_payload is payload diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 7cb7746a0a..12b40079ab 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -15,7 +15,7 @@ from copy import deepcopy from datetime import timedelta from types import SimpleNamespace -from typing import Any, AsyncIterator, Iterator, Literal, Protocol, Self, cast +from typing import Any, AsyncIterator, Iterator, Literal, Self, cast from unittest.mock import ANY, AsyncMock, MagicMock from unittest.mock import call as mock_call @@ -25,15 +25,19 @@ from aiohttp.client_exceptions import ClientConnectorCertificateError from aiohttp.client_reqrep import ConnectionKey, RequestInfo from fastapi import WebSocket +from hypothesis import given, settings +from hypothesis import strategies as st from starlette.requests import Request from starlette.responses import StreamingResponse from websockets.exceptions import ConnectionClosedError from websockets.frames import Close import app.core.clients.proxy as proxy_module +import app.core.openai.requests as openai_requests_module import app.core.resilience.network_recovery as network_recovery_module import app.modules.proxy.load_balancer as load_balancer_module from app.core import shutdown as shutdown_state +from app.core.auth.refresh import RefreshError from app.core.balancer.types import UpstreamError from app.core.clients.proxy import _build_upstream_headers, filter_inbound_headers from app.core.clients.proxy_websocket import ( @@ -46,6 +50,7 @@ from app.core.config.settings import Settings from app.core.crypto import TokenEncryptor from app.core.errors import openai_error +from app.core.exceptions import ProxyReasoningEffortNotAllowed from app.core.openai.models import CompactResponsePayload, OpenAIResponsePayload from app.core.openai.parsing import parse_sse_event from app.core.openai.requests import ResponsesCompactRequest, ResponsesRequest @@ -60,12 +65,13 @@ from app.modules.accounts import auth_manager as auth_manager_module from app.modules.accounts.repository import AccountsRepository from app.modules.api_keys.repository import ApiKeysRepository -from app.modules.api_keys.service import ApiKeyData +from app.modules.api_keys.service import ApiKeyData, ApiKeyUsageReservationData from app.modules.proxy import affinity as proxy_affinity from app.modules.proxy import api as proxy_api from app.modules.proxy import request_policy as proxy_request_policy from app.modules.proxy import service as proxy_service from app.modules.proxy._service import compact as proxy_compact_service +from app.modules.proxy._service import file_ops as proxy_file_ops from app.modules.proxy._service import support as proxy_support from app.modules.proxy._service import warmup as proxy_warmup_service from app.modules.proxy._service.http_bridge import request_submit as proxy_http_bridge_request_submit @@ -98,6 +104,7 @@ from app.modules.request_logs.repository import PreviousResponseOwnerRecord, RequestLogsRepository from app.modules.usage.repository import AdditionalUsageRepository, UsageRepository from tests.unit._proxy_test_helpers import runtime_basic_auth_url +from tests.unit.hypothesis_strategies import json_objects, json_values pytestmark = pytest.mark.unit @@ -169,6 +176,10 @@ def test_compact_wire_budget_rejection_is_account_neutral() -> None: assert proxy_service._is_account_neutral_error_code("responses_compact_input_too_large") is True +def test_stream_idle_timeout_is_account_neutral() -> None: + assert proxy_service._is_account_neutral_error_code("stream_idle_timeout") is True + + @pytest.mark.asyncio async def test_stream_selector_compatibility_drops_unsupported_continuity_owner_hint() -> None: calls: list[tuple[float, str | None]] = [] @@ -348,6 +359,30 @@ async def test_account_scoped_invalid_request_error_still_penalizes_account() -> load_balancer.record_error.assert_awaited_once() +@pytest.mark.asyncio +async def test_stream_idle_timeout_does_not_penalize_account() -> None: + load_balancer = SimpleNamespace( + record_error=AsyncMock(), + mark_rate_limit=AsyncMock(), + mark_quota_exceeded=AsyncMock(), + mark_permanent_failure=AsyncMock(), + ) + proxy = SimpleNamespace(_load_balancer=load_balancer) + + classified = await streaming_helpers_module._handle_stream_error( + proxy, + cast(Account, SimpleNamespace(id="acc-idle")), + {"message": "idle"}, + "stream_idle_timeout", + ) + + assert classified["failure_class"] == "non_retryable" + load_balancer.record_error.assert_not_awaited() + load_balancer.mark_rate_limit.assert_not_awaited() + load_balancer.mark_quota_exceeded.assert_not_awaited() + load_balancer.mark_permanent_failure.assert_not_awaited() + + def test_websocket_archive_request_context_clears_unmatched_frame_request_id(): token = set_request_id("req_previous_response") try: @@ -444,6 +479,150 @@ def test_websocket_account_switch_keeps_anchor_when_fresh_replay_references_file assert request_state.preferred_account_id == "acc_file_owner" +def test_websocket_account_switch_blocks_unanchored_account_bound_request(): + request_state = proxy_service._WebSocketRequestState( + request_id="req_ws_account_bound", + model="gpt-5.6-sol", + service_tier="priority", + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic(), + request_text=( + '{"type":"response.create","model":"gpt-5.6-sol","input":[' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"}]}' + ), + preferred_account_id="acc_owner", + ) + + assert websocket_mixin._prepare_websocket_request_state_for_account_switch(request_state) is None + assert request_state.preferred_account_id == "acc_owner" + + +def test_websocket_account_switch_blocks_account_bound_fresh_replay(): + fresh_text = ( + '{"type":"response.create","model":"gpt-5.6-sol","input":[' + '{"role":"user","content":"hello"},' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"},' + '{"type":"function_call","name":"lookup","call_id":"call_1","arguments":"{}"},' + '{"type":"function_call_output","call_id":"call_1","output":"ok"}]}' + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req_ws_account_bound_fresh", + model="gpt-5.6-sol", + service_tier="priority", + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic(), + request_text='{"type":"response.create","previous_response_id":"resp_proxy"}', + previous_response_id="resp_proxy", + preferred_account_id="acc_owner", + proxy_injected_previous_response_id=True, + fresh_upstream_request_is_retry_safe=True, + fresh_upstream_request_text=fresh_text, + ) + + assert websocket_mixin._prepare_websocket_request_state_for_account_switch(request_state) is None + assert request_state.previous_response_id == "resp_proxy" + assert request_state.preferred_account_id == "acc_owner" + + +def test_websocket_dispatch_owner_rejects_account_bound_socket_reuse(): + request_text = ( + '{"type":"response.create","model":"gpt-5.6-sol","input":[' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"}]}' + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req_ws_dispatch_owner", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + request_text=request_text, + ) + + assert websocket_mixin._bind_websocket_request_dispatch_owner( + request_state, + account_id="acc_owner", + exact_request_text=request_text, + ) + assert not websocket_mixin._bind_websocket_request_dispatch_owner( + request_state, + account_id="acc_other", + exact_request_text=request_text, + ) + assert request_state.preferred_account_id == "acc_owner" + assert request_state.replay_required_account_id == "acc_owner" + + +def test_websocket_verified_fresh_replay_clears_dispatch_owner_atomically(): + fresh_text = '{"type":"response.create","model":"gpt-5.6-sol","input":"portable user input"}' + request_state = proxy_service._WebSocketRequestState( + request_id="req_ws_verified_fresh_owner", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + request_text='{"type":"response.create","previous_response_id":"resp_proxy"}', + previous_response_id="resp_proxy", + preferred_account_id="acc_owner", + replay_required_account_id="acc_owner", + proxy_injected_previous_response_id=True, + fresh_upstream_request_is_retry_safe=True, + fresh_upstream_request_text=fresh_text, + ) + + assert websocket_mixin._install_verified_fresh_replay(request_state) == fresh_text + assert request_state.request_text == fresh_text + assert request_state.previous_response_id is None + assert request_state.preferred_account_id is None + assert request_state.replay_required_account_id is None + + +def test_websocket_bound_auth_replay_allows_one_same_owner_refresh(): + request_text = ( + '{"type":"response.create","model":"gpt-5.6-sol","input":[' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"}]}' + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req_ws_bound_auth_refresh", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + request_text=request_text, + preferred_account_id="acc_owner", + replay_required_account_id="acc_owner", + ) + + assert ( + websocket_mixin._prepare_websocket_request_state_for_auth_replay( + request_state, + current_account_id="acc_other", + ) + is None + ) + assert request_state.auth_replay_count == 0 + assert ( + websocket_mixin._prepare_websocket_request_state_for_auth_replay( + request_state, + current_account_id="acc_owner", + ) + == request_text + ) + assert request_state.auth_replay_count == 1 + assert request_state.replay_required_account_id == "acc_owner" + assert ( + websocket_mixin._prepare_websocket_request_state_for_auth_replay( + request_state, + current_account_id="acc_owner", + ) + is None + ) + + def test_websocket_owner_switch_detects_other_pending_request() -> None: current = proxy_service._WebSocketRequestState( request_id="req_owner_switch", @@ -2060,6 +2239,200 @@ def test_normalize_unsupported_reasoning_effort_rewrites_minimal_to_low(caplog): assert any("reasoning_effort_normalized" in record.message for record in caplog.records) +def test_normalize_unsupported_reasoning_effort_rewrites_a_model_absent_from_the_snapshot(): + """Snapshot membership must not decide whether the workaround applies. + + A populated snapshot can omit a genuine subscription model -- a partial + refresh, an account unavailable during refresh, or an operator-mapped slug + outside the bootstrap set -- and those requests still reach the ChatGPT + backend through the unfiltered fallback. Skipping the rewrite for them + would restore the no-completion hang it exists to prevent, so the rewrite + is unconditional here and only undone once a source is actually selected. + """ + from app.core.openai.requests import ResponsesReasoning + + payload = ResponsesRequest.model_validate( + { + "model": "qwen3.8-max", + "instructions": "hello", + "input": [], + } + ) + payload.reasoning = ResponsesReasoning(effort="minimal") + # Snapshot is populated, but only with an unrelated subscription model. + registry = _build_registry_with_model("gpt-5.5", ["low", "medium", "high", "xhigh"]) + + replaced = proxy_request_policy.normalize_unsupported_reasoning_effort(payload, registry=registry) + + assert payload.reasoning is not None + assert payload.reasoning.effort == "low" + assert replaced == "minimal", "the replaced effort must be reported so a source route can restore it" + + +def test_normalize_unsupported_reasoning_effort_still_rewrites_known_subscription_model(): + """The workaround must stay in place for models the registry does know.""" + from app.core.openai.requests import ResponsesReasoning + + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.5", + "instructions": "hello", + "input": [], + } + ) + payload.reasoning = ResponsesReasoning(effort="minimal") + registry = _build_registry_with_model("gpt-5.5", ["low", "medium", "high", "xhigh"]) + + proxy_request_policy.normalize_unsupported_reasoning_effort(payload, registry=registry) + + assert payload.reasoning is not None + assert payload.reasoning.effort == "low" + + +def _reasoning_model_source(levels: list[str]) -> "ModelSource": + import json as _json + + from app.core.openai.model_registry import MODEL_SOURCE_KIND_OPENAI_COMPATIBLE + from app.db.models import ModelSource, ModelSourceModel + + return ModelSource( + id="src_restore", + name="Restore", + kind=MODEL_SOURCE_KIND_OPENAI_COMPATIBLE, + base_url="http://127.0.0.1:8000/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=True, + supports_audio_transcriptions=False, + models=[ + ModelSourceModel( + model="qwen3.8-max", + is_enabled=True, + supports_streaming=True, + raw_metadata_json=_json.dumps({"supports_reasoning": True, "supported_reasoning_levels": levels}), + ) + ], + ) + + +def _payload_with_effort(model: str, effort: str): + from app.core.openai.requests import ResponsesReasoning + + payload = ResponsesRequest.model_validate({"model": model, "instructions": "hello", "input": []}) + payload.reasoning = ResponsesReasoning(effort=effort) + return payload + + +def test_restore_source_reasoning_effort_undoes_the_rewrite_for_a_declared_effort(): + """The workaround targets a ChatGPT backend quirk that model sources do not + have, so a source that declared the effort must receive it unchanged.""" + payload = _payload_with_effort("qwen3.8-max", "minimal") + registry = _build_registry_with_model("gpt-5.5", ["low", "medium", "high", "xhigh"]) + + replaced = proxy_request_policy.normalize_unsupported_reasoning_effort(payload, registry=registry) + assert payload.reasoning is not None and payload.reasoning.effort == "low" + + proxy_request_policy.restore_source_reasoning_effort( + payload, + _reasoning_model_source(["minimal", "low", "high"]), + pre_normalization_effort=replaced, + ) + assert payload.reasoning.effort == "minimal" + + +def test_restore_source_reasoning_effort_skips_an_undeclared_effort(): + """Sources without the effort in their declared set keep the safe value, so + a source that never advertised ``minimal`` is not sent it.""" + payload = _payload_with_effort("qwen3.8-max", "minimal") + registry = _build_registry_with_model("gpt-5.5", ["low", "medium", "high", "xhigh"]) + + replaced = proxy_request_policy.normalize_unsupported_reasoning_effort(payload, registry=registry) + proxy_request_policy.restore_source_reasoning_effort( + payload, + _reasoning_model_source(["low", "high"]), + pre_normalization_effort=replaced, + ) + assert payload.reasoning is not None and payload.reasoning.effort == "low" + + +def test_ultra_alias_is_never_restored_for_a_source(): + """The ultra -> max alias must hold on every upstream surface. + + An existing requirement makes the proxy forward ``ultra`` as ``max`` on any + outbound Responses payload, with no source carve-out, and real Codex clients + already rewrite it client-side. So the normalizer must not report the alias + as restorable, even for a source that declares ``ultra``. + """ + payload = _payload_with_effort("qwen3.8-max", "ultra") + registry = _build_registry_with_model("gpt-5.5", ["low", "medium", "high", "xhigh"]) + + replaced = proxy_request_policy.normalize_unsupported_reasoning_effort(payload, registry=registry) + + assert payload.reasoning is not None and payload.reasoning.effort == "max" + assert replaced is None, "the wire alias must not be reported as restorable" + + proxy_request_policy.restore_source_reasoning_effort( + payload, + _reasoning_model_source(["ultra", "max", "high"]), + pre_normalization_effort=replaced, + ) + assert payload.reasoning.effort == "max" + + +def test_restore_source_reasoning_effort_uses_the_normalized_effort(): + """The restored value must be normalized, not the raw client string. + + Pre-PR a source would have received the normalized rewrite, so forwarding + ``" MINIMAL "`` verbatim would be a new behaviour with no operator opt-in. + """ + payload = _payload_with_effort("qwen3.8-max", " MINIMAL ") + registry = _build_registry_with_model("gpt-5.5", ["low", "medium", "high", "xhigh"]) + + replaced = proxy_request_policy.normalize_unsupported_reasoning_effort(payload, registry=registry) + assert replaced == "minimal" + + proxy_request_policy.restore_source_reasoning_effort( + payload, + _reasoning_model_source(["minimal", "low"]), + pre_normalization_effort=replaced, + ) + assert payload.reasoning is not None and payload.reasoning.effort == "minimal" + + +def test_restore_source_reasoning_effort_cannot_resurrect_an_enforced_effort(): + """The captured value is post-enforcement, so an API key that pinned an + effort still wins after the restore.""" + from app.core.openai.requests import ResponsesReasoning + + payload = _payload_with_effort("qwen3.8-max", "minimal") + api_key = proxy_service.ApiKeyData( + id="key_effort", + name="effort-enforcement-key", + key_prefix="sk-clb-test", + allowed_models=None, + enforced_model=None, + enforced_reasoning_effort="high", + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + + replaced = proxy_request_policy.apply_api_key_enforcement(payload, api_key).pre_normalization_reasoning_effort + + assert payload.reasoning is not None and payload.reasoning.effort == "high" + assert replaced is None, "nothing was rewritten, so there is nothing to restore" + + proxy_request_policy.restore_source_reasoning_effort( + payload, + _reasoning_model_source(["minimal", "low", "high"]), + pre_normalization_effort=replaced, + ) + assert payload.reasoning.effort == "high" + assert isinstance(payload.reasoning, ResponsesReasoning) + + def test_normalize_unsupported_reasoning_effort_falls_back_to_low_without_registry(): from app.core.openai.model_registry import ModelRegistry from app.core.openai.requests import ResponsesReasoning @@ -2480,6 +2853,156 @@ async def test_resolve_websocket_previous_response_owner_cache_hit_keeps_owner_s assert websocket_helpers_module._websocket_stale_anchor_diagnostics(request_state).same_session is None +@pytest.mark.asyncio +async def test_resolve_websocket_previous_response_owner_suppresses_confirmed_stale_anchor(monkeypatch): + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_stale_anchor_cache", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + previous_response_id="resp_stale_anchor_cache", + session_id="sid-stale-anchor-cache", + ) + monkeypatch.setattr(websocket_helpers_module, "_websocket_stale_previous_response_index", {}) + websocket_helpers_module._remember_websocket_stale_previous_response( + previous_response_id=request_state.previous_response_id, + api_key_id=None, + ) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._resolve_websocket_previous_response_owner( + previous_response_id=request_state.previous_response_id, + api_key=None, + session_id=request_state.session_id, + surface="websocket_stream", + request_state=request_state, + ) + + assert request_logs.lookup_calls == [("resp_stale_anchor_cache", None, "sid-stale-anchor-cache")] + assert request_state.previous_response_owner_lookup_source == "stale_response_cache" + assert request_state.previous_response_owner_lookup_outcome == "hit" + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "stream_incomplete" + + +@pytest.mark.asyncio +async def test_resolve_websocket_previous_response_owner_revalidates_shared_owner_after_stale_cache(monkeypatch): + request_logs = _RequestLogsRecorder() + request_logs.response_owner_by_id[("resp_stale_anchor_shared_owner", None, "sid-shared-owner")] = "acc-shared" + service = proxy_service.ProxyService(_repo_factory(request_logs)) + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_stale_anchor_shared_owner", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + previous_response_id="resp_stale_anchor_shared_owner", + session_id="sid-shared-owner", + ) + monkeypatch.setattr(websocket_helpers_module, "_websocket_stale_previous_response_index", {}) + websocket_helpers_module._remember_websocket_stale_previous_response( + previous_response_id=request_state.previous_response_id, + api_key_id=None, + ) + + owner = await service._resolve_websocket_previous_response_owner( + previous_response_id=request_state.previous_response_id, + api_key=None, + session_id=request_state.session_id, + surface="websocket_stream", + request_state=request_state, + ) + + assert owner == "acc-shared" + assert request_logs.lookup_calls == [("resp_stale_anchor_shared_owner", None, "sid-shared-owner")] + assert request_state.previous_response_owner_lookup_source == "request_logs" + assert request_state.previous_response_owner_lookup_outcome == "hit" + assert not websocket_helpers_module._is_websocket_stale_previous_response( + previous_response_id=request_state.previous_response_id, + api_key_id=None, + ) + + +def test_remember_websocket_previous_response_owner_invalidates_stale_anchor_cache(monkeypatch): + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + monkeypatch.setattr(websocket_helpers_module, "_websocket_stale_previous_response_index", {}) + websocket_helpers_module._remember_websocket_stale_previous_response( + previous_response_id="resp_stale_anchor_invalidate", + api_key_id="key-stale-anchor", + ) + + service._remember_websocket_previous_response_owner( + previous_response_id="resp_stale_anchor_invalidate", + api_key_id="key-stale-anchor", + account_id="acc-new-owner", + ) + + assert not websocket_helpers_module._is_websocket_stale_previous_response( + previous_response_id="resp_stale_anchor_invalidate", + api_key_id="key-stale-anchor", + ) + + +def test_websocket_stale_previous_response_cache_expires(monkeypatch): + clock = {"value": 100.0} + monkeypatch.setattr(websocket_helpers_module.time, "monotonic", lambda: clock["value"]) + monkeypatch.setattr(websocket_helpers_module, "_websocket_stale_previous_response_index", {}) + websocket_helpers_module._remember_websocket_stale_previous_response( + previous_response_id="resp_stale_anchor_expiry", + api_key_id=None, + ) + + assert websocket_helpers_module._is_websocket_stale_previous_response( + previous_response_id="resp_stale_anchor_expiry", + api_key_id=None, + ) + clock["value"] += websocket_helpers_module._WEBSOCKET_STALE_PREVIOUS_RESPONSE_CACHE_TTL_SECONDS + assert not websocket_helpers_module._is_websocket_stale_previous_response( + previous_response_id="resp_stale_anchor_expiry", + api_key_id=None, + ) + + +@pytest.mark.asyncio +async def test_resolve_websocket_previous_response_owner_force_refresh_replaces_stale_cache(): + request_logs = _RequestLogsRecorder() + request_logs.response_owner_by_id[("resp_force_refresh", None, "sid-force-refresh")] = "acc_authoritative" + # Avoid constructing the full service here: its unrelated background + # event-spool settings are intentionally omitted by several lightweight + # test settings fixtures. The owner lookup only needs the repository + # factory and the in-process owner index. + service = object.__new__(proxy_service.ProxyService) + service._repo_factory = _repo_factory(request_logs) + service._websocket_previous_response_account_index = {} + service._remember_websocket_previous_response_owner( + previous_response_id="resp_force_refresh", + api_key_id=None, + account_id="acc_stale_cache", + session_id="sid-force-refresh", + ) + + owner = await service._resolve_websocket_previous_response_owner( + previous_response_id="resp_force_refresh", + api_key=None, + session_id="sid-force-refresh", + surface="http_bridge", + force_request_log_lookup=True, + ) + + assert owner == "acc_authoritative" + assert request_logs.lookup_calls == [("resp_force_refresh", None, "sid-force-refresh")] + assert ( + service._websocket_previous_response_account_index[("resp_force_refresh", None, "sid-force-refresh")] + == "acc_authoritative" + ) + + @pytest.mark.asyncio async def test_resolve_websocket_previous_response_owner_fail_closed_records_metric_and_log(monkeypatch, caplog): request_logs = _RequestLogsRecorder() @@ -4280,6 +4803,53 @@ async def test_compact_fails_closed_when_turn_state_and_file_owners_conflict( selection.assert_not_awaited() +@pytest.mark.asyncio +async def test_compact_owner_lookup_error_survives_settlement_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = _make_proxy_settings() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + api_key = _make_api_key_data("key_compact_owner_cleanup") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_owner_cleanup", + key_id=api_key.id, + model="gpt-5.1", + ) + owner_error = proxy_module.ProxyResponseError( + 502, + openai_error("file_owner_unavailable", "owner lookup failed"), + ) + + async def fail_owner(*args: object, **kwargs: object) -> None: + del args, kwargs + raise owner_error + + async def fail_settle(*args: object, **kwargs: object) -> None: + del args, kwargs + raise proxy_module.ProxyResponseError( + 502, + openai_error("usage_settlement_failed", "Compact API key usage could not be settled"), + failure_phase="usage_settlement", + ) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_resolve_forwarded_file_account_for_responses", fail_owner) + monkeypatch.setattr(service, "_settle_compact_api_key_usage", fail_settle) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service.compact_responses( + payload, + {}, + api_key=api_key, + api_key_reservation=reservation, + ) + + assert exc_info.value is owner_error + assert _proxy_error_code(exc_info.value) == "file_owner_unavailable" + + @pytest.mark.asyncio async def test_compact_real_turn_state_still_blocks_file_pin_fallback(monkeypatch: pytest.MonkeyPatch) -> None: settings = _make_proxy_settings() @@ -4677,6 +5247,32 @@ def __init__(self, chunks: Sequence[bytes]) -> None: self.content = _DummyContent(chunks) +@pytest.mark.asyncio +async def test_compact_sse_terminal_error_preserves_error_envelope() -> None: + response = _DummyResponse( + [ + ( + b'data: {"type":"response.failed","response":{"status_code":400,' + b'"error":{"code":"previous_response_not_found","message":"missing anchor",' + b'"param":"previous_response_id"}}}\n\n' + ) + ] + ) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await proxy_module._compact_response_payload_from_sse( + response, + idle_timeout_seconds=1.0, + max_event_bytes=4096, + ) + + exc = _assert_proxy_response_error(exc_info.value) + assert exc.status_code == 400 + assert _proxy_error_code(exc) == "previous_response_not_found" + assert _proxy_error_message(exc) == "missing anchor" + assert exc.payload["error"]["param"] == "previous_response_id" + + class _TranscribeResponse: def __init__( self, @@ -5524,6 +6120,9 @@ async def test_select_codex_control_account_without_budget_uses_balancer(monkeyp reallocate_sticky=False, sticky_source=None, legacy_sticky_key=None, + legacy_continuity_source=None, + sticky_seed_key=None, + sticky_seed_kind=None, sticky_max_age_seconds=123, prefer_earlier_reset_window="primary", routing_strategy="usage_weighted", @@ -5656,6 +6255,47 @@ async def thread_goal_request(*_args: object, **_kwargs: object) -> dict[str, Js assert request_logs.calls[0]["conversation_id"] == "conv-thread-goal" +@pytest.mark.asyncio +async def test_thread_goal_request_routes_from_payload_thread_identity(monkeypatch: pytest.MonkeyPatch) -> None: + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc-thread-goal-payload") + selection_kwargs: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **kwargs: object) -> AccountSelection: + selection_kwargs.append(kwargs) + return AccountSelection(account=account, error_message=None) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + monkeypatch.setattr(proxy_service, "core_thread_goal_request", AsyncMock(return_value={"ok": True})) + + payload = {"threadId": "thread-from-payload", "goal": "finish"} + headers = { + "session-id": "process-shared", + # The payload is the endpoint's exact subject and must not be replaced + # by a broader or stale generic request header. + "thread-id": "thread-from-header", + "user-agent": "codex/1.2", + } + response = await service.thread_goal_request("set", payload, headers) + + assert response == {"ok": True} + affinity = cast(proxy_service._AffinityPolicy, selection_kwargs[0]["affinity_policy"]) + expected_key = proxy_affinity._codex_backend_identity( + headers, + thread_id="thread-from-payload", + ).thread_selection_key + assert affinity.selection_key == expected_key + assert affinity.codex_session_source == "thread_header" + assert affinity.legacy_selection_key == "process-shared" + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["conversation_id"] == "thread-from-header" + + @pytest.mark.asyncio async def test_thread_goal_401_failover_preserves_dashboard_reset_window(monkeypatch): settings = _make_proxy_settings() @@ -5807,17 +6447,31 @@ async def json(self, *, content_type=None): return self._payload -class _CompactSession: - class _CompactResponseLike(Protocol): - status: int - - async def __aenter__(self) -> Self: ... - - async def __aexit__(self, exc_type: object | None, exc: BaseException | None, tb: object | None) -> bool: ... +def _compact_sse_response( + *, + response_id: str = "resp_compact_1", + encrypted_content: str = "enc_summary_1", +) -> "_SsePostResponse": + return _SsePostResponse( + [ + ( + b'data: {"type":"response.output_item.done","output_index":0,' + b'"item":{"id":"msg_compact_1","type":"message","role":"assistant",' + b'"status":"completed","content":[{"type":"output_text","text":"' + + encrypted_content.encode() + + b'"}]}}\n\n' + ), + ( + b'data: {"type":"response.completed","response":{"object":"response","id":"' + + response_id.encode() + + b'","status":"completed","output":[]}}\n\n' + ), + ] + ) - async def json(self, *, content_type: str | None = None) -> dict[str, object]: ... - def __init__(self, response: _CompactResponseLike) -> None: +class _CompactSession: + def __init__(self, response: object) -> None: self._response = response self.calls: list[dict[str, object]] = [] @@ -5851,6 +6505,7 @@ def request( class _SsePostResponse: def __init__(self, chunks: list[bytes]) -> None: self.status = 200 + self.headers = {"content-type": "text/event-stream"} self.content = _DummyContent(chunks) async def __aenter__(self): @@ -8330,7 +8985,7 @@ async def fake_stream_websocket_events( del enforce_openai_sdk_contract recorded["total_timeout_seconds"] = total_timeout_seconds if False: - yield "" + yield "", None monkeypatch.setattr(proxy_module, "_open_upstream_websocket", fake_open_upstream_websocket) monkeypatch.setattr(proxy_module, "_stream_websocket_events", fake_stream_websocket_events) @@ -8409,7 +9064,9 @@ async def fake_open_upstream_websocket( ] assert len(events) == 1 - assert parse_sse_data_json(events[0]) == raw_payload + event_block, event_type = events[0] + assert parse_sse_data_json(event_block) == raw_payload + assert event_type == "error" assert successes == 1 assert failures == [] @@ -8439,10 +9096,111 @@ async def test_stream_codex_websocket_events_treats_raw_error_as_terminal_when_s ] assert len(events) == 1 - assert parse_sse_data_json(events[0]) == raw_payload + event_block, event_type = events[0] + assert parse_sse_data_json(event_block) == raw_payload + assert event_type == "error" assert websocket._index == 1 +@pytest.mark.asyncio +async def test_stream_responses_websocket_decodes_each_frame_once_and_skips_error_validation(monkeypatch): + # Regression for the parse-once requirement on the websocket hot path: + # each upstream frame is json-decoded exactly once in the receive loop, + # the relay and outer stream loops reuse the threaded event type instead + # of re-parsing the formatted block, and no error-envelope validation + # runs for non-error-shaped frames. + class Settings: + upstream_base_url = "https://chatgpt.com/backend-api" + upstream_stream_transport = "websocket" + upstream_connect_timeout_seconds = 8.0 + stream_idle_timeout_seconds = 45.0 + max_sse_event_bytes = 1024 + image_inline_fetch_enabled = False + trace_channels = frozenset() + proxy_request_budget_seconds = 75.0 + + monkeypatch.setattr(proxy_module, "get_settings", lambda: Settings()) + monkeypatch.setattr(proxy_module, "_maybe_log_upstream_request_start", lambda **kwargs: None) + monkeypatch.setattr(proxy_module, "_maybe_log_upstream_request_complete", lambda **kwargs: None) + + frame_payloads = [ + {"type": "response.created", "response": {"id": "resp_ws_once"}}, + {"type": "response.output_text.delta", "delta": "a"}, + {"type": "response.output_text.delta", "delta": "b"}, + {"type": "response.completed", "response": {"id": "resp_ws_once"}}, + ] + frame_texts = [json.dumps(payload, ensure_ascii=True, separators=(",", ":")) for payload in frame_payloads] + websocket = _WsResponse([_WsMessage(proxy_module.aiohttp.WSMsgType.TEXT, text) for text in frame_texts]) + session = _WsSession(websocket) + + loads_spy = MagicMock(wraps=json.loads) + monkeypatch.setattr(proxy_module.json, "loads", loads_spy) + parse_spy = MagicMock(wraps=proxy_module.parse_sse_data_json) + monkeypatch.setattr(proxy_module, "parse_sse_data_json", parse_spy) + error_validate_spy = MagicMock(wraps=proxy_module.parse_error_payload) + monkeypatch.setattr(proxy_module, "parse_error_payload", error_validate_spy) + + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "hi", + "input": [{"role": "user", "content": "hi"}], + "stream": True, + } + ) + events = [ + event + async for event in proxy_module.stream_responses( + payload, + headers={}, + access_token="token", + account_id="acc_ws_parse_once", + session=cast(proxy_module.aiohttp.ClientSession, session), + ) + ] + + assert events == [proxy_module.format_sse_event(payload) for payload in frame_payloads] + for text in frame_texts: + decode_calls = [c for c in loads_spy.call_args_list if c.args and c.args[0] == text] + assert len(decode_calls) == 1 + assert parse_spy.call_count == 0 + assert error_validate_spy.call_count == 0 + + +def test_normalize_stream_event_payload_validates_error_envelope_only_for_error_shaped_frames(monkeypatch): + # Delta frames never reach the pydantic error-envelope adapter; error + # frames are still validated and rewritten exactly as before. + from app.core.openai import parsing as parsing_module + + adapter_spy = MagicMock(wraps=parsing_module._ERROR_ADAPTER) + monkeypatch.setattr(parsing_module, "_ERROR_ADAPTER", adapter_spy) + + delta_payload: dict[str, Any] = {"type": "response.output_text.delta", "delta": "a"} + assert proxy_module._normalize_stream_event_payload(delta_payload) is delta_payload + in_progress_payload: dict[str, Any] = {"type": "response.in_progress", "response": {"id": "resp_np"}} + assert proxy_module._normalize_stream_event_payload(in_progress_payload) is in_progress_payload + assert adapter_spy.validate_python.call_count == 0 + + envelope_payload: dict[str, Any] = { + "error": {"message": "quota exhausted", "type": "server_error", "code": "insufficient_quota"} + } + rewritten: Any = proxy_module._normalize_stream_event_payload(envelope_payload) + assert adapter_spy.validate_python.call_count == 1 + assert rewritten["type"] == "response.failed" + envelope_error = rewritten["response"]["error"] + assert envelope_error["message"] == "quota exhausted" + assert envelope_error["code"] == proxy_module._normalize_error_code("insufficient_quota", "server_error") + assert envelope_error["type"] == "server_error" + + bare_error_payload: dict[str, Any] = {"type": "error", "code": "rate_limit_exceeded", "message": "slow down"} + rewritten_bare: Any = proxy_module._normalize_stream_event_payload(bare_error_payload) + assert adapter_spy.validate_python.call_count == 2 + assert rewritten_bare["type"] == "response.failed" + bare_error = rewritten_bare["response"]["error"] + assert bare_error["code"] == proxy_module._normalize_error_code("rate_limit_exceeded", "error") + assert bare_error["message"] == "slow down" + + @pytest.mark.asyncio async def test_stream_responses_websocket_broken_pipe_is_not_replayable_upstream_unavailable(monkeypatch): logged_completions: list[dict[str, object]] = [] @@ -9884,6 +10642,8 @@ class Settings: upstream_base_url = "https://chatgpt.com/backend-api" upstream_connect_timeout_seconds = 1.0 upstream_compact_timeout_seconds = 12.0 + stream_idle_timeout_seconds = 45.0 + max_sse_event_bytes = 1024 image_inline_fetch_enabled = True trace_channels = frozenset() @@ -9912,11 +10672,7 @@ def fake_complete(**kwargs): payload = proxy_module.ResponsesCompactRequest.model_validate( {"model": "gpt-5.1", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} ) - session = _CompactSession( - _JsonCompactResponse( - {"object": "response.compaction", "compaction_summary": {"encrypted_content": "enc_summary_1"}} - ) - ) + session = _CompactSession(_compact_sse_response(encrypted_content="enc_summary_1")) result = await proxy_module.compact_responses( payload, @@ -9933,7 +10689,7 @@ def fake_complete(**kwargs): assert timeout.sock_read == pytest.approx(6.5) dumped = result.model_dump(mode="json", exclude_none=True) assert dumped["object"] == "response.compaction" - assert dumped["compaction_summary"]["encrypted_content"] == "enc_summary_1" + assert dumped["output"][0]["encrypted_content"] == "enc_summary_1" assert recorded["started_at"] == 205.5 @@ -9943,6 +10699,8 @@ class Settings: upstream_base_url = "https://chatgpt.com/backend-api" upstream_connect_timeout_seconds = 1.0 upstream_compact_timeout_seconds = 12.0 + stream_idle_timeout_seconds = 45.0 + max_sse_event_bytes = 1024 image_inline_fetch_enabled = False trace_channels = frozenset() @@ -9979,11 +10737,7 @@ class Settings: "reasoning": {"context": "turn", "effort": "high", "vendor_hint": 7}, } ) - session = _CompactSession( - _JsonCompactResponse( - {"object": "response.compaction", "compaction_summary": {"encrypted_content": "enc_summary_1"}} - ) - ) + session = _CompactSession(_compact_sse_response(encrypted_content="enc_summary_1")) await proxy_module.compact_responses( payload, @@ -10005,6 +10759,281 @@ class Settings: } +@pytest.mark.asyncio +async def test_compact_responses_normalizes_preexisting_terminal_trigger_without_losing_tool_tail(monkeypatch): + class Settings: + upstream_base_url = "https://chatgpt.com/backend-api" + upstream_connect_timeout_seconds = 1.0 + upstream_compact_timeout_seconds = 12.0 + stream_idle_timeout_seconds = 45.0 + max_sse_event_bytes = 1024 + image_inline_fetch_enabled = False + trace_channels = frozenset() + + monkeypatch.setattr(proxy_module, "get_settings", lambda: Settings()) + monkeypatch.setattr(proxy_module, "_maybe_log_upstream_request_start", lambda **kwargs: None) + monkeypatch.setattr(proxy_module, "_maybe_log_upstream_request_complete", lambda **kwargs: None) + + payload = proxy_module.ResponsesCompactRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "hi", + "input": [ + {"role": "user", "content": "hello"}, + { + "type": "custom_tool_call", + "name": "functions.exec_command", + "call_id": "call_compact_tail", + "input": '{"cmd":"pytest"}', + }, + { + "type": "custom_tool_call_output", + "call_id": "call_compact_tail", + "output": "failed", + }, + {"type": "compaction_trigger"}, + ], + } + ) + session = _CompactSession( + _JsonCompactResponse( + {"object": "response.compaction", "compaction_summary": {"encrypted_content": "enc_summary_1"}} + ) + ) + + await proxy_module.compact_responses( + payload, + headers={}, + access_token="token", + account_id="acc_1", + session=cast(proxy_module.aiohttp.ClientSession, session), + ) + + upstream_payload = cast(dict[str, object], session.calls[0]["json"]) + compact_input = cast(list[dict[str, object]], upstream_payload["input"]) + assert compact_input[-1] == {"type": "compaction_trigger"} + assert sum(1 for item in compact_input if item.get("type") == "compaction_trigger") == 1 + assert compact_input[:-1] == [ + {"role": "user", "content": "hello"}, + { + "type": "custom_tool_call", + "name": "functions.exec_command", + "call_id": "call_compact_tail", + "input": '{"cmd":"pytest"}', + }, + { + "type": "custom_tool_call_output", + "call_id": "call_compact_tail", + "output": "failed", + }, + ] + + +def test_responses_wire_payload_normalizes_internal_duplicate_compaction_trigger(): + payload = proxy_module.ResponsesRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "compact", + "input": [ + {"role": "user", "content": "hello"}, + {"type": "compaction_trigger"}, + {"type": "compaction_trigger"}, + ], + "stream": True, + "store": False, + } + ).to_payload() + + compact_input = cast(list[dict[str, object]], payload["input"]) + assert compact_input == [ + {"role": "user", "content": "hello"}, + {"type": "compaction_trigger"}, + ] + + +def test_compact_wire_payload_omits_oversized_optional_tool_tail_even_with_terminal_trigger(): + payload = proxy_module.ResponsesCompactRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "", + "input": [ + { + "type": "function_call", + "name": "read_file", + "call_id": "call_route_tail", + "arguments": "x" * 450_000, + }, + { + "type": "function_call_output", + "call_id": "call_route_tail", + "output": "latest ordinary result", + }, + {"type": "compaction_trigger"}, + ], + } + ) + + dumped = payload.to_payload() + compact_input = cast(list[dict[str, object]], dumped["input"]) + + assert compact_input[-1] == {"type": "compaction_trigger"} + assert all(item.get("call_id") != "call_route_tail" for item in compact_input if isinstance(item, dict)) + assert "[compact trim]" in json.dumps(compact_input) + assert "latest ordinary result" not in json.dumps(compact_input) + + +def test_compact_discard_consumed_continuity_output_pairs_treats_terminal_trigger_as_suffix(): + input_items: list[JsonValue] = [ + { + "type": "function_call", + "name": "read_file", + "call_id": "call_reused", + "arguments": '{"path":"old.txt"}', + }, + { + "type": "function_call_output", + "call_id": "call_reused", + "output": "consumed old output", + }, + { + "type": "function_call_output", + "call_id": "call_reused", + "output": "latest output from the prior response", + }, + {"type": "compaction_trigger"}, + ] + selected_indices = {0, 1, 2, 3} + required_indices = {2, 3} + + openai_requests_module._compact_discard_consumed_continuity_output_pairs( + input_items, + selected_indices=selected_indices, + required_indices=required_indices, + ) + + assert selected_indices == {2, 3} + + +@pytest.mark.asyncio +async def test_compact_responses_without_trigger_canonicalization_hits_upstream_duplicate_trigger_error(monkeypatch): + class Settings: + upstream_base_url = "https://chatgpt.com/backend-api" + upstream_connect_timeout_seconds = 1.0 + upstream_compact_timeout_seconds = 12.0 + image_inline_fetch_enabled = False + trace_channels = frozenset() + + class _DuplicateTriggerResponse: + status = 400 + reason = "Bad Request" + + def __init__(self, payload: dict[str, object]) -> None: + self._payload = payload + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + async def json(self, *, content_type=None): + return self._payload + + class _DuplicateTriggerSession: + def __init__(self) -> None: + self.calls: list[dict[str, object]] = [] + + def post(self, url: str, *, json=None, headers=None, timeout=None): + self.calls.append({"url": url, "json": json, "headers": headers, "timeout": timeout}) + upstream_payload = cast(dict[str, object], json) + compact_input = cast(list[dict[str, object]], upstream_payload["input"]) + trigger_count = sum(1 for item in compact_input if item.get("type") == "compaction_trigger") + if trigger_count > 1: + return _DuplicateTriggerResponse( + { + "error": { + "message": "Only one 'compaction_trigger' item may be provided", + "type": "invalid_request_error", + "param": "input", + "code": "invalid_request_error", + } + } + ) + return _JsonCompactResponse( + {"object": "response.compaction", "compaction_summary": {"encrypted_content": "enc_summary_1"}} + ) + + monkeypatch.setattr(proxy_module, "get_settings", lambda: Settings()) + monkeypatch.setattr(proxy_module, "_maybe_log_upstream_request_start", lambda **kwargs: None) + monkeypatch.setattr(proxy_module, "_maybe_log_upstream_request_complete", lambda **kwargs: None) + + payload = proxy_module.ResponsesCompactRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "hi", + "input": [ + {"role": "user", "content": "hello"}, + { + "type": "custom_tool_call", + "name": "functions.exec_command", + "call_id": "call_compact_tail", + "input": '{"cmd":"pytest"}', + }, + { + "type": "custom_tool_call_output", + "call_id": "call_compact_tail", + "output": "failed", + }, + {"type": "compaction_trigger"}, + {"type": "compaction_trigger"}, + ], + } + ) + monkeypatch.setattr( + payload, + "to_payload", + lambda: { + "model": "gpt-5.1", + "instructions": "hi", + "input": [ + {"role": "user", "content": "hello"}, + { + "type": "custom_tool_call", + "name": "functions.exec_command", + "call_id": "call_compact_tail", + "input": '{"cmd":"pytest"}', + }, + { + "type": "custom_tool_call_output", + "call_id": "call_compact_tail", + "output": "failed", + }, + {"type": "compaction_trigger"}, + {"type": "compaction_trigger"}, + ], + "parallel_tool_calls": False, + }, + ) + session = _DuplicateTriggerSession() + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await proxy_module.compact_responses( + payload, + headers={}, + access_token="token", + account_id="acc_1", + session=cast(proxy_module.aiohttp.ClientSession, session), + ) + + exc = _assert_proxy_response_error(exc_info.value) + assert exc.status_code == 400 + assert _proxy_error_code(exc) == "invalid_request_error" + assert _proxy_error_message(exc) == "Only one 'compaction_trigger' item may be provided" + upstream_payload = cast(dict[str, object], session.calls[0]["json"]) + compact_input = cast(list[dict[str, object]], upstream_payload["input"]) + assert sum(1 for item in compact_input if item.get("type") == "compaction_trigger") == 2 + + @pytest.mark.asyncio async def test_compact_responses_rejects_image_inlining_that_exceeds_wire_budget(monkeypatch): class Settings: @@ -10074,11 +11103,14 @@ class Settings: upstream_base_url = "https://chatgpt.com/backend-api" upstream_connect_timeout_seconds = 2.0 upstream_compact_timeout_seconds = 123.0 + stream_idle_timeout_seconds = 45.0 + max_sse_event_bytes = 1024 image_inline_fetch_enabled = False trace_channels = frozenset() class _TimeoutCompactResponse: status = 200 + headers: dict[str, str] = {} async def __aenter__(self): return self @@ -10086,8 +11118,13 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc, tb): return False - async def json(self, *, content_type=None): - raise proxy_module.aiohttp.SocketTimeoutError("Timeout on reading data from socket") + class _TimeoutContent: + async def iter_chunked(self, size: int): + del size + raise proxy_module.aiohttp.SocketTimeoutError("Timeout on reading data from socket") + yield b"" + + content = _TimeoutContent() monkeypatch.setattr(proxy_module, "get_settings", lambda: Settings()) monkeypatch.setattr(proxy_module, "_maybe_log_upstream_request_start", lambda **kwargs: None) @@ -10124,6 +11161,8 @@ class Settings: upstream_base_url = "https://chatgpt.com/backend-api" upstream_connect_timeout_seconds = 2.0 upstream_compact_timeout_seconds = None + stream_idle_timeout_seconds = 45.0 + max_sse_event_bytes = 1024 image_inline_fetch_enabled = False trace_channels = frozenset() @@ -10134,11 +11173,7 @@ class Settings: payload = proxy_module.ResponsesCompactRequest.model_validate( {"model": "gpt-5.1", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} ) - session = _CompactSession( - _JsonCompactResponse( - {"object": "response.compaction", "compaction_summary": {"encrypted_content": "enc_summary_2"}} - ) - ) + session = _CompactSession(_compact_sse_response(response_id="resp_compact_2", encrypted_content="enc_summary_2")) result = await proxy_module.compact_responses( payload, @@ -10155,7 +11190,103 @@ class Settings: assert timeout.sock_read is None dumped = result.model_dump(mode="json", exclude_none=True) assert dumped["object"] == "response.compaction" - assert dumped["compaction_summary"]["encrypted_content"] == "enc_summary_2" + assert dumped["output"][0]["encrypted_content"] == "enc_summary_2" + + +@pytest.mark.asyncio +async def test_compact_responses_preserves_stream_idle_timeout_from_direct_sse(monkeypatch): + class Settings: + upstream_base_url = "https://chatgpt.com/backend-api" + upstream_connect_timeout_seconds = 2.0 + upstream_compact_timeout_seconds = 123.0 + stream_idle_timeout_seconds = 45.0 + max_sse_event_bytes = 1024 + image_inline_fetch_enabled = False + trace_channels = frozenset() + + async def fake_compact_response_payload_from_success_response(*args: object, **kwargs: object) -> JsonValue: + del args, kwargs + raise proxy_module.StreamIdleTimeoutError() + + monkeypatch.setattr(proxy_module, "get_settings", lambda: Settings()) + monkeypatch.setattr(proxy_module, "_maybe_log_upstream_request_start", lambda **kwargs: None) + monkeypatch.setattr(proxy_module, "_maybe_log_upstream_request_complete", lambda **kwargs: None) + monkeypatch.setattr( + proxy_module, + "_compact_response_payload_from_success_response", + fake_compact_response_payload_from_success_response, + ) + + payload = proxy_module.ResponsesCompactRequest.model_validate( + {"model": "gpt-5.1", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} + ) + session = _CompactSession(_compact_sse_response()) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await proxy_module.compact_responses( + payload, + headers={}, + access_token="token", + account_id="acc_1", + session=cast(proxy_module.aiohttp.ClientSession, session), + ) + + exc = _assert_proxy_response_error(exc_info.value) + assert exc.status_code == 502 + assert _proxy_error_code(exc) == "stream_idle_timeout" + assert _proxy_error_message(exc) == "Upstream stream idle timeout" + + +@pytest.mark.asyncio +async def test_compact_responses_preserves_stream_event_too_large_from_routed_sse(monkeypatch): + class Settings: + upstream_base_url = "https://chatgpt.com/backend-api" + upstream_connect_timeout_seconds = 2.0 + upstream_compact_timeout_seconds = 123.0 + stream_idle_timeout_seconds = 45.0 + max_sse_event_bytes = 1024 + image_inline_fetch_enabled = False + trace_channels = frozenset() + + monkeypatch.setattr(proxy_module, "get_settings", lambda: Settings()) + monkeypatch.setattr(proxy_module, "_maybe_log_upstream_request_start", lambda **kwargs: None) + monkeypatch.setattr(proxy_module, "_maybe_log_upstream_request_complete", lambda **kwargs: None) + + route = ResolvedUpstreamRoute( + mode="account_bound", + pool_id="pool_compact", + endpoint=ResolvedProxyEndpoint("ep_compact", "http", "proxy.test", 8080), + ) + oversized_response = _SsePostResponse([b"data: " + b"x" * 2048 + b"\n\n"]) + codex_client = SimpleNamespace( + request_with_route_metadata=AsyncMock( + return_value=SimpleNamespace( + response=oversized_response, + route=route, + fallback_used=False, + ) + ) + ) + payload = proxy_module.ResponsesCompactRequest.model_validate( + {"model": "gpt-5.1", "instructions": "hi", "input": [{"role": "user", "content": "hi"}]} + ) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await proxy_module.compact_responses( + payload, + headers={}, + access_token="token", + account_id="acc_1", + session=cast(proxy_module.aiohttp.ClientSession, object()), + route=route, + codex_client=cast(proxy_module.CodexClient, codex_client), + allow_direct_egress=False, + ) + + exc = _assert_proxy_response_error(exc_info.value) + assert exc.status_code == 502 + assert _proxy_error_code(exc) == "stream_event_too_large" + assert "SSE event exceeded" in (_proxy_error_message(exc) or "") def test_sticky_key_for_responses_request_uses_bounded_cache_affinity(): @@ -10196,6 +11327,204 @@ def test_sticky_key_for_responses_request_keeps_sticky_threads_durable(): assert policy.max_age_seconds is None +def _goal_restart_payload(**updates: object) -> ResponsesRequest: + payload: dict[str, object] = { + "model": "gpt-5.1", + "instructions": "Continue the existing task.", + "input": [ + { + "role": "developer", + "content": ('\nContinue working toward the active thread goal.'), + }, + {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ], + "stream": True, + **updates, + } + return ResponsesRequest.model_validate(payload) + + +def test_goal_restart_affinity_can_abandon_only_legacy_session_owner(): + payload = _goal_restart_payload() + + policy = proxy_service._sticky_key_for_responses_request( + payload, + headers={"session_id": "goal-restart-session"}, + codex_session_affinity=True, + openai_cache_affinity=False, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + turn_state_policy = proxy_service._sticky_key_for_responses_request( + payload, + headers={ + "session_id": "goal-restart-session", + "x-codex-turn-state": "explicit-turn-owner", + }, + codex_session_affinity=True, + openai_cache_affinity=False, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + + thread_policy = proxy_service._sticky_key_for_responses_request( + payload, + headers={ + "session_id": "goal-restart-session", + "thread-id": "goal-restart-thread", + }, + codex_session_affinity=True, + openai_cache_affinity=False, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + thread_only_policy = proxy_service._sticky_key_for_responses_request( + payload, + headers={"thread-id": "goal-restart-thread"}, + codex_session_affinity=True, + openai_cache_affinity=False, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + + assert policy.codex_session_source == "session_header" + assert policy.abandon_unavailable_legacy_owner is True + assert thread_policy.codex_session_source == "thread_header" + assert thread_policy.abandon_unavailable_legacy_owner is True + assert thread_policy.legacy_selection_key == "goal-restart-session" + assert thread_policy.legacy_continuity_source == "session_header" + assert thread_only_policy.codex_session_source == "thread_header" + assert thread_only_policy.abandon_unavailable_legacy_owner is False + assert thread_only_policy.legacy_continuity_source == "thread_header" + assert turn_state_policy.codex_session_source == "turn_state" + assert turn_state_policy.abandon_unavailable_legacy_owner is False + + +def test_goal_restart_affinity_normalizes_accepted_compatibility_controls_before_replay_classification(): + payload = _goal_restart_payload( + max_output_tokens=512, + promptCacheKey="goal-restart-cache", + promptCacheRetention="12h", + ) + + policy = proxy_service._sticky_key_for_responses_request( + payload, + headers={"session_id": "goal-restart-session"}, + codex_session_affinity=True, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + + assert policy.codex_session_source == "session_header" + assert policy.abandon_unavailable_legacy_owner is True + + +@pytest.mark.parametrize( + ("payload_update", "extra_input"), + [ + ({"previous_response_id": "resp_owner"}, []), + ({"conversation": "conv_owner"}, []), + ({}, [{"type": "input_file", "file_id": "file_owner"}]), + ({}, [{"type": "input_image", "file_id": "file_image_owner"}]), + ( + {}, + [ + { + "type": "function_call_output", + "call_id": "call_without_matching_input", + "output": "result", + } + ], + ), + ], +) +def test_goal_restart_affinity_preserves_owner_for_account_dependent_payloads( + payload_update: dict[str, object], + extra_input: list[JsonValue], +): + payload = _goal_restart_payload(**payload_update) + assert isinstance(payload.input, list) + cast(list[JsonValue], payload.input).extend(extra_input) + + policy = proxy_service._sticky_key_for_responses_request( + payload, + headers={"session_id": "goal-restart-session"}, + codex_session_affinity=True, + openai_cache_affinity=False, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + + assert policy.abandon_unavailable_legacy_owner is False + + +def test_goal_restart_affinity_preserves_owner_for_account_dependent_thread_payloads(): + payload = _goal_restart_payload(previous_response_id="resp_owner") + + policy = proxy_service._sticky_key_for_responses_request( + payload, + headers={ + "session_id": "goal-restart-session", + "thread-id": "goal-restart-thread", + }, + codex_session_affinity=True, + openai_cache_affinity=False, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + + assert policy.codex_session_source == "thread_header" + assert policy.abandon_unavailable_legacy_owner is False + + +def test_full_resend_without_goal_marker_cannot_abandon_legacy_owner(): + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "Continue the existing task.", + "input": [{"role": "user", "content": "continue"}], + "stream": True, + } + ) + + policy = proxy_service._sticky_key_for_responses_request( + payload, + headers={"session_id": "goal-restart-session"}, + codex_session_affinity=True, + openai_cache_affinity=False, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + + assert policy.abandon_unavailable_legacy_owner is False + + +@pytest.mark.asyncio +async def test_unavailable_owner_tombstone_locks_account_status_before_retirement() -> None: + from sqlalchemy.dialects import postgresql + + fake_session = SimpleNamespace( + scalar=AsyncMock(return_value=AccountStatus.ACTIVE), + execute=AsyncMock(), + commit=AsyncMock(), + ) + repo = StickySessionsRepository(cast(Any, fake_session)) + + retired = await repo.abandon_legacy_session_header_owner_if_unavailable( + "goal-restart-lock", + kind=StickySessionKind.CODEX_SESSION, + expected_account_id="acc-recovered-before-lock", + ) + + assert retired is False + status_statement = fake_session.scalar.await_args.args[0] + compiled = str(status_statement.compile(dialect=postgresql.dialect())) + assert "FOR UPDATE" in compiled + fake_session.execute.assert_not_awaited() + fake_session.commit.assert_awaited_once() + + def test_sticky_key_for_compact_request_prefers_codex_session_affinity(): payload = ResponsesCompactRequest.model_validate( { @@ -10222,6 +11551,133 @@ def test_sticky_key_for_compact_request_prefers_codex_session_affinity(): assert policy.max_age_seconds is None +def test_backend_codex_thread_affinity_is_shared_by_responses_and_compact_without_rewriting_cache_hint() -> None: + headers_by_thread = { + "root": {"session-id": "process-shared", "thread-id": "thread-root"}, + "child": {"session-id": "process-shared", "thread-id": "thread-child"}, + } + keys_by_surface: dict[str, dict[str, str]] = {"responses": {}, "compact": {}} + + for surface, payload in ( + ( + "responses", + ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "hi", + "input": [], + "prompt_cache_key": "process-shared", + } + ), + ), + ( + "compact", + ResponsesCompactRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "hi", + "input": [], + "prompt_cache_key": "process-shared", + } + ), + ), + ): + for thread_name, headers in headers_by_thread.items(): + request_payload = payload.model_copy() + if isinstance(request_payload, ResponsesRequest): + policy = proxy_service._sticky_key_for_responses_request( + request_payload, + headers=headers, + codex_session_affinity=True, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + else: + policy = proxy_service._sticky_key_for_compact_request( + request_payload, + headers=headers, + codex_session_affinity=True, + openai_cache_affinity=True, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + + assert policy.key is not None + keys_by_surface[surface][thread_name] = policy.key + assert policy.kind == StickySessionKind.PROMPT_CACHE + assert policy.codex_session_source == "thread_header" + assert policy.legacy_selection_key == "process-shared" + assert policy.seed_selection_key == proxy_affinity._codex_session_selection_key("process-shared") + assert policy.seed_selection_kind == StickySessionKind.CODEX_SESSION + assert policy.max_age_seconds == 300 + assert request_payload.prompt_cache_key == "process-shared" + + assert keys_by_surface["responses"] == keys_by_surface["compact"] + assert keys_by_surface["responses"]["root"] != keys_by_surface["responses"]["child"] + + +def test_codex_thread_identity_is_source_separated_and_supports_thread_only_clients() -> None: + process_thread = proxy_affinity._codex_backend_identity( + {"session-id": "same-value", "thread-id": "same-value"} + ).thread_selection_key + thread_only = proxy_affinity._codex_backend_identity({"thread-id": "same-value"}).thread_selection_key + + assert process_thread is not None + assert thread_only is not None + assert process_thread != thread_only + assert process_thread.startswith("\n") + assert thread_only.startswith("\n") + assert "same-value" not in process_thread + assert "same-value" not in thread_only + + +def test_codex_thread_identity_length_frames_client_values() -> None: + left = proxy_affinity._codex_backend_identity( + { + "session-id": "process-a\0thread\0thread-b", + "thread-id": "thread-c", + } + ).thread_selection_key + right = proxy_affinity._codex_backend_identity( + { + "session-id": "process-a", + "thread-id": "thread-b\0thread\0thread-c", + } + ).thread_selection_key + + assert left is not None + assert right is not None + assert left != right + + +def test_codex_thread_identity_normalizes_header_names_and_blank_values() -> None: + identity = proxy_affinity._codex_backend_identity( + { + "SESSION-ID": " process-mixed-case ", + "THREAD-ID": " thread-mixed-case ", + } + ) + expected = proxy_affinity._codex_backend_identity( + { + "session-id": "process-mixed-case", + "thread-id": "thread-mixed-case", + } + ) + blank = proxy_affinity._codex_backend_identity( + { + "session-id": " ", + "thread-id": "\t", + } + ) + + assert identity == expected + assert identity.thread_selection_key is not None + assert blank.process_session is None + assert blank.thread_id is None + assert blank.thread_selection_key is None + + @pytest.mark.parametrize("codex_session_affinity", [False, True]) def test_sticky_key_for_compact_request_prefers_turn_state_over_session_and_cache( codex_session_affinity: bool, @@ -10681,7 +12137,7 @@ def test_logged_error_json_response_emits_proxy_error_log(caplog): assert "proxy_error_response request_id=req_proxy_error_1" in caplog.text assert "method=POST path=/v1/responses status=502" in caplog.text assert 'code="upstream_error"' in caplog.text - assert 'message="provider failed"' in caplog.text + assert "message_present=True message_length=15" in caplog.text @pytest.mark.asyncio @@ -11138,6 +12594,89 @@ async def fake_stream( assert settlement.account_health_error is False +@pytest.mark.asyncio +async def test_stream_once_keeps_first_terminal_frame_success_after_downstream_close(monkeypatch): + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_stream_first_terminal_close") + settlement = proxy_service._StreamSettlement() + completed_line = ( + 'data: {"type":"response.completed","response":{"id":"resp_first_terminal_close","status":"completed"}}\n\n' + ) + + async def fake_stream( + payload, + headers, + access_token, + account_id, + base_url=None, + raise_for_status=False, + enforce_openai_sdk_contract=True, + ): + del payload, headers, access_token, account_id, base_url, raise_for_status, enforce_openai_sdk_contract + yield completed_line + + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_stream) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": [], "stream": True}) + stream = service._stream_once( + account, + payload, + {"session_id": "sid-stream"}, + "req_stream_first_terminal_close", + False, + request_started_at=0.0, + api_key=None, + api_key_reservation=None, + settlement=settlement, + suppress_text_done_events=False, + upstream_stream_transport=None, + request_transport="http", + ) + + first_chunk = await anext(stream) + assert "event: response.completed" in first_chunk + assert '"id":"resp_first_terminal_close"' in first_chunk + await cast(Any, stream).aclose() + + assert settlement.status == "success" + assert settlement.error is None + assert settlement.account_health_error is False + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "success" + assert request_logs.calls[0]["error_code"] is None + assert request_logs.calls[0]["request_id"] == "resp_first_terminal_close" + + +@pytest.mark.asyncio +async def test_streaming_retry_cleanup_helper_finishes_task_before_reporting_cancellation(): + cleanup_started = asyncio.Event() + release_cleanup = asyncio.Event() + cleanup_finished = asyncio.Event() + + async def cleanup() -> str: + cleanup_started.set() + await release_cleanup.wait() + cleanup_finished.set() + return "settled" + + cleanup_task = asyncio.create_task(cleanup()) + waiter_task = asyncio.create_task(streaming_retry_module._await_task_deferring_cancellation(cleanup_task)) + await asyncio.wait_for(cleanup_started.wait(), timeout=1) + + waiter_task.cancel() + await asyncio.sleep(0) + assert not cleanup_task.done() + assert not waiter_task.done() + + release_cleanup.set() + result, cancellation = await asyncio.wait_for(waiter_task, timeout=1) + + assert result == "settled" + assert cancellation is not None + assert cleanup_finished.is_set() + + @pytest.mark.asyncio async def test_stream_once_marks_downstream_cancel_before_first_event(monkeypatch): request_logs = _RequestLogsRecorder() @@ -14578,7 +16117,7 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert event["type"] == "response.completed" assert event["response"]["id"] == "resp_ok" assert seen_excluded_account_ids == [set(), {account_a.id}] - record_error.assert_awaited_once_with(account_a) + record_error.assert_not_awaited() record_success.assert_awaited_once_with(account_b) assert await service.drain_persistence_tasks(timeout_seconds=1) assert [call["status"] for call in request_logs.calls] == ["error", "success"] @@ -14627,7 +16166,7 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert event["response"]["error"]["code"] == "stream_idle_timeout" assert event["response"]["error"]["message"] == "idle" assert seen_excluded_account_ids == [set(), {account.id}] - record_error.assert_awaited_once_with(account) + record_error.assert_not_awaited() record_success.assert_not_awaited() assert await service.drain_persistence_tasks(timeout_seconds=1) assert request_logs.calls[-1]["error_code"] == "stream_idle_timeout" @@ -15097,6 +16636,73 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, record_success.assert_not_awaited() +@pytest.mark.asyncio +async def test_stream_with_retry_finalizes_generated_terminal_failure_before_downstream_close(monkeypatch): + settings = _make_proxy_settings() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_retry_generated_terminal_close") + record_error = AsyncMock() + record_success = AsyncMock() + settle_stream_usage = AsyncMock(return_value=True) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_retry_generated_terminal_close", + key_id="key_retry_generated_terminal_close", + model="gpt-5.1", + ) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(streaming_retry_module.ProcessNetworkRecovery, "wait", AsyncMock(return_value=None)) + monkeypatch.setattr(service._load_balancer, "record_error", record_error) + monkeypatch.setattr(service._load_balancer, "record_success", record_success) + monkeypatch.setattr(service, "_settle_stream_api_key_usage", settle_stream_usage) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + + async def fake_stream_once(*args: object, **kwargs: object): + settlement = cast(proxy_service._StreamSettlement, kwargs["settlement"]) + settlement.downstream_visible = True + settlement.response_id = "resp_retry_generated_terminal_close" + yield ( + 'data: {"type":"response.created","response":{"id":"resp_retry_generated_terminal_close",' + '"status":"in_progress","output":[]}}\n\n' + ) + raise streaming_retry_module._TransientStreamError( + "upstream_unavailable", + {"message": "transport exploded after first event"}, + ) + + monkeypatch.setattr(service, "_stream_once", fake_stream_once) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}) + stream = service._stream_with_retry( + payload, + {"session_id": "sid-retry-generated-terminal-close"}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=reservation, + suppress_text_done_events=False, + request_transport="http", + upstream_stream_transport_override=None, + ) + + first_chunk = await anext(stream) + terminal_chunk = await anext(stream) + assert "response.created" in first_chunk + assert "response.failed" in terminal_chunk + await cast(Any, stream).aclose() + + assert await service.drain_persistence_tasks(timeout_seconds=1) + settle_stream_usage.assert_awaited_once() + record_success.assert_not_awaited() + + @pytest.mark.asyncio async def test_stream_responses_first_event_connection_reset_surfaces_without_replay(monkeypatch): settings = _make_proxy_settings() @@ -15403,7 +17009,10 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert event["response"]["id"] == "resp_reset_event" assert event["response"]["error"]["code"] == "upstream_unavailable" assert seen_excluded_account_ids == [set()] - assert request_logs.calls == [] + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "error" + assert request_logs.calls[0]["account_id"] == account_a.id + assert request_logs.calls[0]["error_code"] == "upstream_unavailable" record_error.assert_not_awaited() record_errors.assert_not_awaited() record_success.assert_not_awaited() @@ -15783,6 +17392,134 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert authorized_lease in released_leases +@pytest.mark.asyncio +async def test_stream_responses_account_bound_pre_dispatch_failure_retries_other_account(monkeypatch): + settings = _make_proxy_settings() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + first_account = _make_account("acc_pre_dispatch_bound_first") + second_account = _make_account("acc_pre_dispatch_bound_second") + select_account = AsyncMock( + side_effect=[ + AccountSelection(account=first_account, error_message=None), + AccountSelection(account=second_account, error_message=None), + ] + ) + attempted_account_ids: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False): + del payload, headers, access_token, base_url, raise_for_status + attempted_account_ids.append(account_id) + if account_id == first_account.chatgpt_account_id: + raise _pre_dispatch_proxy_connect_error("first bound account proxy route unavailable") + yield 'data: {"type":"response.completed","response":{"id":"resp_bound_fallback"}}\n\n' + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service._load_balancer, "select_account", select_account) + monkeypatch.setattr(service._load_balancer, "record_error", AsyncMock()) + monkeypatch.setattr(service._load_balancer, "record_success", AsyncMock()) + monkeypatch.setattr(service, "_ensure_fresh", AsyncMock(side_effect=lambda account, **kwargs: account)) + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_stream) + + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "", + "input": [ + { + "type": "reasoning", + "id": "rs_pre_dispatch_owner", + "encrypted_content": "owner-bound", + } + ], + "stream": True, + } + ) + + chunks = [chunk async for chunk in service.stream_responses(payload, {"session_id": "sid-pre-dispatch"})] + + assert select_account.await_count == 2 + assert attempted_account_ids == [ + first_account.chatgpt_account_id, + second_account.chatgpt_account_id, + ] + assert any("resp_bound_fallback" in chunk for chunk in chunks) + + +@pytest.mark.asyncio +async def test_stream_responses_refuses_account_bound_security_work_retry(monkeypatch): + settings = _make_proxy_settings() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + regular_account = _make_account("acc_regular_security_account_bound") + authorized_account = _make_account("acc_authorized_security_account_bound") + authorized_account.security_work_authorized = True + select_account = AsyncMock( + side_effect=[ + AccountSelection(account=regular_account, error_message=None), + AccountSelection(account=authorized_account, error_message=None), + ] + ) + cyber_message = ( + "This chat was flagged for possible cybersecurity risk. " + "If this seems wrong, try rephrasing your request. " + "To get authorized for security work, join the Trusted Access for Cyber program. " + "https://chatgpt.com/cyber" + ) + dispatched_account_ids: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False): + del payload, headers, access_token, base_url, raise_for_status + dispatched_account_ids.append(account_id) + if account_id == regular_account.chatgpt_account_id: + yield ( + "data: " + + json.dumps( + { + "type": "response.failed", + "response": { + "id": "resp_cyber_account_bound", + "error": { + "code": "invalid_request_error", + "type": "invalid_request_error", + "message": cyber_message, + }, + }, + } + ) + + "\n\n" + ) + return + yield 'data: {"type":"response.completed","response":{"id":"resp_cross_account"}}\n\n' + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service._load_balancer, "select_account", select_account) + monkeypatch.setattr(service._load_balancer, "record_error", AsyncMock()) + monkeypatch.setattr(service._load_balancer, "record_success", AsyncMock()) + monkeypatch.setattr(service, "_ensure_fresh", AsyncMock(side_effect=lambda account, **kwargs: account)) + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_stream) + + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "check api", + "input": [ + { + "type": "reasoning", + "id": "rs_owner", + "encrypted_content": "owner-bound", + } + ], + "stream": True, + } + ) + + chunks = [chunk async for chunk in service.stream_responses(payload, {"session_id": "sid-stream"})] + + assert dispatched_account_ids == [regular_account.chatgpt_account_id] + assert all("resp_cross_account" not in chunk for chunk in chunks) + + @pytest.mark.asyncio async def test_stream_responses_treats_missing_security_work_pool_as_optional(monkeypatch): settings = _make_proxy_settings() @@ -16272,6 +18009,63 @@ async def fake_reconnect_http_bridge_session( assert request_state.event_queue.empty() +@pytest.mark.asyncio +async def test_http_bridge_refuses_account_bound_security_work_retry(monkeypatch): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + regular_account = _make_account("acc_bridge_security_account_bound") + request_text = json.dumps( + { + "type": "response.create", + "model": "gpt-5.1", + "input": [ + { + "type": "reasoning", + "id": "rs_owner", + "encrypted_content": "owner-bound", + } + ], + }, + separators=(",", ":"), + ) + request_state = proxy_service._WebSocketRequestState( + request_id="bridge_req_security_account_bound", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + transport="http", + request_text=request_text, + preferred_account_id=regular_account.id, + ) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "turn-security-owner", None), + headers={}, + affinity=proxy_service._AffinityPolicy(), + request_model="gpt-5.1", + account=regular_account, + upstream=AsyncMock(), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=1.0, + idle_ttl_seconds=300.0, + ) + reconnect = AsyncMock() + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + + assert await service._retry_http_bridge_security_work_request(session, request_state) is False + + reconnect.assert_not_awaited() + assert request_state.preferred_account_id == regular_account.id + assert request_state.excluded_account_ids == set() + assert request_state.request_text == request_text + + @pytest.mark.parametrize( ("item_type", "expected_deferred"), [ @@ -16716,6 +18510,83 @@ async def test_http_bridge_recovery_auth_reconnect_failure_preserves_original_au assert await request_state.event_queue.get() is None +@pytest.mark.asyncio +async def test_retry_http_bridge_bound_auth_refresh_never_sends_on_other_account(monkeypatch): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + owner = _make_account("acc_bridge_bound_auth_owner") + other = _make_account("acc_bridge_bound_auth_other") + request_text = ( + '{"type":"response.create","model":"gpt-5.6-sol","input":[' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"}]}' + ) + request_state = proxy_service._WebSocketRequestState( + request_id="bridge_bound_auth_refresh", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + awaiting_response_created=True, + transport="http", + request_text=request_text, + preferred_account_id=owner.id, + replay_required_account_id=owner.id, + ) + owner_upstream = AsyncMock() + other_upstream = AsyncMock() + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-bound-auth", None), + headers={}, + affinity=proxy_service._AffinityPolicy(), + request_model="gpt-5.6-sol", + account=owner, + upstream=owner_upstream, + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=0.0, + idle_ttl_seconds=30.0, + ) + + async def adversarial_reconnect( + reconnect_session, + *, + request_state, + require_same_account=False, + require_preferred_account=False, + **kwargs, + ): + del request_state, kwargs + if not (require_same_account and require_preferred_account): + reconnect_session.account = other + reconnect_session.upstream = other_upstream + return + raise proxy_module.ProxyResponseError( + 502, + openai_error( + "previous_response_owner_unavailable", + "Request payload owner account is unavailable; retry later.", + error_type="server_error", + ), + ) + + monkeypatch.setattr(service, "_reconnect_http_bridge_session", adversarial_reconnect) + + result = await service._retry_http_bridge_precreated_auth_request( + session, + request_state, + error_message="Authentication failed", + ) + + assert result == "failed" + assert session.account is owner + owner_upstream.send_text.assert_not_awaited() + other_upstream.send_text.assert_not_awaited() + assert request_state.replay_required_account_id == owner.id + + @pytest.mark.asyncio async def test_http_bridge_keeps_previous_response_pinned_security_work_error(monkeypatch): request_logs = _RequestLogsRecorder() @@ -17197,7 +19068,21 @@ async def test_http_bridge_security_retry_clears_codex_affinity_and_turn_aliases sticky_source: str, ) -> None: sticky_sessions = AsyncMock() - sticky_sessions.get_account_id.return_value = None + + async def legacy_owner_for_source( + _key: str, + *, + kind: StickySessionKind, + max_age_seconds: int | None = None, + continuity_source: str | None = None, + ) -> str | None: + assert kind is StickySessionKind.CODEX_SESSION + assert max_age_seconds is None + # Model a session-header-scoped tombstone: unknown callers still see + # the retained raw owner because it may belong to explicit turn state. + return "acc_bridge_security_rejected" if continuity_source is None else None + + sticky_sessions.get_account_id.side_effect = legacy_owner_for_source class _TrackingRepoContext: def __init__(self) -> None: @@ -17345,12 +19230,19 @@ async def fake_reconnect( assert ("turn-security-rejected", None) not in service._http_bridge_turn_state_index assert "x-codex-turn-state" not in session.headers if sticky_source == "session_header": + sticky_sessions.get_account_id.assert_awaited_once_with( + original_affinity.legacy_selection_key, + kind=StickySessionKind.CODEX_SESSION, + max_age_seconds=None, + continuity_source="session_header", + ) sticky_sessions.upsert.assert_awaited_once_with( original_affinity.selection_key, authorized_account.id, kind=StickySessionKind.CODEX_SESSION, ) else: + sticky_sessions.get_account_id.assert_not_awaited() sticky_sessions.upsert.assert_not_awaited() @@ -17495,6 +19387,91 @@ async def __aexit__(self, exc_type, exc, tb) -> bool: assert session.closed is False +@pytest.mark.asyncio +async def test_http_bridge_replacement_uses_legacy_continuity_source_for_raw_row() -> None: + rejected_account = _make_account("acc_bridge_thread_restart_owner") + authorized_account = _make_account("acc_bridge_thread_restart_replacement") + sticky_sessions = AsyncMock() + seen_sources: list[str | None] = [] + + async def legacy_owner_for_source( + _key: str, + *, + kind: StickySessionKind, + max_age_seconds: int | None = None, + continuity_source: str | None = None, + ) -> str | None: + del kind, max_age_seconds + seen_sources.append(continuity_source) + return rejected_account.id if continuity_source == "thread_header" else None + + sticky_sessions.get_account_id.side_effect = legacy_owner_for_source + + class _TrackingRepoContext: + def __init__(self) -> None: + self._repos = ProxyRepositories( + accounts=cast(AccountsRepository, AsyncMock()), + usage=cast(UsageRepository, AsyncMock()), + request_logs=cast(RequestLogsRepository, _RequestLogsRecorder()), + sticky_sessions=cast(StickySessionsRepository, sticky_sessions), + api_keys=cast(ApiKeysRepository, AsyncMock()), + additional_usage=cast(AdditionalUsageRepository, AsyncMock()), + ) + + async def __aenter__(self) -> ProxyRepositories: + return self._repos + + async def __aexit__(self, exc_type, exc, tb) -> bool: + return False + + service = proxy_service.ProxyService(_TrackingRepoContext) + replacement_upstream = AsyncMock() + affinity = proxy_service._AffinityPolicy( + key="thread-restart-rebind", + kind=StickySessionKind.PROMPT_CACHE, + codex_session_source="thread_header", + legacy_codex_session_key="process-restart-rebind", + legacy_continuity_source="session_header", + ) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("thread_header", "thread-restart-rebind", None), + headers={"session_id": "process-restart-rebind", "thread-id": "thread-restart-rebind"}, + affinity=affinity, + request_model="gpt-5.1", + account=rejected_account, + upstream=AsyncMock(), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=300.0, + durable_session_id="durable-thread-restart-rebind", + durable_owner_epoch=2, + ) + durable_claim = AsyncMock() + service._claim_durable_http_bridge_session = durable_claim + + await service._claim_http_bridge_replacement_before_swap( + session, + account_id=authorized_account.id, + upstream=replacement_upstream, + release_selected_account_lease=AsyncMock(), + owner_rebind_affinity=affinity, + ) + + assert seen_sources == ["session_header"] + sticky_sessions.get_account_id.assert_awaited_once_with( + "process-restart-rebind", + kind=StickySessionKind.CODEX_SESSION, + max_age_seconds=None, + continuity_source="session_header", + ) + durable_claim.assert_awaited_once() + replacement_upstream.close.assert_not_awaited() + + @pytest.mark.asyncio async def test_http_bridge_security_retry_restores_codex_affinity_and_turn_aliases_on_failure( monkeypatch: pytest.MonkeyPatch, @@ -18522,6 +20499,68 @@ async def select_account(deadline: float, **kwargs: object) -> AccountSelection: assert request_logs.calls[0]["account_id"] == account_owner.id +@pytest.mark.asyncio +async def test_connect_proxy_websocket_account_bound_replay_stays_on_owner(monkeypatch): + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_owner = _make_account("acc_ws_replay_owner") + account_other = _make_account("acc_ws_replay_other") + select_account = AsyncMock( + side_effect=[ + AccountSelection(account=account_owner, error_message=None), + AccountSelection(account=account_other, error_message=None), + ] + ) + handshake_error = proxy_module.ProxyResponseError( + 429, + openai_error("usage_limit_reached", "usage limit reached"), + ) + monkeypatch.setattr(service, "_select_account_with_budget", select_account) + monkeypatch.setattr(service._load_balancer, "mark_rate_limit", AsyncMock()) + monkeypatch.setattr(service, "_ensure_fresh", AsyncMock(return_value=account_owner)) + open_upstream = AsyncMock(side_effect=[handshake_error]) + monkeypatch.setattr(service, "_open_upstream_websocket", open_upstream) + monkeypatch.setattr(service, "_release_websocket_reservation", AsyncMock()) + + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_account_bound_replay", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + request_text=( + '{"type":"response.create","model":"gpt-5.1","input":[' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"}]}' + ), + ) + websocket_send = AsyncMock() + + selected_account, selected_upstream = await service._connect_proxy_websocket( + {}, + sticky_key=None, + sticky_kind=None, + prefer_earlier_reset=False, + prefer_earlier_reset_window="secondary", + routing_strategy="usage_weighted", + model="gpt-5.1", + request_state=request_state, + api_key=None, + client_send_lock=anyio.Lock(), + websocket=cast(WebSocket, SimpleNamespace(send_text=websocket_send)), + ) + + assert selected_account is None + assert selected_upstream is None + assert select_account.await_count == 2 + assert select_account.await_args_list[1].kwargs["preferred_account_id"] == account_owner.id + open_upstream.assert_awaited_once() + websocket_send_args = websocket_send.await_args + assert websocket_send_args is not None + sent_payload = json.loads(websocket_send_args.args[0]) + assert sent_payload["error"]["code"] == "previous_response_owner_unavailable" + + @pytest.mark.asyncio async def test_connect_proxy_websocket_surfaces_local_connect_overload_without_penalizing_account(monkeypatch): settings = _make_proxy_settings() @@ -20710,6 +22749,83 @@ class Settings: assert fresh_payload["input"] == [*historical_input, new_input] +@pytest.mark.asyncio +async def test_prepare_websocket_goal_restart_keeps_full_resend_without_injected_anchor(monkeypatch): + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + reserve_usage = AsyncMock(return_value=None) + api_key = ApiKeyData( + id="key_ws_goal_restart", + name="ws-goal-restart", + key_prefix="sk-ws-goal", + allowed_models=["gpt-5.1"], + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + + class Settings: + trace_channels = frozenset() + openai_prompt_cache_key_derivation_enabled = True + + historical_input: list[JsonValue] = [ + {"role": "user", "content": [{"type": "input_text", "text": "old question"}]}, + ] + retained_output: JsonValue = { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "old answer"}], + } + new_input: JsonValue = { + "role": "user", + "content": [{"type": "input_text", "text": "continue the goal"}], + } + continuity_state = proxy_service._WebSocketContinuityState( + last_completed_input_count=len(historical_input), + last_completed_response_id="resp_old_goal_owner", + last_completed_input_prefix_fingerprint=proxy_service._fingerprint_input_items(historical_input), + ) + + monkeypatch.setattr(proxy_service, "get_settings", lambda: Settings()) + monkeypatch.setattr(service, "_reserve_websocket_api_key_usage", reserve_usage) + monkeypatch.setattr(service, "_refresh_websocket_api_key_policy", AsyncMock(return_value=api_key)) + + prepared = await service._prepare_websocket_response_create_request( + cast( + dict[str, JsonValue], + { + "type": "response.create", + "model": "gpt-5.1", + "instructions": ( + '\nContinue working toward the active thread goal.' + ), + "input": [*historical_input, retained_output, new_input], + }, + ), + headers={"session_id": "goal-restart-direct-websocket"}, + codex_session_affinity=True, + openai_cache_affinity=True, + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=300, + api_key=api_key, + continuity_state=continuity_state, + ) + + upstream_payload = json.loads(prepared.text_data) + assert "previous_response_id" not in upstream_payload + assert upstream_payload["input"] == [*historical_input, retained_output, new_input] + assert prepared.request_state.previous_response_id is None + assert prepared.request_state.proxy_injected_previous_response_id is False + assert prepared.request_state.fresh_upstream_request_is_retry_safe is True + assert prepared.affinity_policy.codex_session_source == "session_header" + assert prepared.affinity_policy.abandon_unavailable_legacy_owner is True + + @pytest.mark.asyncio async def test_prepare_websocket_response_create_request_does_not_fresh_retry_injected_tool_output_delta( monkeypatch, @@ -21349,6 +23465,147 @@ def test_websocket_continuity_state_reuses_codex_session_scope(): assert unscoped is not first +def test_websocket_continuity_state_isolates_sibling_codex_threads() -> None: + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + root_headers = {"session-id": "process-shared", "thread-id": "thread-root"} + child_headers = {"session-id": "process-shared", "thread-id": "thread-child"} + + root = service._websocket_continuity_state_for_request( + root_headers, + api_key=None, + codex_session_affinity=True, + ) + root.last_completed_response_id = "resp-root" + root.last_pending_tool_call_types["call-root"] = "function_call" + + child = service._websocket_continuity_state_for_request( + child_headers, + api_key=None, + codex_session_affinity=True, + ) + root_reconnect = service._websocket_continuity_state_for_request( + root_headers, + api_key=None, + codex_session_affinity=True, + ) + + assert child is not root + assert child.last_completed_response_id is None + assert child.last_pending_tool_call_types == {} + assert root_reconnect is root + assert root_reconnect.last_completed_response_id == "resp-root" + assert root_reconnect.last_pending_tool_call_types == {"call-root": "function_call"} + + +def test_websocket_explicit_turn_state_precedes_broader_thread_state() -> None: + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + thread_headers = {"session-id": "process-shared", "thread-id": "thread-exact"} + explicit_turn_state = "turn_client_exact" + + broader_thread_state = service._websocket_continuity_state_for_request( + thread_headers, + api_key=None, + codex_session_affinity=True, + ) + broader_thread_state.last_completed_response_id = "resp-thread-broad" + exact_turn_state = service._websocket_continuity_state_for_request( + {"x-codex-turn-state": explicit_turn_state}, + api_key=None, + codex_session_affinity=True, + ) + exact_turn_state.last_completed_response_id = "resp-turn-exact" + + resolved = service._websocket_continuity_state_for_request( + {**thread_headers, "x-codex-turn-state": explicit_turn_state}, + api_key=None, + codex_session_affinity=True, + ) + thread_reconnect = service._websocket_continuity_state_for_request( + thread_headers, + api_key=None, + codex_session_affinity=True, + ) + + assert resolved is exact_turn_state + assert resolved is not broader_thread_state + assert resolved.last_completed_response_id == "resp-turn-exact" + assert thread_reconnect is exact_turn_state + + +def test_websocket_unknown_explicit_turn_state_does_not_reuse_broader_thread_state() -> None: + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + thread_headers = {"session-id": "process-shared", "thread-id": "thread-hard-turn"} + + broader_thread_state = service._websocket_continuity_state_for_request( + thread_headers, + api_key=None, + codex_session_affinity=True, + ) + broader_thread_state.last_completed_response_id = "resp-thread-broad" + + unknown_turn_state = service._websocket_continuity_state_for_request( + {**thread_headers, "x-codex-turn-state": "turn_client_unknown"}, + api_key=None, + codex_session_affinity=True, + ) + thread_reconnect = service._websocket_continuity_state_for_request( + thread_headers, + api_key=None, + codex_session_affinity=True, + ) + + assert unknown_turn_state is not broader_thread_state + assert unknown_turn_state.last_completed_response_id is None + assert thread_reconnect is broader_thread_state + assert thread_reconnect.last_completed_response_id == "resp-thread-broad" + + +@pytest.mark.asyncio +async def test_active_websocket_refreshes_only_its_bounded_thread_affinity() -> None: + sticky_sessions = AsyncMock(spec=StickySessionsRepository) + + class _ThreadAffinityRepoContext: + async def __aenter__(self) -> SimpleNamespace: + return SimpleNamespace(sticky_sessions=sticky_sessions) + + async def __aexit__(self, exc_type: object, exc: object, tb: object) -> bool: + del exc_type, exc, tb + return False + + service = proxy_service.ProxyService(cast(Any, lambda: _ThreadAffinityRepoContext())) + thread_key = proxy_affinity._codex_backend_identity( + {"session-id": "process-active", "thread-id": "thread-active"} + ).thread_selection_key + assert thread_key is not None + request_state = proxy_service._WebSocketRequestState( + request_id="req-active-thread-touch", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + affinity_policy=proxy_service._AffinityPolicy( + key=thread_key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=300, + codex_session_source="thread_header", + legacy_codex_session_key="process-active", + seed_selection_key=proxy_affinity._codex_session_selection_key("process-active"), + seed_selection_kind=StickySessionKind.CODEX_SESSION, + ), + thread_affinity_last_touch_at=0.0, + ) + account = _make_account("acc-active-thread-touch") + + await service._touch_active_websocket_thread_affinity(request_state, account) + + sticky_sessions.upsert.assert_awaited_once_with( + thread_key, + account.id, + kind=StickySessionKind.PROMPT_CACHE, + ) + + def test_websocket_continuity_state_seeds_generated_turn_state_alias(): service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) generated_turn_state = "turn_0123456789abcdef0123456789abcdef" @@ -21979,6 +24236,131 @@ def test_slim_response_create_ignores_malformed_unhashable_item_type(): ] +def _non_user_item(value: JsonValue) -> JsonValue: + if not isinstance(value, dict): + return value + item = dict(value) + if item.get("role") == "user": + item["role"] = "assistant" + return item + + +@given( + extra=json_objects, + historical=st.lists(json_values.map(_non_user_item), max_size=5), + recent=st.lists(json_values.map(_non_user_item), max_size=5), +) +@settings(max_examples=30, deadline=None) +def test_slim_response_create_preserves_recent_suffix_and_top_level_fields(extra, historical, recent): + recent_user = {"role": "user", "content": "latest request"} + payload = cast( + dict[str, JsonValue], + { + **{key: value for key, value in extra.items() if key != "input"}, + "input": [*historical, recent_user, *recent], + }, + ) + original = deepcopy(payload) + original_input = cast(list[JsonValue], original["input"]) + + slimmed_payload, _ = proxy_service._slim_response_create_payload_for_upstream(payload, max_bytes=256) + + assert payload == original + assert {key: value for key, value in slimmed_payload.items() if key != "input"} == { + key: value for key, value in original.items() if key != "input" + } + slimmed_input = cast(list[JsonValue], slimmed_payload["input"]) + preserve_from = len(historical) + assert slimmed_input[preserve_from:] == original_input[preserve_from:] + assert json.dumps(slimmed_input[preserve_from:], ensure_ascii=True, sort_keys=True) == json.dumps( + original_input[preserve_from:], ensure_ascii=True, sort_keys=True + ) + + +@given( + cases=st.lists( + st.sampled_from(["top_image", "content_image", "tool_image", "file_image", "plain"]), + min_size=1, + max_size=5, + ) +) +@settings(max_examples=30, deadline=None) +def test_slim_response_create_counts_historical_image_replacements_and_is_idempotent(cases): + historical: list[JsonValue] = [] + expected_images = 0 + inline_url = "data:image/png;base64,AAAA" + for index, case in enumerate(cases): + if case == "top_image": + historical.append({"type": "input_image", "image_url": inline_url, "id": f"image-{index}"}) + expected_images += 1 + elif case == "content_image": + historical.append( + { + "role": "assistant", + "content": [{"type": "input_image", "image_url": inline_url, "id": f"image-{index}"}], + } + ) + expected_images += 1 + elif case == "tool_image": + historical.append( + { + "type": "function_call_output", + "call_id": f"call-{index}", + "output": [{"type": "input_image", "image_url": inline_url}], + } + ) + expected_images += 1 + elif case == "file_image": + historical.append({"type": "input_image", "image_url": "file-id", "id": f"file-{index}"}) + else: + historical.append({"role": "assistant", "content": f"ordinary-{index}"}) + + latest = {"role": "user", "content": "latest"} + payload = cast(dict[str, JsonValue], {"input": [*historical, latest]}) + original = deepcopy(payload) + + slimmed_payload, summary = proxy_service._slim_response_create_payload_for_upstream(payload, max_bytes=256) + + assert payload == original + if expected_images: + assert summary is not None + assert summary["historical_images_slimmed"] == expected_images + else: + assert summary is None + assert cast(list[JsonValue], slimmed_payload["input"])[-1] == latest + + second_payload, second_summary = proxy_service._slim_response_create_payload_for_upstream( + slimmed_payload, max_bytes=256 + ) + assert second_payload == slimmed_payload + assert second_summary is None + + +@given(sizes=st.lists(st.integers(min_value=0, max_value=34 * 1024), min_size=1, max_size=5)) +@settings(max_examples=30, deadline=None) +def test_slim_response_create_counts_oversized_historical_tool_outputs(sizes): + historical = [ + { + "type": "custom_tool_call_output", + "call_id": f"call-{index}", + "output": "x" * size, + } + for index, size in enumerate(sizes) + ] + latest = {"role": "user", "content": "latest"} + payload = cast(dict[str, JsonValue], {"input": [*historical, latest]}) + + slimmed_payload, summary = proxy_service._slim_response_create_payload_for_upstream(payload, max_bytes=256) + + expected_count = sum(size > 32 * 1024 for size in sizes) + if expected_count: + assert summary is not None + assert summary["historical_tool_outputs_slimmed"] == expected_count + else: + assert summary is None + assert cast(list[JsonValue], slimmed_payload["input"])[-1] == latest + + def test_websocket_receive_timeout_prefers_idle_timeout_when_budget_allows(monkeypatch): monkeypatch.setattr(proxy_service.time, "monotonic", lambda: 100.0) @@ -28036,6 +30418,170 @@ async def release_usage_reservation(self, reservation_id: str) -> None: assert released == ["resv_stream_failed_background"] +@pytest.mark.asyncio +async def test_stream_api_key_cancelled_settlement_transfers_to_release(monkeypatch): + finalize_started = asyncio.Event() + release_completed = asyncio.Event() + repo = SimpleNamespace(api_keys=object()) + + @asynccontextmanager + async def repo_factory() -> AsyncIterator[SimpleNamespace]: + yield repo + + class FakeApiKeysService: + def __init__(self, api_keys_repository: object) -> None: + assert api_keys_repository is repo.api_keys + + async def finalize_usage_reservation(self, reservation_id: str, **kwargs: object) -> None: + del reservation_id, kwargs + finalize_started.set() + await asyncio.Event().wait() + + async def release_usage_reservation(self, reservation_id: str) -> None: + assert reservation_id == "resv_image_cancel" + release_completed.set() + + monkeypatch.setattr(proxy_service, "ApiKeysService", FakeApiKeysService) + + service = proxy_service.ProxyService(cast(proxy_service.ProxyRepoFactory, repo_factory)) + api_key = ApiKeyData( + id="key_image_cancel", + name="image cancel", + key_prefix="sk-image-cancel", + allowed_models=None, + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_image_cancel", + key_id=api_key.id, + model="gpt-image-2", + ) + settlement = proxy_service._StreamSettlement( + status="success", + model="gpt-image-2", + input_tokens=3, + output_tokens=4, + ) + + assert await service._settle_stream_api_key_usage( + api_key, + reservation, + settlement, + request_id="req_image_cancel", + ) + await asyncio.wait_for(finalize_started.wait(), timeout=1.0) + + settlement_task = next(iter(service._background_cleanup_tasks)) + settlement_task.cancel() + with pytest.raises(asyncio.CancelledError): + await settlement_task + + await asyncio.wait_for(release_completed.wait(), timeout=1.0) + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert service._background_cleanup_tasks == set() + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("input_tokens", "output_tokens"), [(3, 4), (0, 0)]) +async def test_image_api_key_settlement_maps_captured_usage_once( + monkeypatch, + input_tokens, + output_tokens, +): + repo = SimpleNamespace(api_keys=object()) + + @asynccontextmanager + async def repo_factory() -> AsyncIterator[SimpleNamespace]: + yield repo + + service = proxy_service.ProxyService(cast(proxy_service.ProxyRepoFactory, repo_factory)) + api_key = ApiKeyData( + id="key_image_handoff", + name="image handoff", + key_prefix="sk-image-handoff", + allowed_models=None, + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_image_handoff", + key_id=api_key.id, + model="gpt-image-2", + ) + captured: list[ + tuple[ + ApiKeyData | None, + proxy_service.ApiKeyUsageReservationData | None, + proxy_service._StreamSettlement, + str, + bool, + ] + ] = [] + + async def settle_spy( + api_key_arg: ApiKeyData | None, + reservation_arg: proxy_service.ApiKeyUsageReservationData | None, + settlement: proxy_service._StreamSettlement, + request_id: str, + *, + wait_for_settlement: bool = False, + ) -> bool: + settlement.usage_settlement_transferred = True + captured.append( + ( + api_key_arg, + reservation_arg, + settlement, + request_id, + wait_for_settlement, + ) + ) + return True + + monkeypatch.setattr(service, "_settle_stream_api_key_usage", settle_spy) + + assert await service.settle_image_api_key_usage( + api_key, + reservation, + model="gpt-image-2", + input_tokens=input_tokens, + output_tokens=output_tokens, + cached_input_tokens=None, + request_id="req_image_handoff", + ) + + assert len(captured) == 1 + ( + captured_api_key, + captured_reservation, + settlement, + request_id, + wait_for_settlement, + ) = captured[0] + assert captured_api_key is api_key + assert captured_reservation is reservation + assert settlement.status == "success" + assert settlement.model == "gpt-image-2" + assert settlement.input_tokens == input_tokens + assert settlement.output_tokens == output_tokens + assert settlement.cached_input_tokens == 0 + assert settlement.service_tier is None + assert settlement.usage_settlement_transferred + assert request_id == "req_image_handoff" + assert wait_for_settlement is False + + @pytest.mark.asyncio async def test_stream_api_key_release_retries_bound_concurrent_repository_attempts(monkeypatch): retry_concurrency = proxy_service._STREAM_API_KEY_RELEASE_RETRY_MAX_CONCURRENCY @@ -32452,6 +34998,9 @@ def test_http_bridge_should_attempt_local_previous_response_recovery_invalid_req def test_http_bridge_should_attempt_local_previous_response_recovery_normalizes_upstream_error_frames(): + # The terse parameterless rejection classified on the websocket path by + # #1818: no ``code``, no ``param``, classifiable only after normalizing + # ``type`` into the code slot (issue #1830). terse_parameterless_error = proxy_module.ProxyResponseError( 400, { @@ -32461,19 +35010,41 @@ def test_http_bridge_should_attempt_local_previous_response_recovery_normalizes_ } }, ) + # Frames that carry the classifiable code only in ``type``. type_only_not_found_error = proxy_module.ProxyResponseError( 404, { "error": { "type": "previous_response_not_found", - "code": " ", "message": "Previous response with id 'resp_prev_anchor' not found.", } }, ) - assert proxy_service._http_bridge_should_attempt_local_previous_response_recovery(terse_parameterless_error) - assert proxy_service._http_bridge_should_attempt_local_previous_response_recovery(type_only_not_found_error) + assert proxy_service._http_bridge_should_attempt_local_previous_response_recovery(terse_parameterless_error) is True + assert proxy_service._http_bridge_should_attempt_local_previous_response_recovery(type_only_not_found_error) is True + + +def test_http_bridge_server_recovery_mode_retries_ambiguous_transport_once(monkeypatch: pytest.MonkeyPatch): + ambiguous_error = proxy_module.ProxyResponseError( + 502, + { + "error": { + "type": "server_error", + "code": "upstream_request_timeout", + "message": "Upstream did not acknowledge response.create", + } + }, + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once" + ), + ) + + assert proxy_service._http_bridge_should_attempt_local_previous_response_recovery(ambiguous_error) is True def test_http_bridge_should_rollover_after_context_overflow(): @@ -32560,6 +35131,7 @@ def test_maybe_rewrite_websocket_previous_response_not_found_masks_lost_local_an def test_sanitize_websocket_connect_failure_rewrites_previous_response_not_found(monkeypatch, caplog): fixed_now = utcnow() + monkeypatch.setattr(websocket_helpers_module, "_websocket_stale_previous_response_index", {}) request_state = proxy_service._WebSocketRequestState( request_id="ws_req_prev_connect_failure", model="gpt-5.1", @@ -32631,6 +35203,10 @@ def test_sanitize_websocket_connect_failure_rewrites_previous_response_not_found "value": 1.0, } ] + assert not websocket_helpers_module._is_websocket_stale_previous_response( + previous_response_id="resp_prev_anchor", + api_key_id=None, + ) def test_sanitize_websocket_terminal_stale_error_marks_missing_anchor_source_unknown(): @@ -32719,6 +35295,20 @@ def test_wrapped_websocket_error_event_masks_previous_response_not_found(): assert "resp_prev_anchor" not in json.dumps(event) +def test_app_error_websocket_event_preserves_error_param(): + event = websocket_helpers_module._app_error_to_websocket_event( + ProxyReasoningEffortNotAllowed("Reasoning effort is not allowed", param="reasoning.effort") + ) + + assert event["status"] == 403 + assert event["error"] == { + "message": "Reasoning effort is not allowed", + "type": "permission_error", + "code": "reasoning_effort_not_allowed", + "param": "reasoning.effort", + } + + def test_sanitize_websocket_connect_failure_rewrites_missing_tool_output(): request_state = proxy_service._WebSocketRequestState( request_id="ws_req_missing_tool_output_connect", @@ -34343,6 +36933,51 @@ def test_cross_transport_fresh_replay_requires_matching_ws_continuity_prefix(): assert fresh.input == full_input +def test_cross_transport_fresh_replay_rejects_account_bound_payload(): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + first_input: list[JsonValue] = [ + {"role": "user", "content": [{"type": "input_text", "text": "call echo"}]}, + ] + full_input: list[JsonValue] = [ + *first_input, + { + "type": "reasoning", + "id": "rs_owner", + "encrypted_content": "owner-bound", + }, + { + "type": "function_call", + "name": "echo", + "call_id": "call_1", + "arguments": '{"value":"ok"}', + }, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + ] + service._websocket_continuity_index[("turn_generated_by_ws", None)] = proxy_service._WebSocketContinuityState( + last_completed_response_id="resp_ws_owner", + last_completed_input_count=len(first_input), + last_completed_input_prefix_fingerprint=proxy_service._fingerprint_input_items(first_input), + ) + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "test", + "previous_response_id": "resp_ws_owner", + "input": full_input, + } + ) + + assert ( + streaming_retry_module._verified_cross_transport_fresh_replay( + cast(Any, service), + payload=payload, + headers={"x-codex-session-id": "sid-cross-transport"}, + api_key=None, + ) + is None + ) + + def test_cross_transport_fresh_replay_rejects_unverified_client_full_resend(): service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) payload = ResponsesRequest.model_validate( @@ -34609,6 +37244,7 @@ async def test_compact_usage_settlement_surfaces_when_fail_safe_release_fails(mo assert _proxy_error_code(exc) == "usage_settlement_failed" assert exc.failure_phase == "usage_settlement" assert exc.failure_detail == "compact_api_key_usage_persistence_failed" + assert exc.reservation_released is False assert exc.failure_exception_type == "RuntimeError" assert isinstance(exc.__cause__, RuntimeError) assert str(exc.__cause__) == "compact finalize failed" @@ -34626,6 +37262,102 @@ async def test_compact_usage_settlement_surfaces_when_fail_safe_release_fails(mo fail_safe_service.release_usage_reservation.assert_awaited_once_with(reservation.reservation_id) +@pytest.mark.asyncio +async def test_compact_usage_settlement_marks_released_when_fail_safe_succeeds(monkeypatch): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + api_key = _make_api_key_data("key_compact_fail_safe_release_succeeds") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_fail_safe_release_succeeds", + key_id=api_key.id, + model="gpt-5.1", + ) + response = CompactResponsePayload.model_validate( + { + "object": "response.compaction", + "model": "gpt-5.1", + "output": [], + "usage": {"input_tokens": 7, "output_tokens": 3, "total_tokens": 10}, + } + ) + primary_service = SimpleNamespace( + finalize_usage_reservation=AsyncMock(side_effect=RuntimeError("compact finalize failed")), + release_usage_reservation=AsyncMock(), + ) + fail_safe_service = SimpleNamespace( + finalize_usage_reservation=AsyncMock(), + release_usage_reservation=AsyncMock(), + ) + service_factory = MagicMock(side_effect=[primary_service, fail_safe_service]) + monkeypatch.setattr(proxy_service, "ApiKeysService", service_factory) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._settle_compact_api_key_usage( + api_key=api_key, + api_key_reservation=reservation, + response=response, + request_service_tier=None, + ) + + exc = _assert_proxy_response_error(exc_info.value) + assert exc.status_code == 502 + assert _proxy_error_code(exc) == "usage_settlement_failed" + assert exc.failure_phase == "usage_settlement" + assert exc.reservation_released is True + assert isinstance(exc.__cause__, RuntimeError) + primary_service.finalize_usage_reservation.assert_awaited_once() + primary_service.release_usage_reservation.assert_not_awaited() + fail_safe_service.release_usage_reservation.assert_awaited_once_with(reservation.reservation_id) + + +@pytest.mark.asyncio +async def test_compact_usage_settlement_signals_cleanup_ready_when_both_writes_fail( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + api_key = _make_api_key_data("key_compact_double_settlement_handoff") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_double_settlement_handoff", + key_id=api_key.id, + model="gpt-5.1", + ) + response = CompactResponsePayload.model_validate( + { + "object": "response.compaction", + "model": "gpt-5.1", + "output": [], + "usage": {"input_tokens": 7, "output_tokens": 3, "total_tokens": 10}, + } + ) + service_factory = MagicMock( + side_effect=[ + SimpleNamespace( + finalize_usage_reservation=AsyncMock(side_effect=RuntimeError("compact finalize failed")), + release_usage_reservation=AsyncMock(), + ), + SimpleNamespace( + finalize_usage_reservation=AsyncMock(), + release_usage_reservation=AsyncMock(side_effect=OSError("compact fail-safe release failed")), + ), + ] + ) + monkeypatch.setattr(proxy_service, "ApiKeysService", service_factory) + cleanup_ready = asyncio.Event() + token = proxy_support._bind_propagated_responses_service_cleanup_ready(cleanup_ready) + try: + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._settle_compact_api_key_usage( + api_key=api_key, + api_key_reservation=reservation, + response=response, + request_service_tier=None, + ) + finally: + proxy_support._reset_propagated_responses_service_cleanup_ready(token) + + assert _proxy_error_code(exc_info.value) == "usage_settlement_failed" + assert cleanup_ready.is_set() + + @pytest.mark.asyncio @pytest.mark.parametrize( "failure_path", @@ -35386,6 +38118,901 @@ async def silent_upstream(payload, headers, access_token, account_id): assert request_logs.calls[0]["error_code"] == "upstream_request_timeout" +@pytest.mark.asyncio +async def test_compact_failover_next_settles_before_account_health(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_a = _make_account("acc_compact_failover_settle_a") + account_b = _make_account("acc_compact_failover_settle_b") + seen_excluded_account_ids: list[set[str]] = [] + call_order: list[str] = [] + api_key = _make_api_key_data("key_compact_failover_settle") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_failover_settle", + key_id=api_key.id, + model="gpt-5.1", + ) + + async def handle_stream_error(*args: object, **kwargs: object): + del args, kwargs + call_order.append("handle_stream_error") + return {"failure_class": "quota"} + + async def settle_compact_api_key_usage(**kwargs: object) -> None: + del kwargs + call_order.append("settle_compact_api_key_usage") + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token + if account_id == account_a.chatgpt_account_id: + raise proxy_module.ProxyResponseError( + 429, + openai_error("quota_exceeded", "quota exceeded"), + failure_phase="status", + ) + return CompactResponsePayload.model_validate({"object": "response.compaction", "output": []}) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=[account_a, account_b])) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock(side_effect=handle_stream_error)) + monkeypatch.setattr(service, "_settle_compact_api_key_usage", AsyncMock(side_effect=settle_compact_api_key_usage)) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + _install_two_account_selection(monkeypatch, service, account_a, account_b, seen_excluded_account_ids) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + result = await service.compact_responses( + payload, + {"session_id": "sid-compact-failover-settle"}, + api_key=api_key, + api_key_reservation=reservation, + ) + + assert result.object == "response.compaction" + assert account_a.id in seen_excluded_account_ids[-1] + assert call_order == ["settle_compact_api_key_usage", "handle_stream_error"] + + +@pytest.mark.asyncio +async def test_compact_http_500_failover_settles_before_account_health(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_a = _make_account("acc_compact_500_settle_a") + account_b = _make_account("acc_compact_500_settle_b") + seen_excluded_account_ids: list[set[str]] = [] + call_order: list[str] = [] + api_key = _make_api_key_data("key_compact_500_settle") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_500_settle", + key_id=api_key.id, + model="gpt-5.1", + ) + + async def handle_proxy_error(*args: object, **kwargs: object) -> None: + del args, kwargs + call_order.append("handle_proxy_error") + + async def record_errors(*args: object, **kwargs: object) -> None: + del args, kwargs + call_order.append("record_errors") + + async def settle_compact_api_key_usage(**kwargs: object) -> None: + del kwargs + call_order.append("settle_compact_api_key_usage") + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token + if account_id == account_a.chatgpt_account_id: + raise proxy_module.ProxyResponseError( + 500, + openai_error("server_error", "server error"), + failure_phase="status", + retryable_same_contract=True, + ) + return CompactResponsePayload.model_validate({"object": "response.compaction", "output": []}) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_compact_service, "_max_transient_same_account_retries", lambda: 1) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=[account_a, account_b])) + monkeypatch.setattr(service, "_handle_proxy_error", AsyncMock(side_effect=handle_proxy_error)) + monkeypatch.setattr(service._load_balancer, "record_errors", AsyncMock(side_effect=record_errors)) + monkeypatch.setattr(service, "_settle_compact_api_key_usage", AsyncMock(side_effect=settle_compact_api_key_usage)) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + _install_two_account_selection(monkeypatch, service, account_a, account_b, seen_excluded_account_ids) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + result = await service.compact_responses( + payload, + {"session_id": "sid-compact-500-settle"}, + api_key=api_key, + api_key_reservation=reservation, + ) + + assert result.object == "response.compaction" + assert call_order[0] == "settle_compact_api_key_usage" + assert "handle_proxy_error" in call_order + assert call_order.index("settle_compact_api_key_usage") < call_order.index("handle_proxy_error") + + +@pytest.mark.asyncio +async def test_compact_route_error_after_failover_flushes_deferred_health(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_a = _make_account("acc_compact_route_flush_a") + account_b = _make_account("acc_compact_route_flush_b") + seen_excluded_account_ids: list[set[str]] = [] + call_order: list[str] = [] + api_key = _make_api_key_data("key_compact_route_flush") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_route_flush", + key_id=api_key.id, + model="gpt-5.1", + ) + + async def handle_stream_error(*args: object, **kwargs: object): + del args, kwargs + call_order.append("handle_stream_error") + return {"failure_class": "quota"} + + async def settle_compact_api_key_usage(**kwargs: object) -> None: + del kwargs + call_order.append("settle_compact_api_key_usage") + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token + if account_id == account_a.chatgpt_account_id: + raise proxy_module.ProxyResponseError( + 429, + openai_error("quota_exceeded", "quota exceeded"), + failure_phase="status", + ) + raise AssertionError("account B should fail at route resolution") + + async def resolve_route(account: Account, *, operation: str) -> None: + del operation + if account.id == account_b.id: + raise proxy_service.UpstreamProxyRouteError("pool_unavailable", account_id=account.id) + return None + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=[account_a, account_b])) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock(side_effect=handle_stream_error)) + monkeypatch.setattr(service, "_settle_compact_api_key_usage", AsyncMock(side_effect=settle_compact_api_key_usage)) + monkeypatch.setattr(service, "_resolve_upstream_route_for_account", resolve_route) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + _install_two_account_selection(monkeypatch, service, account_a, account_b, seen_excluded_account_ids) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service.compact_responses( + payload, + {"session_id": "sid-compact-route-flush"}, + api_key=api_key, + api_key_reservation=reservation, + ) + + assert _proxy_error_code(exc_info.value) == "upstream_proxy_unavailable" + assert call_order == ["settle_compact_api_key_usage", "handle_stream_error"] + + +@pytest.mark.asyncio +async def test_compact_refresh_connect_failover_settles_before_account_health(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_a = _make_account("acc_compact_refresh_settle_a") + account_b = _make_account("acc_compact_refresh_settle_b") + seen_excluded_account_ids: list[set[str]] = [] + call_order: list[str] = [] + api_key = _make_api_key_data("key_compact_refresh_settle") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_refresh_settle", + key_id=api_key.id, + model="gpt-5.1", + ) + + async def handle_stream_error(*args: object, **kwargs: object): + del args, kwargs + call_order.append("handle_stream_error") + return {"failure_class": "retryable_transient"} + + async def settle_compact_api_key_usage(**kwargs: object) -> None: + del kwargs + call_order.append("settle_compact_api_key_usage") + + async def ensure_fresh(account: Account, **kwargs: object) -> Account: + del kwargs + if account.id == account_a.id: + raise aiohttp.ClientError("connection reset") + return account + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token + if account_id == account_a.chatgpt_account_id: + raise AssertionError("account A should fail during freshness") + return CompactResponsePayload.model_validate({"object": "response.compaction", "output": []}) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=ensure_fresh)) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock(side_effect=handle_stream_error)) + monkeypatch.setattr(service, "_settle_compact_api_key_usage", AsyncMock(side_effect=settle_compact_api_key_usage)) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + _install_two_account_selection(monkeypatch, service, account_a, account_b, seen_excluded_account_ids) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + result = await service.compact_responses( + payload, + {"session_id": "sid-compact-refresh-settle"}, + api_key=api_key, + api_key_reservation=reservation, + ) + + assert result.object == "response.compaction" + assert call_order == ["settle_compact_api_key_usage", "handle_stream_error"] + + +@pytest.mark.asyncio +async def test_compact_post_401_refresh_failover_settles_before_account_health(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_a = _make_account("acc_compact_401_refresh_a") + account_b = _make_account("acc_compact_401_refresh_b") + seen_excluded_account_ids: list[set[str]] = [] + call_order: list[str] = [] + api_key = _make_api_key_data("key_compact_401_refresh") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_401_refresh", + key_id=api_key.id, + model="gpt-5.1", + ) + + async def handle_stream_error(*args: object, **kwargs: object): + del args, kwargs + call_order.append("handle_stream_error") + return {"failure_class": "retryable_transient"} + + async def settle_compact_api_key_usage(**kwargs: object) -> None: + del kwargs + call_order.append("settle_compact_api_key_usage") + + async def ensure_fresh(account: Account, **kwargs: object) -> Account: + if account.id == account_a.id and kwargs.get("force"): + raise aiohttp.ClientError("connection reset") + return account + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token + if account_id == account_a.chatgpt_account_id: + raise proxy_module.ProxyResponseError( + 401, + openai_error("invalid_api_key", "token expired"), + failure_phase="status", + ) + return CompactResponsePayload.model_validate({"object": "response.compaction", "output": []}) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=ensure_fresh)) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock(side_effect=handle_stream_error)) + monkeypatch.setattr(service, "_settle_compact_api_key_usage", AsyncMock(side_effect=settle_compact_api_key_usage)) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + _install_two_account_selection(monkeypatch, service, account_a, account_b, seen_excluded_account_ids) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + result = await service.compact_responses( + payload, + {"session_id": "sid-compact-401-refresh"}, + api_key=api_key, + api_key_reservation=reservation, + ) + + assert result.object == "response.compaction" + assert call_order == ["settle_compact_api_key_usage", "handle_stream_error"] + + +@pytest.mark.asyncio +async def test_compact_second_401_failover_settles_before_account_health(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_a = _make_account("acc_compact_second_401_a") + account_b = _make_account("acc_compact_second_401_b") + seen_excluded_account_ids: list[set[str]] = [] + call_order: list[str] = [] + api_key = _make_api_key_data("key_compact_second_401") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_second_401", + key_id=api_key.id, + model="gpt-5.1", + ) + + async def handle_proxy_error(*args: object, **kwargs: object) -> None: + del args, kwargs + call_order.append("handle_proxy_error") + + async def settle_compact_api_key_usage(**kwargs: object) -> None: + del kwargs + call_order.append("settle_compact_api_key_usage") + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token + if account_id == account_a.chatgpt_account_id: + raise proxy_module.ProxyResponseError( + 401, + openai_error("invalid_api_key", "token expired"), + failure_phase="status", + ) + return CompactResponsePayload.model_validate({"object": "response.compaction", "output": []}) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=[account_a, account_a, account_b])) + monkeypatch.setattr(service, "_handle_proxy_error", AsyncMock(side_effect=handle_proxy_error)) + monkeypatch.setattr(service, "_settle_compact_api_key_usage", AsyncMock(side_effect=settle_compact_api_key_usage)) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + _install_two_account_selection(monkeypatch, service, account_a, account_b, seen_excluded_account_ids) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + result = await service.compact_responses( + payload, + {"session_id": "sid-compact-second-401"}, + api_key=api_key, + api_key_reservation=reservation, + ) + + assert result.object == "response.compaction" + assert call_order[0] == "settle_compact_api_key_usage" + assert "handle_proxy_error" in call_order + assert call_order.index("settle_compact_api_key_usage") < call_order.index("handle_proxy_error") + + +@pytest.mark.asyncio +async def test_compact_permanent_refresh_settles_before_mark_permanent(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_compact_permanent_refresh") + call_order: list[str] = [] + api_key = _make_api_key_data("key_compact_permanent_refresh") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_permanent_refresh", + key_id=api_key.id, + model="gpt-5.1", + ) + + async def mark_permanent_failure(*args: object, **kwargs: object) -> None: + del args, kwargs + call_order.append("mark_permanent_failure") + + async def settle_compact_api_key_usage(**kwargs: object) -> None: + del kwargs + call_order.append("settle_compact_api_key_usage") + + async def ensure_fresh(target: Account, **kwargs: object) -> Account: + if kwargs.get("force"): + raise RefreshError("invalid_grant", "refresh rejected", True) + return target + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token, account_id + raise proxy_module.ProxyResponseError( + 401, + openai_error("invalid_api_key", "token expired"), + failure_phase="status", + ) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=ensure_fresh)) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service._load_balancer, "mark_permanent_failure", AsyncMock(side_effect=mark_permanent_failure)) + monkeypatch.setattr(service, "_settle_compact_api_key_usage", AsyncMock(side_effect=settle_compact_api_key_usage)) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service.compact_responses( + payload, + {"session_id": "sid-compact-permanent-refresh"}, + api_key=api_key, + api_key_reservation=reservation, + ) + + assert exc_info.value.status_code == 401 + assert call_order == ["settle_compact_api_key_usage", "mark_permanent_failure"] + + +@pytest.mark.asyncio +async def test_compact_fallback_release_flushes_deferred_health(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_a = _make_account("acc_compact_fallback_flush_a") + account_b = _make_account("acc_compact_fallback_flush_b") + seen_excluded_account_ids: list[set[str]] = [] + call_order: list[str] = [] + api_key = _make_api_key_data("key_compact_fallback_flush") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_fallback_flush", + key_id=api_key.id, + model="gpt-5.1", + ) + + async def handle_stream_error(*args: object, **kwargs: object): + del args, kwargs + call_order.append("handle_stream_error") + return {"failure_class": "quota"} + + async def finalize_usage_reservation(*args: object, **kwargs: object) -> None: + del args, kwargs + call_order.append("finalize_usage_reservation") + raise RuntimeError("compact finalize failed") + + async def release_usage_reservation(*args: object, **kwargs: object) -> None: + del args, kwargs + call_order.append("release_usage_reservation") + + primary_service = SimpleNamespace( + finalize_usage_reservation=AsyncMock(side_effect=finalize_usage_reservation), + release_usage_reservation=AsyncMock(), + ) + fail_safe_service = SimpleNamespace( + finalize_usage_reservation=AsyncMock(), + release_usage_reservation=AsyncMock(side_effect=release_usage_reservation), + ) + monkeypatch.setattr(proxy_service, "ApiKeysService", MagicMock(side_effect=[primary_service, fail_safe_service])) + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token + if account_id == account_a.chatgpt_account_id: + raise proxy_module.ProxyResponseError( + 429, + openai_error("quota_exceeded", "quota exceeded"), + failure_phase="status", + ) + return CompactResponsePayload.model_validate( + { + "object": "response.compaction", + "model": "gpt-5.1", + "output": [], + "usage": {"input_tokens": 7, "output_tokens": 3, "total_tokens": 10}, + } + ) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=[account_a, account_b])) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock(side_effect=handle_stream_error)) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + _install_two_account_selection(monkeypatch, service, account_a, account_b, seen_excluded_account_ids) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service.compact_responses( + payload, + {"session_id": "sid-compact-fallback-flush"}, + api_key=api_key, + api_key_reservation=reservation, + ) + + exc = _assert_proxy_response_error(exc_info.value) + assert _proxy_error_code(exc) == "usage_settlement_failed" + assert exc.reservation_released is True + assert call_order == [ + "finalize_usage_reservation", + "release_usage_reservation", + "handle_stream_error", + ] + + +@pytest.mark.asyncio +async def test_compact_unreleased_settlement_keeps_deferred_health(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_a = _make_account("acc_compact_unreleased_health_a") + account_b = _make_account("acc_compact_unreleased_health_b") + seen_excluded_account_ids: list[set[str]] = [] + call_order: list[str] = [] + api_key = _make_api_key_data("key_compact_unreleased_health") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_unreleased_health", + key_id=api_key.id, + model="gpt-5.1", + ) + + async def handle_stream_error(*args: object, **kwargs: object): + del args, kwargs + call_order.append("handle_stream_error") + return {"failure_class": "quota"} + + async def finalize_usage_reservation(*args: object, **kwargs: object) -> None: + del args, kwargs + call_order.append("finalize_usage_reservation") + raise RuntimeError("compact finalize failed") + + async def release_usage_reservation(*args: object, **kwargs: object) -> None: + del args, kwargs + call_order.append("release_usage_reservation") + raise OSError("compact fail-safe release failed") + + primary_service = SimpleNamespace( + finalize_usage_reservation=AsyncMock(side_effect=finalize_usage_reservation), + release_usage_reservation=AsyncMock(), + ) + fail_safe_service = SimpleNamespace( + finalize_usage_reservation=AsyncMock(), + release_usage_reservation=AsyncMock(side_effect=release_usage_reservation), + ) + monkeypatch.setattr(proxy_service, "ApiKeysService", MagicMock(side_effect=[primary_service, fail_safe_service])) + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token + if account_id == account_a.chatgpt_account_id: + raise proxy_module.ProxyResponseError( + 429, + openai_error("quota_exceeded", "quota exceeded"), + failure_phase="status", + ) + return CompactResponsePayload.model_validate( + { + "object": "response.compaction", + "model": "gpt-5.1", + "output": [], + "usage": {"input_tokens": 7, "output_tokens": 3, "total_tokens": 10}, + } + ) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=[account_a, account_b])) + handle_stream_error_mock = AsyncMock(side_effect=handle_stream_error) + monkeypatch.setattr(service, "_handle_stream_error", handle_stream_error_mock) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + _install_two_account_selection(monkeypatch, service, account_a, account_b, seen_excluded_account_ids) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service.compact_responses( + payload, + {"session_id": "sid-compact-unreleased-health"}, + api_key=api_key, + api_key_reservation=reservation, + ) + + exc = _assert_proxy_response_error(exc_info.value) + assert _proxy_error_code(exc) == "usage_settlement_failed" + assert exc.reservation_released is False + assert call_order == ["finalize_usage_reservation", "release_usage_reservation"] + handle_stream_error_mock.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_compact_success_survives_deferred_health_persistence_failure(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_a = _make_account("acc_compact_health_persist_a") + account_b = _make_account("acc_compact_health_persist_b") + seen_excluded_account_ids: list[set[str]] = [] + call_order: list[str] = [] + api_key = _make_api_key_data("key_compact_health_persist") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_health_persist", + key_id=api_key.id, + model="gpt-5.1", + ) + + async def handle_stream_error(*args: object, **kwargs: object): + del args, kwargs + call_order.append("handle_stream_error") + raise RuntimeError("deferred compact health persist failed") + + async def settle_compact_api_key_usage(**kwargs: object) -> None: + del kwargs + call_order.append("settle_compact_api_key_usage") + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token + if account_id == account_a.chatgpt_account_id: + raise proxy_module.ProxyResponseError( + 429, + openai_error("quota_exceeded", "quota exceeded"), + failure_phase="status", + ) + return CompactResponsePayload.model_validate({"object": "response.compaction", "output": []}) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=[account_a, account_b])) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock(side_effect=handle_stream_error)) + monkeypatch.setattr(service, "_settle_compact_api_key_usage", AsyncMock(side_effect=settle_compact_api_key_usage)) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + _install_two_account_selection(monkeypatch, service, account_a, account_b, seen_excluded_account_ids) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + result = await service.compact_responses( + payload, + {"session_id": "sid-compact-health-persist"}, + api_key=api_key, + api_key_reservation=reservation, + ) + + assert result.object == "response.compaction" + assert call_order == ["settle_compact_api_key_usage", "handle_stream_error"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "exit_error", + [RuntimeError("compact account B exploded"), asyncio.CancelledError()], + ids=["runtime-error", "cancelled"], +) +async def test_compact_unexpected_exit_flushes_deferred_health(monkeypatch, exit_error: BaseException): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_a = _make_account("acc_compact_unexpected_exit_a") + account_b = _make_account("acc_compact_unexpected_exit_b") + seen_excluded_account_ids: list[set[str]] = [] + call_order: list[str] = [] + api_key = _make_api_key_data("key_compact_unexpected_exit") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_unexpected_exit", + key_id=api_key.id, + model="gpt-5.1", + ) + + async def handle_stream_error(*args: object, **kwargs: object): + del args, kwargs + call_order.append("handle_stream_error") + return {"failure_class": "quota"} + + async def settle_compact_api_key_usage(**kwargs: object) -> None: + del kwargs + call_order.append("settle_compact_api_key_usage") + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token + if account_id == account_a.chatgpt_account_id: + raise proxy_module.ProxyResponseError( + 429, + openai_error("quota_exceeded", "quota exceeded"), + failure_phase="status", + ) + raise exit_error + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=[account_a, account_b])) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock(side_effect=handle_stream_error)) + monkeypatch.setattr(service, "_settle_compact_api_key_usage", AsyncMock(side_effect=settle_compact_api_key_usage)) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + _install_two_account_selection(monkeypatch, service, account_a, account_b, seen_excluded_account_ids) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + with pytest.raises(type(exit_error)): + await service.compact_responses( + payload, + {"session_id": "sid-compact-unexpected-exit"}, + api_key=api_key, + api_key_reservation=reservation, + ) + + assert call_order == ["settle_compact_api_key_usage", "handle_stream_error"] + + +@pytest.mark.asyncio +async def test_compact_flush_completes_when_cancelled_during_health_write(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_a = _make_account("acc_compact_flush_cancel_a") + account_b = _make_account("acc_compact_flush_cancel_b") + seen_excluded_account_ids: list[set[str]] = [] + call_order: list[str] = [] + flush_started = asyncio.Event() + release_flush = asyncio.Event() + api_key = _make_api_key_data("key_compact_flush_cancel") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_flush_cancel", + key_id=api_key.id, + model="gpt-5.1", + ) + + async def handle_stream_error(*args: object, **kwargs: object): + del args, kwargs + flush_started.set() + await release_flush.wait() + call_order.append("handle_stream_error") + return {"failure_class": "quota"} + + async def settle_compact_api_key_usage(**kwargs: object) -> None: + del kwargs + call_order.append("settle_compact_api_key_usage") + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token + if account_id == account_a.chatgpt_account_id: + raise proxy_module.ProxyResponseError( + 429, + openai_error("quota_exceeded", "quota exceeded"), + failure_phase="status", + ) + return CompactResponsePayload.model_validate({"object": "response.compaction", "output": []}) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=[account_a, account_b])) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock(side_effect=handle_stream_error)) + monkeypatch.setattr(service, "_settle_compact_api_key_usage", AsyncMock(side_effect=settle_compact_api_key_usage)) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + _install_two_account_selection(monkeypatch, service, account_a, account_b, seen_excluded_account_ids) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + task = asyncio.create_task( + service.compact_responses( + payload, + {"session_id": "sid-compact-flush-cancel"}, + api_key=api_key, + api_key_reservation=reservation, + ) + ) + await asyncio.wait_for(flush_started.wait(), timeout=1) + task.cancel() + await asyncio.sleep(0) + release_flush.set() + with pytest.raises(asyncio.CancelledError): + await task + assert call_order == ["settle_compact_api_key_usage", "handle_stream_error"] + + +@pytest.mark.asyncio +async def test_compact_flush_continues_after_one_deferred_health_failure(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_a = _make_account("acc_compact_flush_partial_a") + account_b = _make_account("acc_compact_flush_partial_b") + account_c = _make_account("acc_compact_flush_partial_c") + seen_excluded_account_ids: list[set[str]] = [] + call_order: list[str] = [] + api_key = _make_api_key_data("key_compact_flush_partial") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_flush_partial", + key_id=api_key.id, + model="gpt-5.1", + ) + + async def handle_stream_error(failed_account: Account, *args: object, **kwargs: object): + del args, kwargs + call_order.append(f"handle_stream_error:{failed_account.id}") + if failed_account.id == account_a.id: + raise RuntimeError("first deferred health persist failed") + return {"failure_class": "quota"} + + async def settle_compact_api_key_usage(**kwargs: object) -> None: + del kwargs + call_order.append("settle_compact_api_key_usage") + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token + if account_id in {account_a.chatgpt_account_id, account_b.chatgpt_account_id}: + raise proxy_module.ProxyResponseError( + 429, + openai_error("quota_exceeded", "quota exceeded"), + failure_phase="status", + ) + return CompactResponsePayload.model_validate({"object": "response.compaction", "output": []}) + + async def select_account(**kwargs: object) -> AccountSelection: + excluded_account_ids = set(cast(set[str] | None, kwargs.get("exclude_account_ids")) or set()) + seen_excluded_account_ids.append(excluded_account_ids) + for account in (account_a, account_b, account_c): + if account.id not in excluded_account_ids: + return AccountSelection(account=account, error_message=None) + return AccountSelection(account=None, error_message="no accounts") + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_compact_service, "_compact_max_account_attempts", lambda: 3) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(side_effect=[account_a, account_b, account_c])) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock(side_effect=handle_stream_error)) + monkeypatch.setattr(service, "_settle_compact_api_key_usage", AsyncMock(side_effect=settle_compact_api_key_usage)) + monkeypatch.setattr(service._load_balancer, "select_account", select_account) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + result = await service.compact_responses( + payload, + {"session_id": "sid-compact-flush-partial"}, + api_key=api_key, + api_key_reservation=reservation, + ) + + assert result.object == "response.compaction" + assert call_order == [ + "settle_compact_api_key_usage", + f"handle_stream_error:{account_a.id}", + f"handle_stream_error:{account_b.id}", + ] + + +@pytest.mark.asyncio +async def test_compact_selection_timeout_after_failover_flushes_deferred_health(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_a = _make_account("acc_compact_select_timeout_a") + call_order: list[str] = [] + api_key = _make_api_key_data("key_compact_select_timeout") + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_compact_select_timeout", + key_id=api_key.id, + model="gpt-5.1", + ) + select_calls = 0 + + async def handle_stream_error(*args: object, **kwargs: object): + del args, kwargs + call_order.append("handle_stream_error") + return {"failure_class": "quota"} + + async def settle_compact_api_key_usage(**kwargs: object) -> None: + del kwargs + call_order.append("settle_compact_api_key_usage") + + async def fake_compact(payload, headers, access_token, account_id): + del payload, headers, access_token, account_id + raise proxy_module.ProxyResponseError( + 429, + openai_error("quota_exceeded", "quota exceeded"), + failure_phase="status", + ) + + async def select_with_budget(*args: object, **kwargs: object) -> AccountSelection: + del args, kwargs + nonlocal select_calls + select_calls += 1 + if select_calls == 1: + return AccountSelection(account=account_a, error_message=None) + raise proxy_module.ProxyResponseError( + 502, + openai_error("upstream_request_timeout", "Proxy request budget exhausted"), + ) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account_a)) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock(side_effect=handle_stream_error)) + monkeypatch.setattr(service, "_settle_compact_api_key_usage", AsyncMock(side_effect=settle_compact_api_key_usage)) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_with_budget) + monkeypatch.setattr(proxy_service, "core_compact_responses", fake_compact) + + payload = ResponsesCompactRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": []}) + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service.compact_responses( + payload, + {"session_id": "sid-compact-select-timeout"}, + api_key=api_key, + api_key_reservation=reservation, + ) + + assert _proxy_error_code(exc_info.value) == "upstream_request_timeout" + assert call_order == ["settle_compact_api_key_usage", "handle_stream_error"] + + @pytest.mark.asyncio async def test_ensure_fresh_with_timeout_bounds_whole_singleflight_wait(monkeypatch): service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) @@ -35729,38 +39356,6 @@ async def fake_ensure_fresh(self, target, *, force: bool = False): assert refreshed is account -@pytest.mark.asyncio -async def test_ensure_fresh_uses_per_operation_repository_without_opening_request_repo(monkeypatch): - settings = _make_proxy_settings() - explicit_repo = cast(Any, object()) - - def fail_repo_factory(): - raise AssertionError("request repository must not be opened for token refresh") - - @asynccontextmanager - async def refresh_repo_factory(): - yield explicit_repo - - service = proxy_service.ProxyService( - fail_repo_factory, - refresh_repo_factory=refresh_repo_factory, - ) - account = _make_account("acc_per_operation_refresh_repo") - - async def fake_ensure_fresh(manager, target, *, force: bool = False): - assert manager._repo is explicit_repo - assert manager._refresh_repo_factory is refresh_repo_factory - assert force is True - return target - - monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) - monkeypatch.setattr(proxy_service.AuthManager, "ensure_fresh", fake_ensure_fresh) - - refreshed = await service._ensure_fresh(account, force=True) - - assert refreshed is account - - @pytest.mark.asyncio async def test_ensure_fresh_same_stale_account_joins_singleflight_before_refresh_admission(monkeypatch): auth_manager_module._clear_refresh_singleflight_state() @@ -36052,6 +39647,7 @@ async def test_response_create_admission_stuck_gate_retire_ignores_draining_pend bridge_session, [stale_gate_holder], detail="response_create_gate_timeout_stuck_pending", + retry_circuit_attempt_selection=proxy_support._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), ) @@ -36158,6 +39754,57 @@ async def release_account_lease(received_lease: AccountLease | None) -> None: assert request_state.account_response_create_release is None +@pytest.mark.asyncio +async def test_response_create_gate_release_reraises_caller_cancellation_after_cleanup(): + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_gate_cancel", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + ) + response_create_gate = asyncio.Semaphore(1) + await response_create_gate.acquire() + request_state.response_create_gate_acquired = True + request_state.response_create_gate = response_create_gate + lease = AccountLease( + lease_id="lease_gate_cancel", + account_id="acc_gate_cancel", + kind="response_create", + acquired_at=0.0, + ) + request_state.account_response_create_lease = lease + release_started = asyncio.Event() + release_allowed = asyncio.Event() + + async def release_account_lease(received_lease: AccountLease | None) -> None: + assert received_lease == lease + release_started.set() + await release_allowed.wait() + + request_state.account_response_create_release = release_account_lease + + release_task = asyncio.create_task( + proxy_service._release_websocket_response_create_gate(request_state, response_create_gate) + ) + await release_started.wait() + release_task.cancel() + await asyncio.sleep(0) + + assert response_create_gate.locked() is True + assert request_state.response_create_gate_acquired is True + + release_allowed.set() + with pytest.raises(asyncio.CancelledError): + await release_task + + assert response_create_gate.locked() is False + assert request_state.response_create_gate_acquired is False + assert request_state.account_response_create_lease is None + assert request_state.account_response_create_release is None + + @pytest.mark.asyncio async def test_compact_selection_budget_exhaustion_returns_request_timeout(monkeypatch): settings = _make_proxy_settings() @@ -36344,6 +39991,50 @@ async def test_select_account_with_budget_reconciles_sticky_mapping_for_preferre assert select_account.await_args.kwargs["legacy_sticky_key"] is None +@pytest.mark.asyncio +async def test_select_account_with_budget_keeps_thread_seed_for_first_exact_owner( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = _make_proxy_settings() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + owner = _make_account("acc-thread-first-exact-owner") + select_account = AsyncMock(return_value=AccountSelection(account=owner, error_message=None)) + process_key = proxy_affinity._codex_session_selection_key("process-first-exact-owner") + thread_policy = proxy_service._AffinityPolicy( + key="thread-first-exact-owner", + kind=proxy_service.StickySessionKind.PROMPT_CACHE, + codex_session_source="thread_header", + legacy_codex_session_key="process-first-exact-owner", + seed_selection_key=process_key, + seed_selection_kind=proxy_service.StickySessionKind.CODEX_SESSION, + ) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(service._load_balancer, "select_account", select_account) + monkeypatch.setattr(proxy_service, "_remaining_budget_seconds", lambda _deadline: 10.0) + + await service._select_account_with_budget( + deadline=123.0, + request_id="req-thread-first-exact-owner", + kind="stream", + request_stage="first_turn", + **thread_policy.selection_kwargs(), + preferred_account_id=owner.id, + preferred_account_is_continuity_owner=True, + lease_kind="stream", + ) + + select_account.assert_awaited_once() + assert select_account.await_args is not None + assert select_account.await_args.kwargs["required_account_id"] == owner.id + assert select_account.await_args.kwargs["sticky_key"] is None + assert select_account.await_args.kwargs["sticky_kind"] == proxy_service.StickySessionKind.CODEX_SESSION + assert select_account.await_args.kwargs["sticky_source"] == "thread_header" + assert select_account.await_args.kwargs["legacy_sticky_key"] == "process-first-exact-owner" + assert select_account.await_args.kwargs["sticky_seed_key"] == process_key + assert select_account.await_args.kwargs["sticky_seed_kind"] == proxy_service.StickySessionKind.CODEX_SESSION + + @pytest.mark.asyncio async def test_select_account_with_budget_preserves_conversation_check_for_preferred_owner( monkeypatch: pytest.MonkeyPatch, @@ -37131,25 +40822,469 @@ async def __aexit__(self, exc_type, exc, tb) -> None: @pytest.mark.asyncio -async def test_lookup_file_pin_returns_live_entry_and_evicts_expired(monkeypatch): +async def test_responses_file_owner_resolution_batches_one_durable_lookup(monkeypatch): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + lookup_batches: list[set[str]] = [] + + async def get_live_account_ids(_repository, file_ids) -> dict[str, str]: + lookup_batches.append(set(file_ids)) + return { + "file_batch_a": "acc_batch_owner", + "file_batch_b": "acc_batch_owner", + } + + monkeypatch.setattr( + proxy_file_ops.FileAccountPinRepository, + "get_live_account_ids", + get_live_account_ids, + ) + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "Read both files.", + "input": [ + {"type": "input_file", "file_id": "file_batch_a"}, + {"type": "input_file", "file_id": "file_batch_b"}, + {"type": "input_file", "file_id": "file_batch_a"}, + ], + } + ) + + assert await service._resolve_file_account_for_responses(payload, {}) == "acc_batch_owner" + assert lookup_batches == [{"file_batch_a", "file_batch_b"}] + + +@pytest.mark.asyncio +async def test_finalize_file_database_lookup_failure_stops_before_upstream(monkeypatch): request_logs = _RequestLogsRecorder() service = proxy_service.ProxyService(_repo_factory(request_logs)) - fake_now = [100.0] + selection = AsyncMock() + + async def fail_lookup(_repository, _file_id: str) -> str | None: + raise RuntimeError("file pin database unavailable") + + monkeypatch.setattr( + proxy_file_ops.FileAccountPinRepository, + "get_live_account_id", + fail_lookup, + ) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", selection) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service.finalize_file("file_db_failure", {}) - monkeypatch.setattr(proxy_service.time, "monotonic", lambda: fake_now[0]) + assert _proxy_error_code(exc_info.value) == "file_owner_unavailable" + selection.assert_not_awaited() + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "error" + assert request_logs.calls[0]["error_code"] == "file_owner_unavailable" + assert request_logs.calls[0]["account_id"] is None - await service._pin_file_account("file_live", "acc_live") - entry = await service._lookup_file_pin("file_live") +@pytest.mark.asyncio +async def test_create_file_database_write_failure_does_not_return_upstream_result(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_unpersisted") + selection = AsyncMock(return_value=AccountSelection(account=account, error_message=None)) + upstream_create = AsyncMock(return_value={"file_id": "file_unpersisted", "upload_url": "https://upload.invalid"}) + claim_calls: list[tuple[str, str, int]] = [] - assert entry is not None - assert entry.account_id == "acc_live" + async def fail_claim( + _repository, + file_id: str, + account_id: str, + *, + ttl_seconds: int, + ) -> None: + claim_calls.append((file_id, account_id, ttl_seconds)) + raise RuntimeError("file pin database unavailable") - fake_now[0] += service._FILE_ACCOUNT_PIN_TTL_SECONDS + 1 + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_file_ops.FileAccountPinRepository, "claim", fail_claim) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", selection) + monkeypatch.setattr(service, "_ensure_previsible_unary_fresh_with_failover", AsyncMock(return_value=account)) + monkeypatch.setattr(service._load_balancer, "record_success", AsyncMock()) + monkeypatch.setattr(proxy_service, "core_create_file", upstream_create) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service.create_file({"file_name": "document.txt"}, {}) + + assert _proxy_error_code(exc_info.value) == "file_owner_unavailable" + upstream_create.assert_awaited_once() + assert claim_calls == [ + ( + "file_unpersisted", + "acc_unpersisted", + service._FILE_ACCOUNT_PIN_TTL_SECONDS, + ) + ] + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "error" + assert request_logs.calls[0]["error_code"] == "file_owner_unavailable" + assert request_logs.calls[0]["account_id"] == account.id - expired = await service._lookup_file_pin("file_live") - assert expired is None +@pytest.mark.asyncio +async def test_finalize_file_database_renewal_failure_logs_error_without_upstream_retry(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_finalize_unrenewed") + selection = AsyncMock(return_value=AccountSelection(account=account, error_message=None)) + upstream_finalize = AsyncMock(return_value={"status": "success"}) + unexpected_failover = AsyncMock( + side_effect=AssertionError("post-success pin persistence failure must not retry upstream") + ) + + async def resolve_owner(_repository, _file_id: str) -> str: + return account.id + + async def fail_claim( + _repository, + _file_id: str, + _account_id: str, + *, + ttl_seconds: int, + ) -> None: + assert ttl_seconds == service._FILE_ACCOUNT_PIN_TTL_SECONDS + raise RuntimeError("file pin database unavailable") + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_file_ops.FileAccountPinRepository, "get_live_account_id", resolve_owner) + monkeypatch.setattr(proxy_file_ops.FileAccountPinRepository, "claim", fail_claim) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", selection) + monkeypatch.setattr(service, "_ensure_previsible_unary_fresh_with_failover", AsyncMock(return_value=account)) + monkeypatch.setattr(service, "_retry_previsible_unary_call_failover", unexpected_failover) + monkeypatch.setattr(service._load_balancer, "record_success", AsyncMock()) + monkeypatch.setattr(proxy_service, "core_finalize_file", upstream_finalize) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service.finalize_file("file_finalize_unrenewed", {}) + + assert _proxy_error_code(exc_info.value) == "file_owner_unavailable" + upstream_finalize.assert_awaited_once() + unexpected_failover.assert_not_awaited() + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert len(request_logs.calls) == 1 + assert request_logs.calls[0]["status"] == "error" + assert request_logs.calls[0]["error_code"] == "file_owner_unavailable" + assert request_logs.calls[0]["account_id"] == account.id + + +def test_origin_release_follows_owner_forward_dispatch_outcome() -> None: + cleanup_ready = asyncio.Event() + dispatched = asyncio.Event() + rejected = asyncio.Event() + + assert proxy_api._responses_origin_may_release_reservation( + service_cleanup_ready_event=cleanup_ready, + owner_forward_dispatched_event=dispatched, + owner_forward_rejected_event=rejected, + ) + + dispatched.set() + assert not proxy_api._responses_origin_may_release_reservation( + service_cleanup_ready_event=cleanup_ready, + owner_forward_dispatched_event=dispatched, + owner_forward_rejected_event=rejected, + ) + + rejected.set() + assert proxy_api._responses_origin_may_release_reservation( + service_cleanup_ready_event=cleanup_ready, + owner_forward_dispatched_event=dispatched, + owner_forward_rejected_event=rejected, + ) + + cleanup_ready.set() + assert not proxy_api._responses_origin_may_release_reservation( + service_cleanup_ready_event=cleanup_ready, + owner_forward_dispatched_event=dispatched, + owner_forward_rejected_event=rejected, + ) + + +def test_owner_forward_failure_allows_local_recovery_only_for_safe_outcomes() -> None: + from app.modules.proxy._service.http_bridge.owner_forwarding import ( + _owner_forward_failure_allows_local_recovery, + _OwnerForwardOutcome, + _OwnerForwardRequestError, + ) + + base = proxy_module.ProxyResponseError( + 503, + openai_error("bridge_owner_unreachable", "HTTP bridge owner request failed"), + ) + assert _owner_forward_failure_allows_local_recovery(base) + assert _owner_forward_failure_allows_local_recovery( + _OwnerForwardRequestError(base, outcome=_OwnerForwardOutcome.NOT_DISPATCHED) + ) + assert _owner_forward_failure_allows_local_recovery( + _OwnerForwardRequestError(base, outcome=_OwnerForwardOutcome.RECEIVER_REJECTED) + ) + assert not _owner_forward_failure_allows_local_recovery( + _OwnerForwardRequestError(base, outcome=_OwnerForwardOutcome.DISPATCH_AMBIGUOUS) + ) + assert not _owner_forward_failure_allows_local_recovery( + _OwnerForwardRequestError(base, outcome=_OwnerForwardOutcome.RECEIVER_ACKNOWLEDGED) + ) + + +@pytest.mark.asyncio +async def test_forwarded_receiver_cleanup_handoff_timeout_returns_503() -> None: + async def never_ready() -> AsyncIterator[str]: + await asyncio.Event().wait() + yield "data: never\n\n" + + cleanup_ready = asyncio.Event() + stream, startup_error = await proxy_api._probe_stream_startup_error( + never_ready(), + timeout_seconds=0.01, + service_cleanup_ready_event=cleanup_ready, + ) + + assert isinstance(startup_error, proxy_module.ProxyResponseError) + assert startup_error.status_code == 503 + assert _proxy_error_code(startup_error) == "upstream_unavailable" + assert startup_error.failure_detail == "cleanup_handoff_timeout" + aclose = getattr(stream, "aclose", None) + if callable(aclose): + await aclose() + + +@pytest.mark.asyncio +async def test_startup_cleanup_guard_runs_after_initial_heartbeat_disconnect( + monkeypatch: pytest.MonkeyPatch, +) -> None: + released: list[str] = [] + started = asyncio.Event() + + async def pending_probe() -> str: + started.set() + await asyncio.Event().wait() + return "data: late\n\n" + + async def hanging_service_stream() -> AsyncIterator[str]: + await asyncio.Event().wait() + yield "data: never\n\n" + + async def record_release( + reservation: object, + *, + action: str, + scheduler: object, + request_id: str, + ) -> None: + del reservation, scheduler, request_id + released.append(action) + + monkeypatch.setattr(proxy_api, "_release_reservation_best_effort", record_release) + startup_task = asyncio.create_task(pending_probe()) + await started.wait() + cleanup_ready = asyncio.Event() + reservation_cleanup = proxy_api._ResponsesReservationCleanup( + owns_reservation=True, + reservation=None, + scheduler=None, + request_id="startup-guard-heartbeat", + ) + service_stream = hanging_service_stream() + stream = proxy_api._prepend_initial_sse_heartbeat( + service_stream, + ": keepalive\n\n", + request_id="startup-guard-heartbeat", + ) + stream = proxy_api._guard_responses_startup_handoff( + stream, + startup_task=startup_task, + streams_to_close=(service_stream,), + reservation_cleanup=reservation_cleanup, + responses_service_cleanup_ready_event=cleanup_ready, + responses_owner_forward_dispatched_event=asyncio.Event(), + responses_owner_forward_rejected_event=asyncio.Event(), + ) + + assert await anext(stream) == ": keepalive\n\n" + close = getattr(stream, "aclose", None) + assert callable(close) + await close() + await asyncio.sleep(0) + + assert startup_task.cancelled() + assert released == ["responses startup handoff"] + + +@pytest.mark.asyncio +async def test_startup_cleanup_guard_closes_stream_when_probe_already_completed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + closed: list[str] = [] + released: list[str] = [] + + class _CompletedProbeStream: + def __aiter__(self) -> "_CompletedProbeStream": + return self + + async def __anext__(self) -> str: + await asyncio.Event().wait() + raise StopAsyncIteration + + async def aclose(self) -> None: + closed.append("service") + + async def record_release( + reservation: object, + *, + action: str, + scheduler: object, + request_id: str, + ) -> None: + del reservation, scheduler, request_id + released.append(action) + + monkeypatch.setattr(proxy_api, "_release_reservation_best_effort", record_release) + reservation_cleanup = proxy_api._ResponsesReservationCleanup( + owns_reservation=True, + reservation=None, + scheduler=None, + request_id="completed-probe-guard", + ) + service_stream = _CompletedProbeStream() + stream = proxy_api._prepend_initial_sse_heartbeat( + service_stream, + ": keepalive\n\n", + request_id="completed-probe-guard", + ) + stream = proxy_api._guard_responses_startup_handoff( + stream, + startup_task=None, + streams_to_close=(service_stream,), + reservation_cleanup=reservation_cleanup, + responses_service_cleanup_ready_event=asyncio.Event(), + responses_owner_forward_dispatched_event=asyncio.Event(), + responses_owner_forward_rejected_event=asyncio.Event(), + ) + + assert await anext(stream) == ": keepalive\n\n" + close = getattr(stream, "aclose", None) + assert callable(close) + await close() + + assert closed == ["service"] + assert released == ["responses startup handoff"] + + +@pytest.mark.asyncio +async def test_reservation_cleanup_schedules_retry_after_persistence_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + attempts: list[str] = [] + scheduled: list[tuple[str, str]] = [] + pending: list[Any] = [] + + async def fail_first_release(reservation: proxy_service.ApiKeyUsageReservationData) -> None: + attempts.append(reservation.reservation_id) + if len(attempts) == 1: + raise RuntimeError("reservation release persistence failed") + + class _Scheduler: + def _schedule_cancel_safe_cleanup( + self, + coro: Any, + *, + action: str, + request_id: str, + ) -> None: + scheduled.append((action, request_id)) + pending.append(coro) + + monkeypatch.setattr(proxy_api, "_release_reservation", fail_first_release) + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_release_retry", + key_id="key_release_retry", + model="gpt-5.1", + ) + cleanup = proxy_api._ResponsesReservationCleanup( + owns_reservation=True, + reservation=reservation, + scheduler=cast(proxy_api._ResponsesCleanupScheduler, _Scheduler()), + request_id="req_release_retry", + ) + + await cleanup.release(action="responses startup error") + + assert attempts == ["resv_release_retry"] + assert scheduled == [("responses_startup_error_retry", "req_release_retry")] + assert pending + await pending[0] + assert attempts == ["resv_release_retry", "resv_release_retry"] + await cleanup.release(action="responses startup error") + assert attempts == ["resv_release_retry", "resv_release_retry"] + + +@pytest.mark.asyncio +async def test_forwarded_compact_fallback_settlement_keeps_http_200( + monkeypatch: pytest.MonkeyPatch, +) -> None: + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_forwarded_compact_fallback", + key_id="key_forwarded_compact_fallback", + model="gpt-5.1", + ) + + async def skip_limits(*args: object, **kwargs: object) -> proxy_service.ApiKeyUsageReservationData: + del args, kwargs + return reservation + + async def compact_responses(*args: object, **kwargs: object): + del args, kwargs + proxy_support._signal_propagated_responses_service_cleanup_ready() + raise proxy_module.ProxyResponseError( + 502, + openai_error("usage_settlement_failed", "Compact API key usage could not be settled"), + failure_phase="usage_settlement", + ) + + monkeypatch.setattr(proxy_api, "_enforce_request_limits", skip_limits) + context = SimpleNamespace( + service=SimpleNamespace( + rate_limit_headers=AsyncMock(return_value={}), + compact_responses=compact_responses, + stream_http_responses=AsyncMock(), + stream_responses=AsyncMock(), + ) + ) + request = Request({"type": "http", "method": "POST", "path": "/v1/responses", "headers": []}) + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "hi", + "input": [{"role": "user", "content": "hello"}, {"type": "compaction_trigger"}], + "stream": True, + } + ) + response = await proxy_api._stream_responses( + request, + payload, + context=cast(proxy_api.ProxyContext, context), + api_key=None, + codex_session_affinity=True, + skip_limit_enforcement=True, + api_key_reservation_override=reservation, + forwarded_request=True, + ) + + assert isinstance(response, StreamingResponse) + assert response.status_code == 200 + chunks = [chunk async for chunk in response.body_iterator] + body = "".join(chunk.decode() if isinstance(chunk, bytes) else str(chunk) for chunk in chunks) + assert "response.failed" in body + assert "usage_settlement_failed" in body + assert "Compact API key usage could not be settled" in body @pytest.mark.asyncio @@ -37318,7 +41453,10 @@ async def test_stream_http_bridge_or_retry_rejects_input_image_sediment_url(monk @pytest.mark.asyncio -async def test_stream_http_bridge_or_retry_routes_input_file_file_id_without_rejecting(monkeypatch): +async def test_stream_http_bridge_or_retry_routes_input_file_file_id_without_rejecting( + db_setup, + monkeypatch, +): request_logs = _RequestLogsRecorder() service = proxy_service.ProxyService(_repo_factory(request_logs)) settings = _make_proxy_settings() @@ -37385,6 +41523,21 @@ def test_classify_upstream_close_clean_for_clean_close_before_any_response_event assert proxy_service._classify_upstream_close(1011, response_events_seen=0) == "transient" +def test_account_neutral_transport_drop_requires_no_close_frame_and_no_response_events(): + # Issue #1754: only a frame-less drop before any application-layer + # response event is account-neutral; any close frame or streamed events + # keep the account penalty semantics. + assert proxy_service._is_account_neutral_transport_drop(None, response_events_seen=0) is True + assert proxy_service._is_account_neutral_transport_drop(None, response_events_seen=8) is False + assert proxy_service._is_account_neutral_transport_drop(1000, response_events_seen=0) is False + assert proxy_service._is_account_neutral_transport_drop(1008, response_events_seen=0) is False + assert proxy_service._is_account_neutral_transport_drop(1011, response_events_seen=0) is False + # RFC 6455 reserves 1006: it never travels in an actual close frame, so a + # synthesized abnormal-closure code counts as frame-less. + assert proxy_service._is_account_neutral_transport_drop(1006, response_events_seen=0) is True + assert proxy_service._is_account_neutral_transport_drop(1006, response_events_seen=1) is False + + @pytest.mark.asyncio async def test_open_upstream_websocket_dns_failure_recovers_on_same_account(monkeypatch): service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) @@ -37574,72 +41727,6 @@ async def select_account(**kwargs: object) -> AccountSelection: assert session.upstream is new_upstream -@pytest.mark.asyncio -async def test_reconnect_http_bridge_soft_key_same_account_requirement_rejects_selector_drift(monkeypatch): - settings = _make_proxy_settings() - service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) - owner_account = _make_account("acc_bridge_soft_owner") - foreign_account = _make_account("acc_bridge_soft_foreign") - observed_selection: dict[str, object] = {} - - monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) - monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) - monkeypatch.setattr(proxy_service.time, "monotonic", lambda: 10.0) - - async def select_account(_deadline: float, **kwargs: object) -> AccountSelection: - observed_selection.update(kwargs) - # Deliberately violate the selection contract. The reconnect layer must - # independently reject this instead of carrying the soft session and - # turn-state handshake to another account. - return AccountSelection(account=foreign_account, error_message=None) - - monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) - ensure_fresh = AsyncMock(return_value=foreign_account) - open_upstream = AsyncMock() - monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) - monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", open_upstream) - - request_state = proxy_service._WebSocketRequestState( - request_id="req_bridge_soft_owner", - model="gpt-5.5", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=10.0, - ) - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-soft-owner", None), - headers={"x-codex-turn-state": "turn-state-soft-owner"}, - affinity=proxy_service._AffinityPolicy(key="bridge-soft-owner"), - request_model="gpt-5.5", - account=owner_account, - upstream=AsyncMock(), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque([request_state]), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=1, - last_used_at=0.0, - idle_ttl_seconds=30.0, - upstream_turn_state="turn-state-soft-owner", - downstream_turn_state="turn-state-soft-owner", - last_upstream_close_code=1000, - ) - - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._reconnect_http_bridge_session( - session, - request_state=request_state, - require_same_account=True, - ) - - assert _proxy_error_code(exc_info.value) == "previous_response_owner_unavailable" - assert observed_selection["preferred_account_id"] == owner_account.id - assert observed_selection["fallback_on_preferred_account_unavailable"] is False - ensure_fresh.assert_not_awaited() - open_upstream.assert_not_awaited() - - @pytest.mark.asyncio async def test_reconnect_http_bridge_discards_model_fallback_before_selected_replacement_failure(monkeypatch): settings = _make_proxy_settings() @@ -39035,7 +43122,10 @@ async def fake_transcribe( @pytest.mark.asyncio -async def test_files_create_persists_conversation_id_on_refresh_connection_reset_failover(monkeypatch): +async def test_files_create_persists_conversation_id_on_refresh_connection_reset_failover( + db_setup, + monkeypatch, +): request_logs = _RequestLogsRecorder() service = proxy_service.ProxyService(_repo_factory(request_logs)) account_a = _make_account("acc_files_create_refresh_a") @@ -39130,7 +43220,10 @@ async def fake_create_file( @pytest.mark.asyncio -async def test_files_finalize_pinned_refresh_connection_reset_fails_closed(monkeypatch): +async def test_files_finalize_pinned_refresh_connection_reset_fails_closed( + db_setup, + monkeypatch, +): request_logs = _RequestLogsRecorder() service = proxy_service.ProxyService(_repo_factory(request_logs)) account = _make_account("acc_files_finalize_pinned") @@ -39173,7 +43266,10 @@ async def select_account(**kwargs: object) -> AccountSelection: @pytest.mark.asyncio -async def test_files_finalize_pinned_initial_selection_does_not_fall_back(monkeypatch): +async def test_files_finalize_pinned_initial_selection_does_not_fall_back( + db_setup, + monkeypatch, +): request_logs = _RequestLogsRecorder() service = proxy_service.ProxyService(_repo_factory(request_logs)) pinned_account = _make_account("acc_files_finalize_initial_pinned") @@ -40932,7 +45028,8 @@ async def capture_send_text(_text: str) -> None: session, request_state=request_state, restart_reader=True, - require_same_account=True, + require_same_account=False, + require_preferred_account=False, ) send_text.assert_awaited_once_with('{"type":"response.create","model":"gpt-5.1","input":"retry"}') assert send_request_ids == ["archive_bridge_retry_fresh"] @@ -41401,6 +45498,171 @@ async def test_retry_http_bridge_precreated_request_migrates_only_safe_initial_t upstream.send_text.assert_awaited_once_with(request_state.request_text) +@pytest.mark.asyncio +async def test_retry_http_bridge_precreated_request_keeps_account_bound_body_on_owner(monkeypatch): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_bridge_account_bound") + request_state = proxy_service._WebSocketRequestState( + request_id="req_bridge_account_bound", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + awaiting_response_created=True, + transport="http", + request_text=( + '{"type":"response.create","model":"gpt-5.6-sol","input":[' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"}]}' + ), + preferred_account_id=account.id, + ) + upstream = AsyncMock() + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-account-bound", None), + headers={"x-codex-turn-state": "turn_state_owner"}, + affinity=proxy_service._AffinityPolicy(), + request_model="gpt-5.6-sol", + account=account, + upstream=upstream, + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=0.0, + idle_ttl_seconds=30.0, + upstream_turn_state="turn_state_owner", + downstream_turn_state="turn_state_owner", + ) + reconnect = AsyncMock(return_value=None) + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + + assert await service._retry_http_bridge_precreated_request(session) is True + + reconnect.assert_awaited_once_with( + session, + request_state=request_state, + require_same_account=True, + require_preferred_account=True, + ) + assert request_state.preferred_account_id == account.id + assert request_state.excluded_account_ids == set() + assert session.upstream_turn_state == "turn_state_owner" + upstream.send_text.assert_awaited_once_with(request_state.request_text) + + +@pytest.mark.asyncio +async def test_retry_http_bridge_precreated_request_keeps_operation_id_on_owner(monkeypatch): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_bridge_operation_owner") + request_text = '{"type":"response.create","model":"gpt-5.6-sol","input":"portable user input"}' + request_state = proxy_service._WebSocketRequestState( + request_id="req_bridge_operation_owner", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + awaiting_response_created=True, + transport="http", + request_text=request_text, + preferred_account_id=account.id, + archive_request_id="archive_bridge_operation_owner", + operation_id="op_bridge_owner", + ) + upstream = AsyncMock() + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-operation-owner", None), + headers={}, + affinity=proxy_service._AffinityPolicy(), + request_model="gpt-5.6-sol", + account=account, + upstream=upstream, + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=0.0, + idle_ttl_seconds=30.0, + ) + reconnect = AsyncMock(return_value=None) + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + + assert await service._retry_http_bridge_precreated_request(session) is True + + reconnect.assert_awaited_once_with( + session, + request_state=request_state, + require_same_account=True, + require_preferred_account=True, + ) + assert request_state.excluded_account_ids == set() + assert request_state.operation_id == "op_bridge_owner" + upstream.send_text.assert_awaited_once() + send_args = upstream.send_text.await_args + assert send_args is not None + assert json.loads(send_args.args[0]) == { + "type": "response.create", + "model": "gpt-5.6-sol", + "input": "portable user input", + "client_metadata": {"codex_lb_operation_id": "op_bridge_owner"}, + } + + +@pytest.mark.asyncio +async def test_retry_http_bridge_precreated_request_allows_prepared_neutral_archive(monkeypatch): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_bridge_prepared_neutral") + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": "portable user input", + "stream": True, + } + ) + request_state, request_text = service._prepare_response_bridge_request_state( + payload, + api_key=None, + api_key_reservation=None, + include_type_field=True, + attach_event_queue=False, + transport=proxy_service._REQUEST_TRANSPORT_HTTP, + client_metadata=None, + ) + request_state.preferred_account_id = account.id + upstream = AsyncMock() + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-prepared-neutral", None), + headers={}, + affinity=proxy_service._AffinityPolicy(), + request_model="gpt-5.6-sol", + account=account, + upstream=upstream, + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=0.0, + idle_ttl_seconds=30.0, + ) + reconnect = AsyncMock(return_value=None) + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + + assert request_state.archive_request_id is not None + assert request_state.operation_id is None + assert await service._retry_http_bridge_precreated_request(session) is True + + reconnect.assert_awaited_once_with( + session, + request_state=request_state, + ) + upstream.send_text.assert_awaited_once_with(request_text) + + @pytest.mark.asyncio async def test_retry_http_bridge_precreated_request_keeps_hard_session_owner_bound(monkeypatch): service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) @@ -42151,7 +46413,7 @@ async def select_account(_deadline: float, **kwargs: object) -> AccountSelection assert request_state.previous_response_id == "resp_file_anchor" assert request_state.preferred_account_id == owner_account.id assert request_state.excluded_account_ids == set() - assert request_state.error_code_override == "no_accounts" + assert request_state.error_code_override == "previous_response_owner_unavailable" @pytest.mark.asyncio @@ -42595,6 +46857,7 @@ def make_state(request_id: str) -> "proxy_service._WebSocketRequestState": @pytest.mark.asyncio async def test_submit_http_bridge_request_reinlines_final_text(monkeypatch): service = proxy_service.ProxyService.__new__(proxy_service.ProxyService) + service._durable_bridge = None proxy_service._initialize_http_bridge_retry_circuit(service) original_text = json.dumps( { @@ -42683,6 +46946,7 @@ async def capture_send_text(_text: str) -> None: @pytest.mark.asyncio async def test_submit_http_bridge_network_send_failure_is_neutral_and_not_replayed(monkeypatch): service = proxy_service.ProxyService.__new__(proxy_service.ProxyService) + service._durable_bridge = None proxy_service._initialize_http_bridge_retry_circuit(service) request_state = proxy_service._WebSocketRequestState( request_id="req_submit_network_failure", @@ -42754,9 +47018,147 @@ async def cleanup(*_args: object, **_kwargs: object) -> None: close.assert_awaited_once() +@pytest.mark.asyncio +async def test_submit_http_bridge_marks_ambiguous_operation_before_releasing_owner(monkeypatch): + service = proxy_service.ProxyService.__new__(proxy_service.ProxyService) + events: list[str] = [] + service._durable_bridge = SimpleNamespace( + mark_operation_unknown=AsyncMock(side_effect=lambda **_kwargs: events.append("mark") or True), + ) + proxy_service._initialize_http_bridge_retry_circuit(service) + request_state = proxy_service._WebSocketRequestState( + request_id="req-submit-owner-fence-order", + model="gpt-5.5", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.5"}', + operation_id="operation-owner-fence-order", + operation_registered=True, + ) + send_error = UpstreamWebSocketTransportError( + "upstream websocket closed after dispatch", + error_code="proxy_network_unavailable", + ) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-owner-fence-order", None), + headers={}, + affinity=proxy_service._AffinityPolicy(key="sid-owner-fence-order"), + request_model="gpt-5.5", + account=_make_account("acc-owner-fence-order"), + upstream=cast( + proxy_service.UpstreamWebSocket, + SimpleNamespace(send_text=AsyncMock(side_effect=send_error), close=AsyncMock()), + ), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=0.0, + idle_ttl_seconds=120.0, + durable_session_id="durable-owner-fence-order", + durable_owner_epoch=4, + ) + + async def cleanup(*_args: object, **_kwargs: object) -> None: + events.append("cleanup") + + monkeypatch.setattr(service, "_inline_http_bridge_image_urls", AsyncMock(return_value=request_state.request_text)) + monkeypatch.setattr(service, "_maybe_prewarm_http_bridge_session", AsyncMock()) + monkeypatch.setattr(service, "_acquire_request_state_response_create_admission", AsyncMock()) + monkeypatch.setattr(service, "_start_request_state_api_key_reservation_heartbeat", lambda *args, **kwargs: None) + monkeypatch.setattr(service, "_cleanup_http_bridge_submit_interruption", cleanup) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", AsyncMock()) + + with pytest.raises(proxy_module.ProxyResponseError): + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "", + queue_limit=1, + ) + + assert events == ["mark", "cleanup"] + service._durable_bridge.mark_operation_unknown.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_submit_http_bridge_preflight_failure_keeps_operation_pre_dispatch(monkeypatch): + service = proxy_service.ProxyService.__new__(proxy_service.ProxyService) + service._durable_bridge = None + proxy_service._initialize_http_bridge_retry_circuit(service) + request_state = proxy_service._WebSocketRequestState( + request_id="req-submit-preflight", + model="gpt-5.5", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.5"}', + operation_id="operation-preflight", + ) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-submit-preflight", None), + headers={}, + affinity=proxy_service._AffinityPolicy(key="sid-submit-preflight"), + request_model="gpt-5.5", + account=_make_account("acc_submit_preflight"), + upstream=cast( + proxy_service.UpstreamWebSocket, + SimpleNamespace(send_text=AsyncMock(), close=AsyncMock()), + ), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=0.0, + idle_ttl_seconds=120.0, + ) + send_frame = AsyncMock( + side_effect=proxy_module.ProxyResponseError( + 400, + {"error": {"code": "payload_too_large", "message": "response.create is too large"}}, + ) + ) + cleanup_dispatched: list[bool] = [] + + async def cleanup(*_args: object, **_kwargs: object) -> None: + cleanup_dispatched.append(request_state.operation_dispatched) + + monkeypatch.setattr(proxy_http_bridge_request_submit, "_send_http_bridge_request_text_with_archive_id", send_frame) + monkeypatch.setattr(service, "_inline_http_bridge_image_urls", AsyncMock(return_value=request_state.request_text)) + monkeypatch.setattr(service, "_maybe_prewarm_http_bridge_session", AsyncMock()) + monkeypatch.setattr(service, "_acquire_request_state_response_create_admission", AsyncMock()) + monkeypatch.setattr(service, "_start_request_state_api_key_reservation_heartbeat", lambda *args, **kwargs: None) + monkeypatch.setattr(service, "_cleanup_http_bridge_submit_interruption", cleanup) + monkeypatch.setattr(service, "_retire_http_bridge_after_drain_if_ready", AsyncMock()) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "", + queue_limit=1, + ) + + assert exc_info.value.status_code == 400 + send_frame.assert_awaited_once() + assert cleanup_dispatched == [False] + assert request_state.recovery_attempt_dispatched is False + assert request_state.operation_dispatched is False + + @pytest.mark.asyncio async def test_submit_http_bridge_request_checks_queue_before_inlining(monkeypatch): service = proxy_service.ProxyService.__new__(proxy_service.ProxyService) + service._durable_bridge = None proxy_service._initialize_http_bridge_retry_circuit(service) request_state = proxy_service._WebSocketRequestState( request_id="req_submit_queue_full_inline", @@ -43106,3 +47508,756 @@ async def __aexit__(self, *args): assert exc_info.value.status_code == 400 assert "image_download_failed" in json.dumps(exc_info.value.payload) + + +_COMPACT_REPLAY_NEUTRAL_INPUT: list[dict[str, object]] = [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [{"type": "output_text", "text": "hi there"}]}, + {"role": "user", "content": "please compact"}, +] + + +def _compact_replay_request(**overrides: object) -> ResponsesCompactRequest: + source: dict[str, object] = { + "model": "gpt-5.1", + "instructions": "hi", + "input": _COMPACT_REPLAY_NEUTRAL_INPUT, + "previous_response_id": "resp_anchor", + } + source.update(overrides) + return ResponsesCompactRequest.model_validate(source) + + +def test_compact_account_neutral_replay_payload_accepts_verified_full_resend() -> None: + replay = proxy_compact_service._compact_account_neutral_replay_payload(_compact_replay_request()) + assert replay is not None + assert getattr(replay, "previous_response_id", None) is None + assert replay.input == _COMPACT_REPLAY_NEUTRAL_INPUT + assert "previous_response_id" not in replay.to_payload() + + +def test_compact_account_neutral_replay_payload_requires_previous_response_anchor() -> None: + payload = ResponsesCompactRequest.model_validate( + {"model": "gpt-5.1", "instructions": "hi", "input": _COMPACT_REPLAY_NEUTRAL_INPUT} + ) + assert proxy_compact_service._compact_account_neutral_replay_payload(payload) is None + + +def test_compact_account_neutral_replay_payload_rejects_single_item_and_string_inputs() -> None: + single_item = _compact_replay_request(input=[{"role": "user", "content": "hello"}]) + assert proxy_compact_service._compact_account_neutral_replay_payload(single_item) is None + string_input = _compact_replay_request(input="hello there") + assert proxy_compact_service._compact_account_neutral_replay_payload(string_input) is None + + +def test_compact_account_neutral_replay_payload_rejects_server_assigned_item_ids() -> None: + payload = _compact_replay_request( + input=[ + {"role": "user", "content": "hello"}, + { + "type": "message", + "id": "msg_server_assigned", + "role": "assistant", + "content": [{"type": "output_text", "text": "hi there"}], + }, + {"role": "user", "content": "please compact"}, + ] + ) + assert proxy_compact_service._compact_account_neutral_replay_payload(payload) is None + + +def test_compact_account_neutral_replay_payload_rejects_encrypted_compaction_state() -> None: + payload = _compact_replay_request( + input=[ + {"type": "compaction", "encrypted_content": "enc_owner_scoped"}, + *_COMPACT_REPLAY_NEUTRAL_INPUT, + ] + ) + assert proxy_compact_service._compact_account_neutral_replay_payload(payload) is None + + +def test_compact_account_neutral_replay_payload_rejects_history_without_retained_output() -> None: + # Two fresh user turns may be a delta the owner resolves through the + # anchor; without retained assistant output the full resend is unproven. + payload = _compact_replay_request( + input=[ + {"role": "user", "content": "first delta turn"}, + {"role": "user", "content": "second delta turn"}, + ] + ) + assert proxy_compact_service._compact_account_neutral_replay_payload(payload) is None + + +def test_compact_account_neutral_replay_payload_rejects_history_without_fresh_followup() -> None: + # A transcript that ends on assistant output has no new client input after + # the retained output, so the retained-prior-output proof fails closed. + payload = _compact_replay_request( + input=[ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [{"type": "output_text", "text": "hi there"}]}, + ] + ) + assert proxy_compact_service._compact_account_neutral_replay_payload(payload) is None + + +def test_compact_account_neutral_replay_payload_rejects_wire_trimmed_history() -> None: + # An oversized history is trimmed on the wire to a head, marker, and tail; + # replaying that shortened transcript would compact an incomplete + # conversation, so the wire input must stay item-for-item identical. + oversized = "x" * 600_000 + payload = _compact_replay_request( + input=[ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [{"type": "output_text", "text": oversized}]}, + {"role": "assistant", "content": [{"type": "output_text", "text": "hi there"}]}, + {"role": "user", "content": "please compact"}, + ] + ) + assert proxy_compact_service._compact_account_neutral_replay_payload(payload) is None + + +def test_compact_account_neutral_replay_payload_accepts_canonical_lite_full_resend() -> None: + # A Responses-Lite history opens with the additional_tools bundle and its + # canonical developer instruction; the shared projection must recognize + # that developer message so the Lite surface stays recoverable. + payload = _compact_replay_request( + input=[ + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "function", "name": "exec", "parameters": {"type": "object"}}], + }, + {"type": "message", "role": "developer", "content": [{"type": "input_text", "text": "instructions"}]}, + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [{"type": "output_text", "text": "hi there"}]}, + {"role": "user", "content": "please compact"}, + ] + ) + replay = proxy_compact_service._compact_account_neutral_replay_payload(payload) + assert replay is not None + assert getattr(replay, "previous_response_id", None) is None + + +@pytest.mark.asyncio +async def test_stream_with_retry_validates_only_lifecycle_frames_and_settles_usage(monkeypatch): + # Delta frames must skip pydantic validation entirely (event=None) while + # lifecycle frames keep it, with byte-identical SSE output and unchanged + # usage settlement from the validated response.completed frame. + from app.modules.proxy import tool_call_dedupe + from app.modules.proxy._service.streaming import mixin as streaming_mixin_module + + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_lifecycle_only_validation") + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + mixin_validate = MagicMock(wraps=streaming_mixin_module.parse_sse_event_payload) + monkeypatch.setattr(streaming_mixin_module, "parse_sse_event_payload", mixin_validate) + dedupe_validate = MagicMock(wraps=tool_call_dedupe.parse_sse_event_payload) + monkeypatch.setattr(tool_call_dedupe, "parse_sse_event_payload", dedupe_validate) + + async def fake_core_stream_responses(*_args: object, **_kwargs: object): + yield 'data: {"type":"response.created","response":{"id":"resp_lifecycle_only"}}\n\n' + yield 'data: {"type":"response.output_text.delta","delta":"a"}\n\n' + yield 'data: {"type":"response.output_text.delta","delta":"b"}\n\n' + yield ( + 'data: {"type":"response.completed","response":{"id":"resp_lifecycle_only",' + '"usage":{"input_tokens":3,"output_tokens":5,' + '"input_tokens_details":{"cached_tokens":2},' + '"output_tokens_details":{"reasoning_tokens":1}}}}\n\n' + ) + + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_core_stream_responses) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}) + chunks = [ + chunk + async for chunk in service._stream_with_retry( + payload, + {"session_id": "sid-lifecycle-only"}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + request_transport="http", + upstream_stream_transport_override="http", + ) + ] + + # Only the created + completed lifecycle frames are validated; the two + # deltas are classified from the parsed dict and never re-validated by + # the parallel-tool-call rewrite either. + assert mixin_validate.call_count == 2 + assert dedupe_validate.call_count == 0 + # SSE output stays byte-identical to the canonical re-encode. + assert chunks[1] == 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"a"}\n\n' + assert json.loads(chunks[-1].split("data: ", 1)[1])["type"] == "response.completed" + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "success" + assert request_logs.calls[0]["request_id"] == "resp_lifecycle_only" + assert request_logs.calls[0]["input_tokens"] == 3 + assert request_logs.calls[0]["output_tokens"] == 5 + assert request_logs.calls[0]["cached_input_tokens"] == 2 + assert request_logs.calls[0]["reasoning_tokens"] == 1 + + +@pytest.mark.asyncio +async def test_stream_with_retry_rewrites_terminal_error_after_unvalidated_deltas(monkeypatch): + # A bare upstream ``error`` frame after unvalidated deltas must still be + # rewritten to a terminal response.failed under the SDK contract and + # settle as an error. + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_lifecycle_error_rewrite") + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + + async def fake_core_stream_responses(*_args: object, **_kwargs: object): + yield 'data: {"type":"response.created","response":{"id":"resp_lifecycle_err"}}\n\n' + yield 'data: {"type":"response.output_text.delta","delta":"a"}\n\n' + yield 'data: {"type":"error","message":"upstream exploded"}\n\n' + + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_core_stream_responses) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}) + chunks = [ + chunk + async for chunk in service._stream_with_retry( + payload, + {"session_id": "sid-lifecycle-error"}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + request_transport="http", + upstream_stream_transport_override="http", + ) + ] + + terminal = json.loads(chunks[-1].split("data: ", 1)[1]) + assert terminal["type"] == "response.failed" + assert terminal["response"]["error"]["message"] == "upstream exploded" + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "error" + + +@pytest.mark.asyncio +async def test_process_upstream_websocket_text_validates_only_lifecycle_frames(monkeypatch): + # Websocket relay: response.created keeps validated response-id + # assignment, deltas skip validation and are relayed with their original + # bytes when no response-id rewrite applies, and response.completed still + # hands a validated usage-bearing event to finalization. + from app.core.openai.models import OpenAIEvent + from app.modules.proxy import tool_call_dedupe + from app.modules.proxy._service.websocket import mixin as websocket_mixin_module + + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_lifecycle_validation") + finalize_request_state = AsyncMock() + monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize_request_state) + ws_validate = MagicMock(wraps=websocket_mixin_module.parse_sse_event_payload) + monkeypatch.setattr(websocket_mixin_module, "parse_sse_event_payload", ws_validate) + dedupe_validate = MagicMock(wraps=tool_call_dedupe.parse_sse_event_payload) + monkeypatch.setattr(tool_call_dedupe, "parse_sse_event_payload", dedupe_validate) + + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_lifecycle_validation", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + response_create_gate_acquired=True, + ) + pending_requests = deque([request_state]) + pending_lock = anyio.Lock() + upstream_control = proxy_service._WebSocketUpstreamControl() + response_create_gate = asyncio.Semaphore(0) + + async def relay(text: str) -> str: + return await service._process_upstream_websocket_text( + text, + account=account, + account_id_value=account.id, + pending_requests=pending_requests, + pending_lock=pending_lock, + api_key=None, + upstream_control=upstream_control, + response_create_gate=response_create_gate, + ) + + created_text = json.dumps( + {"type": "response.created", "response": {"id": "resp_ws_lifecycle", "status": "in_progress"}}, + separators=(",", ":"), + ) + await relay(created_text) + assert request_state.response_id == "resp_ws_lifecycle" + assert ws_validate.call_count == 1 + + delta_text = json.dumps( + { + "type": "response.output_text.delta", + "response_id": "resp_ws_lifecycle", + "sequence_number": 3, + "delta": "hello", + }, + separators=(",", ":"), + ) + downstream_delta = await relay(delta_text) + assert ws_validate.call_count == 1 + assert dedupe_validate.call_count == 0 + assert downstream_delta == delta_text + finalize_request_state.assert_not_awaited() + + completed_text = json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_ws_lifecycle", + "status": "completed", + "usage": {"input_tokens": 11, "output_tokens": 7}, + }, + }, + separators=(",", ":"), + ) + await relay(completed_text) + assert ws_validate.call_count == 2 + finalize_request_state.assert_awaited_once() + finalize_call = finalize_request_state.await_args + assert finalize_call is not None + assert finalize_call.kwargs["event_type"] == "response.completed" + completed_event = finalize_call.kwargs["event"] + assert isinstance(completed_event, OpenAIEvent) + assert completed_event.response is not None + assert completed_event.response.usage is not None + assert completed_event.response.usage.input_tokens == 11 + assert completed_event.response.usage.output_tokens == 7 + + +@pytest.mark.asyncio +async def test_stream_with_retry_rewrites_malformed_error_when_it_is_the_first_frame(monkeypatch): + # Regression: a malformed first upstream frame like + # {"type":"error","message":"..."} classifies as "error" but carries no + # error envelope (event=None). The first-frame path must apply the same + # SDK-contract fallback as the later-frame loop: rewrite to a terminal + # response.failed and settle as an error instead of leaking the raw frame + # with a success settlement. + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_first_frame_error_rewrite") + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + + async def fake_core_stream_responses(*_args: object, **_kwargs: object): + yield 'data: {"type":"error","message":"upstream exploded first"}\n\n' + + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_core_stream_responses) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}) + chunks = [ + chunk + async for chunk in service._stream_with_retry( + payload, + {"session_id": "sid-first-frame-error"}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + request_transport="http", + upstream_stream_transport_override="http", + ) + ] + + assert chunks + terminal = json.loads(chunks[-1].split("data: ", 1)[1]) + assert terminal["type"] == "response.failed" + assert terminal["response"]["error"]["message"] == "upstream exploded first" + assert terminal["response"]["error"]["code"] == "upstream_error" + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] != "success" + assert request_logs.calls[0]["status"] == "error" + + +@pytest.mark.asyncio +async def test_process_upstream_websocket_text_keeps_frame_bytes_when_response_id_already_matches(): + # Regression for the response-id rewrite identity fast-path: when the + # frame already carries the assigned downstream response id, the rewrite + # helper must hand back the original payload object so the relay forwards + # the upstream text without re-encoding it. + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_response_id_identity") + + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_response_id_identity", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + response_id="resp_ws_identity", + replay_downstream_response_id="resp_ws_identity", + ) + pending_requests = deque([request_state]) + upstream_control = proxy_service._WebSocketUpstreamControl() + + delta_text = json.dumps( + { + "type": "response.output_text.delta", + "response_id": "resp_ws_identity", + "sequence_number": 7, + "delta": "hello", + }, + separators=(",", ":"), + ) + downstream_delta = await service._process_upstream_websocket_text( + delta_text, + account=account, + account_id_value=account.id, + pending_requests=pending_requests, + pending_lock=anyio.Lock(), + api_key=None, + upstream_control=upstream_control, + response_create_gate=asyncio.Semaphore(0), + ) + + assert downstream_delta is delta_text + assert upstream_control.downstream_sequence_number == 7 + assert upstream_control.downstream_sequence_request_state is request_state + + +@pytest.mark.asyncio +async def test_process_and_forward_upstream_websocket_text_decodes_once_and_validates_lifecycle_only(monkeypatch): + # Product-path regression for the parse-once requirement: every direct + # upstream websocket text frame flows through + # _process_and_forward_upstream_websocket_text, which must json-decode the + # frame exactly once (shared by archive attribution and relay processing) + # and run pydantic validation only for lifecycle frames. + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_forward_parse_once") + finalize_request_state = AsyncMock() + monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize_request_state) + sent_downstream: list[str] = [] + + async def record_downstream(_websocket: object, *, text: str, **_kwargs: object) -> None: + sent_downstream.append(text) + + monkeypatch.setattr(service, "_send_downstream_websocket_text", record_downstream) + ws_validate = MagicMock(wraps=websocket_mixin_module.parse_sse_event_payload) + monkeypatch.setattr(websocket_mixin_module, "parse_sse_event_payload", ws_validate) + + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_forward_parse_once", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + response_create_gate_acquired=True, + archive_request_id="archive_ws_forward_parse_once", + ) + pending_requests = deque([request_state]) + pending_lock = anyio.Lock() + upstream_control = proxy_service._WebSocketUpstreamControl() + downstream_activity = proxy_service._DownstreamWebSocketActivity() + archived: list[tuple[object, str | None]] = [] + + class _ArchivingUpstream: + def archive_received(self, message: object) -> None: + archived.append((message, get_request_id())) + + upstream = _ArchivingUpstream() + + frame_payloads: list[dict[str, Any]] = [ + {"type": "response.created", "response": {"id": "resp_ws_forward_once", "status": "in_progress"}}, + {"type": "response.output_text.delta", "response_id": "resp_ws_forward_once", "delta": "hello"}, + { + "type": "response.completed", + "response": { + "id": "resp_ws_forward_once", + "status": "completed", + "usage": {"input_tokens": 4, "output_tokens": 2}, + }, + }, + ] + frame_texts = [json.dumps(frame, separators=(",", ":")) for frame in frame_payloads] + + loads_spy = MagicMock(wraps=json.loads) + monkeypatch.setattr(websocket_mixin_module.json, "loads", loads_spy) + + for frame_text in frame_texts: + terminal = await websocket_mixin_module._process_and_forward_upstream_websocket_text( + cast(Any, service), + cast(Any, SimpleNamespace()), + cast(Any, upstream), + message=SimpleNamespace(kind="text", text=frame_text), + text=frame_text, + account=account, + account_id_value=account.id, + pending_requests=pending_requests, + pending_lock=pending_lock, + client_send_lock=anyio.Lock(), + api_key=None, + upstream_control=upstream_control, + response_create_gate=asyncio.Semaphore(0), + downstream_activity=downstream_activity, + continuity_state=None, + codex_session_affinity=False, + ) + assert terminal is False + + # Exactly one json decode per frame across archive attribution and relay. + for frame_text in frame_texts: + decode_calls = [c for c in loads_spy.call_args_list if c.args and c.args[0] == frame_text] + assert len(decode_calls) == 1 + # Only the created + completed lifecycle frames are pydantic-validated; + # the delta is classified from the parsed dict without validation. + assert ws_validate.call_count == 2 + # Archive attribution still resolves the owning request from the shared + # parsed frame for every message. + assert [request_id for _message, request_id in archived] == ["archive_ws_forward_parse_once"] * 3 + # The delta frame is forwarded downstream with its original bytes. + assert sent_downstream[1] is frame_texts[1] + finalize_request_state.assert_awaited_once() + assert finalize_request_state.await_args is not None + assert finalize_request_state.await_args.kwargs["event_type"] == "response.completed" + + +def test_normalize_sse_event_block_rewrites_alias_on_both_event_and_data_lines(): + # A legacy alias must be rewritten on the SSE `event:` framing line too, + # not just inside the JSON payload — under verbatim relay a stale + # `event:` line would otherwise reach clients with mismatched framing. + block = 'event: response.text.delta\ndata: {"type":"response.text.delta","delta":"hi"}\n\n' + + normalized = proxy_module._normalize_sse_event_block(block) + + assert normalized == ( + 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"hi"}\n\n' + ) + + +def test_normalize_sse_event_block_rewrites_alias_split_across_data_lines(): + # A legal SSE payload split across multiple `data:` lines is only + # decodable as the combined value (fragments joined with "\n"); the alias + # rewrite must still land on both the payload and the `event:` line. + block = 'event: response.text.delta\ndata: {"type":"response.text.delta",\ndata: "delta":"hi"}\n\n' + + normalized = proxy_module._normalize_sse_event_block(block) + + assert normalized == ( + 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"hi"}\n\n' + ) + + +def test_normalize_sse_event_block_leaves_undecodable_multi_line_data_untouched(): + # When the combined multi-line payload cannot be decoded, neither surface + # may be rewritten: rewriting only the `event:` framing line would emit a + # frame whose framing and payload disagree about the event type. + block = 'event: response.text.delta\ndata: {"type":"response.te\ndata: xt.delta","delta":"hi"}\n\n' + + normalized = proxy_module._normalize_sse_event_block(block) + + assert normalized == block + + +def test_normalize_sse_event_block_skips_json_parsing_for_non_alias_types(monkeypatch) -> None: + # The alias normalizer only inspects blocks carrying one of the legacy + # alias names; canonical event types pass through without a JSON parse. + block = 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + def fail_json_parse(_: str) -> object: + raise AssertionError("json.loads should not run for blocks without an alias marker") + + monkeypatch.setattr(proxy_module.json, "loads", fail_json_parse) + + assert proxy_module._normalize_sse_event_block(block) == block + + +def test_normalize_stream_payload_for_http_block_skips_parse_for_canonical_frames(monkeypatch) -> None: + block = 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + def fail_json_parse(_: str) -> object: + raise AssertionError("json.loads should not run for canonical non-error frames") + + monkeypatch.setattr(proxy_module.json, "loads", fail_json_parse) + + assert proxy_module._normalize_stream_payload_for_http_block(block) == (block, "response.output_text.delta") + assert proxy_module._normalize_stream_payload_for_http_block(block, enforce_openai_sdk_contract=False) == ( + block, + "response.output_text.delta", + ) + + +def test_normalize_stream_payload_for_http_block_still_rewrites_error_frames(): + block = 'event: error\ndata: {"type":"error","message":"boom"}\n\n' + + normalized_block, normalized_type = proxy_module._normalize_stream_payload_for_http_block(block) + + assert normalized_type == "response.failed" + assert '"boom"' in normalized_block + + +def test_normalize_stream_payload_for_http_block_still_rewrites_error_envelopes_on_non_error_types(): + # A payload carrying a top-level error envelope is rewritten regardless of + # its event type; the `"error"` substring guard keeps it on the full-parse + # path. + block = ( + "event: response.output_text.delta\n" + 'data: {"type":"response.output_text.delta","error":{"message":"broken"},"delta":"hi"}\n\n' + ) + + normalized_block, normalized_type = proxy_module._normalize_stream_payload_for_http_block(block) + + assert normalized_type == "response.failed" + assert '"broken"' in normalized_block + + +@pytest.mark.asyncio +async def test_stream_with_retry_relays_unmodified_canonical_delta_frames_verbatim(monkeypatch): + # After the TTFT window settles, canonically framed delta frames are + # relayed with upstream bytes (raw UTF-8, upstream spacing) and are never + # JSON-parsed; usage settlement from the parsed terminal frame is + # unchanged. + from app.modules.proxy._service.streaming import mixin as streaming_mixin_module + + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_verbatim_relay") + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + mixin_parse = MagicMock(wraps=streaming_mixin_module.parse_sse_data_json) + monkeypatch.setattr(streaming_mixin_module, "parse_sse_data_json", mixin_parse) + + verbatim_delta = ( + 'event: response.output_text.delta\ndata: {"type": "response.output_text.delta", "delta": "안녕 upstream"}\n\n' + ) + + async def fake_core_stream_responses(*_args: object, **_kwargs: object): + yield 'event: response.created\ndata: {"type":"response.created","response":{"id":"resp_verbatim"}}\n\n' + yield 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"a"}\n\n' + yield verbatim_delta + yield ( + 'event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_verbatim",' + '"usage":{"input_tokens":3,"output_tokens":5}}}\n\n' + ) + + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_core_stream_responses) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}) + chunks = [ + chunk + async for chunk in service._stream_with_retry( + payload, + {"session_id": "sid-verbatim-relay"}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + request_transport="http", + upstream_stream_transport_override="http", + ) + ] + + # created (lifecycle), the first delta (TTFT window still open), and + # completed (lifecycle) are parsed; the settled second delta is relayed + # without any JSON parse. + assert mixin_parse.call_count == 3 + # Upstream bytes are preserved exactly: raw UTF-8 and upstream key + # spacing, not the ensure_ascii canonical re-encode. + assert chunks[2] == verbatim_delta + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "success" + assert request_logs.calls[0]["input_tokens"] == 3 + assert request_logs.calls[0]["output_tokens"] == 5 + + +@pytest.mark.asyncio +async def test_stream_with_retry_reframes_data_only_delta_frames_after_ttft(monkeypatch): + # Data-only frames (no `event:` line, e.g. bridge rewrite leftovers) never + # take the verbatim path: they are parsed and re-serialized with canonical + # `event: ` framing so named-event (EventSource) clients keep seeing + # the event name — the 5ee532cb regression class. + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_verbatim_data_only") + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + + async def fake_core_stream_responses(*_args: object, **_kwargs: object): + yield 'event: response.created\ndata: {"type":"response.created","response":{"id":"resp_data_only"}}\n\n' + yield 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"a"}\n\n' + yield 'data: {"type":"response.output_text.delta","delta":"b"}\n\n' + yield ( + 'event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_data_only",' + '"usage":{"input_tokens":1,"output_tokens":1}}}\n\n' + ) + + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_core_stream_responses) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}) + chunks = [ + chunk + async for chunk in service._stream_with_retry( + payload, + {"session_id": "sid-verbatim-data-only"}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + request_transport="http", + upstream_stream_transport_override="http", + ) + ] + + assert chunks[2] == 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"b"}\n\n' + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "success" diff --git a/tests/unit/test_proxy_websocket_client.py b/tests/unit/test_proxy_websocket_client.py index 75c1e60166..547eb7aa0e 100644 --- a/tests/unit/test_proxy_websocket_client.py +++ b/tests/unit/test_proxy_websocket_client.py @@ -10,18 +10,10 @@ import aiohttp import pytest from websockets.asyncio.server import serve as websocket_serve -from websockets.client import ClientProtocol from websockets.datastructures import Headers -from websockets.exceptions import ( - ConnectionClosedError, - InvalidHandshake, - InvalidProxy, - InvalidProxyMessage, - InvalidStatus, -) +from websockets.exceptions import ConnectionClosedError, InvalidHandshake, InvalidProxy, InvalidStatus from websockets.frames import Close from websockets.http11 import Response -from websockets.uri import parse_uri import app.core.clients.proxy_websocket as proxy_websocket_module from app.core.clients.codex import CodexTransportError, CodexWebSocketResult @@ -420,11 +412,11 @@ async def fake_websocket_connect(url: str, **kwargs): assert kwargs["origin"] == "https://chatgpt.com" assert kwargs["user_agent_header"] == "Codex CLI Test" assert kwargs["proxy"] is None - assert "create_connection" not in kwargs assert kwargs["open_timeout"] == 7.0 assert "ping_interval" not in kwargs assert kwargs["ping_timeout"] == 120.0 assert kwargs["max_size"] == 4321 + assert kwargs["compression"] is None assert "subprotocols" not in kwargs additional_headers = cast(dict[str, str], kwargs["additional_headers"]) assert additional_headers["Authorization"] == "Bearer access-token" @@ -436,274 +428,6 @@ async def fake_websocket_connect(url: str, **kwargs): assert "Origin" not in additional_headers -@pytest.mark.asyncio -async def test_proxy_connection_lost_before_connection_made_is_safe(): - protocol = ClientProtocol(parse_uri("wss://chatgpt.com/backend-api/codex/responses")) - connection = proxy_websocket_module._ProxySetupSafeClientConnection(protocol) - failure = ConnectionResetError("proxy TLS transport closed") - - assert not hasattr(connection, "recv_messages") - - connection.connection_lost(failure) - - assert not hasattr(connection, "recv_messages") - assert connection.connection_lost_waiter.done() - assert connection.recv_exc is failure - assert protocol.state.name == "CLOSED" - assert protocol.handshake_exc is not None - - -@pytest.mark.asyncio -async def test_proxy_connection_lost_after_connection_made_delegates(monkeypatch): - protocol = ClientProtocol(parse_uri("wss://chatgpt.com/backend-api/codex/responses")) - connection = proxy_websocket_module._ProxySetupSafeClientConnection(protocol) - connection.recv_messages = cast(Any, object()) - delegated: list[tuple[object, Exception | None]] = [] - - def fake_connection_lost(self, exc): - delegated.append((self, exc)) - - monkeypatch.setattr(proxy_websocket_module.ClientConnection, "connection_lost", fake_connection_lost) - failure = ConnectionResetError("established transport closed") - - connection.connection_lost(failure) - - assert delegated == [(connection, failure)] - - -@pytest.mark.asyncio -async def test_proxied_websocket_uses_safe_connection_adapter(monkeypatch): - fake_connection = _FakeConnection() - seen: dict[str, object] = {} - - async def fake_websocket_connect(url: str, **kwargs): - seen["url"] = url - seen["kwargs"] = kwargs - return fake_connection - - monkeypatch.setattr(proxy_websocket_module, "websocket_connect", fake_websocket_connect) - monkeypatch.setattr( - proxy_websocket_module, - "resolve_websocket_proxy_from_env", - lambda url, env: "http://proxy.test:3128", - ) - monkeypatch.setattr( - proxy_websocket_module, - "get_settings", - lambda: SimpleNamespace( - upstream_base_url="https://chatgpt.com/backend-api", - upstream_connect_timeout_seconds=7.0, - proxy_downstream_websocket_idle_timeout_seconds=120.0, - max_sse_event_bytes=4321, - upstream_websocket_trust_env=True, - upstream_websocket_proxy_env=lambda: {}, - ), - ) - - await connect_responses_websocket( - {"openai-beta": "responses_websockets=2026-02-06"}, - "access-token", - "account-123", - allow_direct_egress=True, - ) - - kwargs = cast(dict[str, object], seen["kwargs"]) - assert kwargs["proxy"] == "http://proxy.test:3128" - assert kwargs["create_connection"] is proxy_websocket_module._ProxySetupSafeClientConnection - - -@pytest.mark.asyncio -async def test_proxied_websocket_early_close_retries_fresh_tunnel_on_same_account(monkeypatch): - fake_connection = _FakeConnection() - attempts = 0 - - async def failing_websocket_connect(url: str, **kwargs): - nonlocal attempts - del url, kwargs - attempts += 1 - if attempts == 1: - raise ConnectionResetError("proxy TLS transport closed") - return fake_connection - - monkeypatch.setattr(proxy_websocket_module, "websocket_connect", failing_websocket_connect) - monkeypatch.setattr( - proxy_websocket_module, - "resolve_websocket_proxy_from_env", - lambda url, env: "http://proxy.test:3128", - ) - monkeypatch.setattr( - proxy_websocket_module, - "get_settings", - lambda: SimpleNamespace( - upstream_base_url="https://chatgpt.com/backend-api", - upstream_connect_timeout_seconds=7.0, - proxy_downstream_websocket_idle_timeout_seconds=120.0, - max_sse_event_bytes=4321, - upstream_websocket_trust_env=True, - upstream_websocket_proxy_env=lambda: {}, - ), - ) - - websocket = await connect_responses_websocket( - {"openai-beta": "responses_websockets=2026-02-06"}, - "access-token", - "account-123", - allow_direct_egress=True, - ) - await websocket.send_text("hello") - - assert attempts == 2 - assert fake_connection.sent == ["hello"] - - -@pytest.mark.asyncio -async def test_proxied_websocket_invalid_proxy_message_retries_fresh_tunnel_on_same_account(monkeypatch): - fake_connection = _FakeConnection() - attempts = 0 - - async def failing_websocket_connect(url: str, **kwargs): - nonlocal attempts - del url, kwargs - attempts += 1 - if attempts == 1: - raise InvalidProxyMessage("did not receive a valid HTTP response from proxy") - return fake_connection - - monkeypatch.setattr(proxy_websocket_module, "websocket_connect", failing_websocket_connect) - monkeypatch.setattr( - proxy_websocket_module, - "resolve_websocket_proxy_from_env", - lambda url, env: "http://proxy.test:3128", - ) - monkeypatch.setattr( - proxy_websocket_module, - "get_settings", - lambda: SimpleNamespace( - upstream_base_url="https://chatgpt.com/backend-api", - upstream_connect_timeout_seconds=7.0, - proxy_downstream_websocket_idle_timeout_seconds=120.0, - max_sse_event_bytes=4321, - upstream_websocket_trust_env=True, - upstream_websocket_proxy_env=lambda: {}, - ), - ) - - websocket = await connect_responses_websocket( - {"openai-beta": "responses_websockets=2026-02-06"}, - "access-token", - "account-123", - allow_direct_egress=True, - ) - await websocket.send_text("hello") - - assert attempts == 2 - assert fake_connection.sent == ["hello"] - - -@pytest.mark.asyncio -async def test_proxied_websocket_invalid_proxy_message_exhaustion_is_typed_pre_dispatch(monkeypatch): - attempts = 0 - - async def failing_websocket_connect(url: str, **kwargs): - nonlocal attempts - del url, kwargs - attempts += 1 - raise InvalidProxyMessage("did not receive a valid HTTP response from proxy") - - monkeypatch.setattr(proxy_websocket_module, "websocket_connect", failing_websocket_connect) - monkeypatch.setattr( - proxy_websocket_module, - "resolve_websocket_proxy_from_env", - lambda url, env: "http://proxy.test:3128", - ) - monkeypatch.setattr( - proxy_websocket_module, - "get_settings", - lambda: SimpleNamespace( - upstream_base_url="https://chatgpt.com/backend-api", - upstream_connect_timeout_seconds=7.0, - proxy_downstream_websocket_idle_timeout_seconds=120.0, - max_sse_event_bytes=4321, - upstream_websocket_trust_env=True, - upstream_websocket_proxy_env=lambda: {}, - ), - ) - - with pytest.raises(ProxyResponseError) as exc_info: - await connect_responses_websocket( - {"openai-beta": "responses_websockets=2026-02-06"}, - "access-token", - "account-123", - allow_direct_egress=True, - ) - - assert attempts == 2 - assert exc_info.value.failure_phase == "connect" - assert exc_info.value.failure_detail == "shared_proxy_connect_pre_dispatch_exhausted" - assert exc_info.value.failure_exception_type == "InvalidProxyMessage" - assert exc_info.value.retryable_same_contract is False - - -@pytest.mark.asyncio -async def test_real_proxy_tunnel_close_before_tls_has_no_event_loop_callback_failure(monkeypatch): - proxy_connections = 0 - - async def closing_proxy(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - nonlocal proxy_connections - proxy_connections += 1 - await reader.readuntil(b"\r\n\r\n") - writer.write(b"HTTP/1.1 200 Connection Established\r\n\r\n") - await writer.drain() - writer.close() - await writer.wait_closed() - - server = await asyncio.start_server(closing_proxy, "127.0.0.1", 0) - proxy_port = server.sockets[0].getsockname()[1] - loop = asyncio.get_running_loop() - previous_exception_handler = loop.get_exception_handler() - callback_failures: list[dict[str, object]] = [] - loop.set_exception_handler(lambda _loop, context: callback_failures.append(context)) - monkeypatch.setattr( - proxy_websocket_module, - "resolve_websocket_proxy_from_env", - lambda url, env: f"http://127.0.0.1:{proxy_port}", - ) - monkeypatch.setattr( - proxy_websocket_module, - "get_settings", - lambda: SimpleNamespace( - upstream_base_url="https://chatgpt.com/backend-api", - upstream_connect_timeout_seconds=2.0, - proxy_downstream_websocket_idle_timeout_seconds=120.0, - max_sse_event_bytes=4321, - upstream_websocket_trust_env=True, - upstream_websocket_proxy_env=lambda: {}, - ), - ) - - try: - with pytest.raises(ProxyResponseError) as exc_info: - await connect_responses_websocket( - {"openai-beta": "responses_websockets=2026-02-06"}, - "access-token", - "account-123", - allow_direct_egress=True, - ) - await asyncio.sleep(0) - finally: - server.close() - await server.wait_closed() - loop.set_exception_handler(previous_exception_handler) - - assert exc_info.value.failure_phase == "connect" - assert exc_info.value.failure_detail == "shared_proxy_connect_pre_dispatch_exhausted" - assert exc_info.value.failure_exception_type == "ConnectionResetError" - assert exc_info.value.retryable_same_contract is False - assert is_confirmed_pre_dispatch_transport_error(exc_info.value) is False - assert proxy_connections == 2 - assert callback_failures == [] - - @pytest.mark.asyncio async def test_direct_websocket_network_send_and_receive_are_typed_and_rotate_without_reconnect(monkeypatch): class _NetworkFailureConnection(_FakeConnection): diff --git a/tests/unit/test_proxy_websocket_model_source_guard.py b/tests/unit/test_proxy_websocket_model_source_guard.py new file mode 100644 index 0000000000..8488f0a087 --- /dev/null +++ b/tests/unit/test_proxy_websocket_model_source_guard.py @@ -0,0 +1,913 @@ +"""Tests for the WebSocket model-source guard. + +Model sources are only reachable from the HTTP request path, so the WebSocket +transport must refuse them. Two guards cover the two ways a turn can reach an +upstream: + +* the connect guard, which fails the connect with a service-level ``503`` that + Codex clients transparently fall back from onto HTTP; +* the reuse guard, which fails a later ``response.create`` that switches to a + source-owned model on an already-open subscription upstream. +""" + +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager +from types import SimpleNamespace +from typing import cast +from unittest.mock import AsyncMock + +import anyio +import pytest +from fastapi import WebSocket + +import app.modules.model_sources.selection as source_selection +import app.modules.proxy._service.websocket.mixin as ws_mixin +from app.modules.api_keys.service import ApiKeyData +from app.modules.model_sources.selection import ( + effective_model_for_api_key, + responses_model_is_source_owned, +) +from app.modules.proxy import service as proxy_service +from tests.unit.test_proxy_utils import ( + _make_account, + _make_proxy_settings, + _QueuedTestUpstreamWebSocket, + _repo_factory, + _RequestLogsRecorder, + _SettingsCache, +) + +pytestmark = pytest.mark.unit + + +def _api_key(*, enforced_model: str | None = None) -> ApiKeyData: + from datetime import datetime + + return ApiKeyData( + id="key_ws_guard", + name="ws guard", + key_prefix="sk-test-ws-guard", + allowed_models=[], + enforced_model=enforced_model, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=datetime(2026, 1, 1), + last_used_at=None, + ) + + +def _request_state(model: str) -> ws_mixin._WebSocketRequestState: + return proxy_service._WebSocketRequestState( + request_id="req-ws-guard", + model=model, + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=anyio.current_time(), + ) + + +def test_effective_model_prefers_enforced_model() -> None: + assert effective_model_for_api_key(None, "gpt-5.6-sol") == "gpt-5.6-sol" + assert effective_model_for_api_key(_api_key(), "gpt-5.6-sol") == "gpt-5.6-sol" + assert effective_model_for_api_key(_api_key(enforced_model="qwen3.8-max"), "gpt-5.6-sol") == "qwen3.8-max" + + +@pytest.mark.asyncio +async def test_source_ownership_fails_open_when_resolution_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """A database failure must not be able to reject a subscription turn. + + The lookup runs after the turn's usage reservation is acquired but before it + is registered for cleanup, so a propagating error would tear the session + down and strand the reservation. Failing open degrades to the behaviour that + existed before the guard. + """ + + async def boom(*args, **kwargs): # noqa: ANN002, ANN003, ANN202 + raise RuntimeError("model_sources table is unavailable") + + monkeypatch.setattr(source_selection, "select_responses_model_source", boom) + + assert await responses_model_is_source_owned("qwen3.8-max", None) is False + + +async def _run_connect_guard( + monkeypatch: pytest.MonkeyPatch, + *, + is_source_owned: bool, + api_key: ApiKeyData | None = None, + request_state_api_key: ApiKeyData | None = None, +): + """Drive ``_connect_proxy_websocket`` far enough to observe the connect guard. + + ``_select_websocket_connect_account`` stands in for the failover loop the + guard short-circuits, so reaching it means the guard did not fire. + """ + emitted: dict[str, object] = {} + selection_calls = 0 + seen_api_keys: list[ApiKeyData | None] = [] + + async def fake_is_source_owned(model, key, *, raw_model=None): # noqa: ANN001 + seen_api_keys.append(key) + return is_source_owned + + async def fake_emit(self, websocket, **kwargs): # noqa: ANN001 + emitted.update(kwargs) + + async def fake_select(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + nonlocal selection_calls + selection_calls += 1 + return None + + settings = _make_proxy_settings() + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(ws_mixin, "responses_model_is_source_owned", fake_is_source_owned) + monkeypatch.setattr(proxy_service.ProxyService, "_emit_websocket_connect_failure", fake_emit) + monkeypatch.setattr(proxy_service.ProxyService, "_select_websocket_connect_account", fake_select) + + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + request_state = _request_state("qwen3.8-max") + request_state.api_key = request_state_api_key + + account, upstream = await service._connect_proxy_websocket( + {}, + sticky_key=None, + sticky_kind=None, + prefer_earlier_reset=False, + routing_strategy="capacity_weighted", + model="qwen3.8-max", + request_state=request_state, + api_key=api_key, + client_send_lock=anyio.Lock(), + websocket=AsyncMock(), + ) + return account, upstream, emitted, selection_calls, seen_api_keys + + +@pytest.mark.asyncio +async def test_connect_guard_fails_session_for_source_owned_model(monkeypatch: pytest.MonkeyPatch) -> None: + account, upstream, emitted, selection_calls, _ = await _run_connect_guard(monkeypatch, is_source_owned=True) + + assert account is None + assert upstream is None + assert selection_calls == 0, "the guard must short-circuit before account selection" + assert emitted["error_code"] == "model_source_requires_http_transport" + assert emitted["status_code"] == 503, "a 4xx is terminal client-side and would strand the fallback" + assert emitted["account_id"] is None + + +@pytest.mark.asyncio +async def test_connect_guard_ignores_subscription_models(monkeypatch: pytest.MonkeyPatch) -> None: + account, _upstream, emitted, selection_calls, _ = await _run_connect_guard(monkeypatch, is_source_owned=False) + + assert account is None # the stubbed selector returns no account + assert selection_calls >= 1, "subscription models must proceed to account selection" + assert emitted == {} + + +@pytest.mark.asyncio +async def test_connect_guard_uses_the_per_request_api_key(monkeypatch: pytest.MonkeyPatch) -> None: + """A policy refresh mid-session must not be judged against the stale session key. + + ``request_state.api_key`` is refreshed per request; the session key captured + at connect time can be arbitrarily old on a long-lived socket, and the reuse + guard already consults the fresh one. + """ + session_key = _api_key() + refreshed_key = _api_key(enforced_model="qwen3.8-max") + + *_, seen_api_keys = await _run_connect_guard( + monkeypatch, + is_source_owned=True, + api_key=session_key, + request_state_api_key=refreshed_key, + ) + + assert seen_api_keys == [refreshed_key] + + +def _text_frame(payload: dict[str, object]) -> SimpleNamespace: + return SimpleNamespace( + kind="text", + text=json.dumps(payload, separators=(",", ":")), + data=None, + close_code=None, + error=None, + error_code=None, + ) + + +def _completed_turn(response_id: str) -> list[SimpleNamespace]: + return [ + _text_frame({"type": "response.created", "response": {"id": response_id, "status": "in_progress"}}), + _text_frame( + { + "type": "response.completed", + "response": { + "id": response_id, + "status": "completed", + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + } + ), + ] + + +class _Downstream: + """A downstream socket that replays a scripted sequence of client frames.""" + + def __init__(self, request_texts: list[str]) -> None: + self.pending = list(request_texts) + self.done = asyncio.Event() + self.sent_text: list[str] = [] + self.turn_completed = asyncio.Event() + + async def receive(self) -> dict[str, object]: + if self.pending: + # Wait for the previous turn to settle so the frames stay ordered. + if len(self.pending) < 1 or self.sent_text: + await self.turn_completed.wait() + self.turn_completed.clear() + return {"type": "websocket.receive", "text": self.pending.pop(0)} + await self.done.wait() + return {"type": "websocket.disconnect"} + + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + payload = json.loads(text) + if payload.get("type") in {"response.completed", "response.failed", "error"}: + self.turn_completed.set() + if not self.pending: + self.done.set() + + async def send_bytes(self, _data: bytes) -> None: + return None + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + del code, reason + self.done.set() + + +def _create_frame(model: str) -> str: + return json.dumps( + { + "type": "response.create", + "model": model, + "instructions": "", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}], + "stream": True, + }, + separators=(",", ":"), + ) + + +@pytest.mark.asyncio +async def test_first_turn_reaches_connect_guard_not_the_reuse_guard( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A fresh socket must reach the connect guard, which emits the 503 that + makes Codex clients fall back to HTTP. + + The per-frame reuse guard runs before connection, so if it were not gated on + an already-open upstream it would emit a terminal ``invalid_request_error`` + for the very first ``response.create`` and preempt the fallback, leaving + model sources unreachable. + """ + settings = _make_proxy_settings() + settings.stream_idle_timeout_seconds = 300.0 + settings.proxy_downstream_websocket_idle_timeout_seconds = 120.0 + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_source_guard_first_turn") + upstream = _QueuedTestUpstreamWebSocket(_completed_turn("resp_first_turn")) + + connect_called = False + + async def fake_connect(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + nonlocal connect_called + connect_called = True + return account, upstream + + async def always_source_owned(*args, **kwargs) -> bool: # noqa: ANN002, ANN003 + return True + + monkeypatch.setattr(ws_mixin, "responses_model_is_source_owned", always_source_owned) + monkeypatch.setattr(proxy_service.ProxyService, "_connect_proxy_websocket", fake_connect) + monkeypatch.setattr(service, "_resolve_compact_turn_state_owner", AsyncMock(return_value=None)) + + downstream = _Downstream([_create_frame("qwen3.8-max")]) + + await service.proxy_responses_websocket( + cast(WebSocket, downstream), + {}, + codex_session_affinity=False, + openai_cache_affinity=False, + api_key=None, + ) + + assert connect_called, "first turn must reach the connect path, not the per-frame reuse guard" + assert not any("model_source_requires_http_transport" in text for text in downstream.sent_text), ( + "the reuse guard must not preempt the connect-path 503 on a fresh socket" + ) + + +@pytest.mark.asyncio +async def test_reuse_guard_rejects_a_later_source_owned_turn(monkeypatch: pytest.MonkeyPatch) -> None: + """A second turn that switches to a source-owned model must not be forwarded. + + Socket reuse skips connection entirely, so without the reuse guard the frame + would go to the subscription account already attached to the open upstream + and be rejected by the backend with the unsupported-model error. + """ + settings = _make_proxy_settings() + settings.stream_idle_timeout_seconds = 300.0 + settings.proxy_downstream_websocket_idle_timeout_seconds = 120.0 + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_source_guard_reuse") + upstream = _QueuedTestUpstreamWebSocket(_completed_turn("resp_turn_one")) + + async def fake_connect(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + return account, upstream + + # Only the second turn's model is source-owned. + async def source_owned_for_qwen(model, _api_key, *, raw_model=None): # noqa: ANN001 + return model == "qwen3.8-max" + + released = AsyncMock() + + monkeypatch.setattr(ws_mixin, "responses_model_is_source_owned", source_owned_for_qwen) + monkeypatch.setattr(proxy_service.ProxyService, "_connect_proxy_websocket", fake_connect) + monkeypatch.setattr(proxy_service.ProxyService, "_release_websocket_request_state_reservation", released) + monkeypatch.setattr(service, "_resolve_compact_turn_state_owner", AsyncMock(return_value=None)) + + downstream = _Downstream([_create_frame("gpt-5.6-sol"), _create_frame("qwen3.8-max")]) + + await service.proxy_responses_websocket( + cast(WebSocket, downstream), + {}, + codex_session_affinity=False, + openai_cache_affinity=False, + api_key=None, + ) + + assert any("resp_turn_one" in text for text in downstream.sent_text), "the subscription turn must complete" + assert any("model_source_requires_http_transport" in text for text in downstream.sent_text), ( + "the source-owned turn must be rejected by the reuse guard" + ) + assert len(upstream.sent_text) == 1, "the rejected turn must not be forwarded upstream" + assert released.await_count >= 1, "the rejected turn must release its usage reservation" + + +def _alias_allowlist_api_key() -> ApiKeyData: + """A key that allowlists exactly the alias an alias-named source exposes. + + ``validate_model_access`` resolves aliases on both sides, so this key also + admits plain ``gpt-5`` requests — but ``select_responses_model_source`` + filters candidates against the allowlist *exactly*, which keeps the + normalized ``gpt-5`` candidate away from source lookup. That makes the raw + alias the only candidate that can match the source, in the unit fake, the + integration database, and production alike. + """ + from datetime import datetime + + return ApiKeyData( + id="key_ws_guard_alias", + name="ws guard alias", + key_prefix="sk-test-ws-alias", + allowed_models=["gpt-5-high"], + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=datetime(2026, 1, 1), + last_used_at=None, + ) + + +class _AliasSourceCatalog: + """Fake only the I/O seams underneath ``select_responses_model_source``. + + The candidate construction in ``responses_model_is_source_owned`` and the + allowlist/registry filtering in ``select_responses_model_source`` stay + real; this stands in for the database session/repository and records which + candidates were actually offered to the catalog, so tests can assert the + raw client alias physically reached source selection (monkeypatching + ``responses_model_is_source_owned`` itself would test the stub instead). + """ + + def __init__(self, source_models: set[str]) -> None: + self.source_models = source_models + self.seen_candidates: list[str] = [] + + def install(self, monkeypatch: pytest.MonkeyPatch) -> None: + catalog = self + + class _FakeRepository: + def __init__(self, _session: object) -> None: + pass + + async def find_responses_source_for_model( + self, + candidate: str, + *, + allowed_source_ids=None, # noqa: ANN001 + require_streaming: bool = False, + ): # noqa: ANN202 + catalog.seen_candidates.append(candidate) + if candidate in catalog.source_models: + return SimpleNamespace(id="src_alias", name="alias-source", enabled=True) + return None + + @asynccontextmanager + async def fake_session(): # noqa: ANN202 + yield object() + + monkeypatch.setattr(source_selection, "ModelSourcesRepository", _FakeRepository) + monkeypatch.setattr(source_selection, "get_background_session", fake_session) + monkeypatch.setattr(source_selection, "detach_session_objects", lambda _session: None) + + +class _TurnDrivenUpstream: + """An upstream that releases each scripted turn only after its request. + + ``_QueuedTestUpstreamWebSocket`` queues every frame up front, which would + let a second turn's events race ahead of the second ``response.create``. + Here turn N's events become readable only after the Nth upstream send, so + a turn that is (correctly) rejected before forwarding leaves its events + unread, and a (buggy) forwarded turn completes cleanly instead of hanging + the session — the pre-fix failure stays a crisp assertion failure. + """ + + def __init__(self, turns: list[list[SimpleNamespace]]) -> None: + self._turns = list(turns) + self._messages: asyncio.Queue[SimpleNamespace] = asyncio.Queue() + self.close_seen = asyncio.Event() + self.sent_text: list[str] = [] + + def response_header(self, name: str) -> str | None: + del name + return None + + async def receive(self) -> SimpleNamespace: + message = await self._messages.get() + if message.kind == "close": + self.close_seen.set() + return message + + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + if self._turns: + for event in self._turns.pop(0): + self._messages.put_nowait(event) + + async def send_bytes(self, _data: bytes) -> None: + return None + + async def close(self) -> None: + self.close_seen.set() + + +@pytest.mark.asyncio +async def test_reuse_guard_sees_the_raw_model_alias(monkeypatch: pytest.MonkeyPatch) -> None: + """A later turn asking for an alias-only source model must be rejected. + + ``apply_api_key_enforcement`` normalizes ``gpt-5-high`` to ``gpt-5`` + during request preparation, so a guard that judges only + ``request_state.model`` misses a source that exposes exactly + ``gpt-5-high`` — while the HTTP path routes the identical request to the + source via its pre-enforcement ``raw_source_model``. The real + ``responses_model_is_source_owned`` and ``select_responses_model_source`` + run here; only the catalog I/O is faked (regression for the raw-alias P2). + """ + settings = _make_proxy_settings() + settings.stream_idle_timeout_seconds = 300.0 + settings.proxy_downstream_websocket_idle_timeout_seconds = 120.0 + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + catalog = _AliasSourceCatalog({"gpt-5-high"}) + catalog.install(monkeypatch) + + api_key = _alias_allowlist_api_key() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_source_guard_alias") + upstream = _TurnDrivenUpstream([_completed_turn("resp_turn_one"), _completed_turn("resp_turn_two")]) + + async def fake_connect(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + return account, upstream + + async def fake_refresh(key): # noqa: ANN001, ANN202 + return api_key + + released = AsyncMock() + monkeypatch.setattr(proxy_service.ProxyService, "_connect_proxy_websocket", fake_connect) + monkeypatch.setattr(service, "_refresh_websocket_api_key_policy", fake_refresh) + monkeypatch.setattr(service, "_reserve_websocket_api_key_usage", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_service.ProxyService, "_release_websocket_request_state_reservation", released) + monkeypatch.setattr(service, "_resolve_compact_turn_state_owner", AsyncMock(return_value=None)) + + downstream = _Downstream([_create_frame("gpt-5"), _create_frame("gpt-5-high")]) + + await service.proxy_responses_websocket( + cast(WebSocket, downstream), + {}, + codex_session_affinity=False, + openai_cache_affinity=False, + api_key=api_key, + ) + + assert any("resp_turn_one" in text for text in downstream.sent_text), "the subscription turn must complete" + assert "gpt-5-high" in catalog.seen_candidates, ( + "the client's raw alias must reach source selection; the normalized " + "'gpt-5' is filtered out by the key's exact allowlist" + ) + assert any("model_source_requires_http_transport" in text for text in downstream.sent_text), ( + "the alias-owned turn must be rejected by the reuse guard" + ) + assert len(upstream.sent_text) == 1, "the alias turn must not be forwarded to the subscription upstream" + assert released.await_count >= 1, "the rejected turn must release its usage reservation" + + +@pytest.mark.asyncio +async def test_connect_guard_sees_the_raw_model_alias(monkeypatch: pytest.MonkeyPatch) -> None: + """The connect guard must judge the pre-enforcement alias too. + + The session loop hands ``_connect_proxy_websocket`` the post-enforcement + ``request_state.model``, so the raw alias must ride on the prepared + request state itself for the connect-time check to see it. This drives the + real ``_prepare_websocket_response_create_request`` (where enforcement + normalizes the alias) into the real connect guard. + """ + settings = _make_proxy_settings() + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + catalog = _AliasSourceCatalog({"gpt-5-high"}) + catalog.install(monkeypatch) + + api_key = _alias_allowlist_api_key() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + + emitted: dict[str, object] = {} + selection_calls = 0 + + async def fake_refresh(key): # noqa: ANN001, ANN202 + return api_key + + async def fake_emit(self, websocket, **kwargs): # noqa: ANN001, ANN003, ANN202 + emitted.update(kwargs) + + async def fake_select(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + nonlocal selection_calls + selection_calls += 1 + return None + + monkeypatch.setattr(service, "_refresh_websocket_api_key_policy", fake_refresh) + monkeypatch.setattr(service, "_reserve_websocket_api_key_usage", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_service.ProxyService, "_emit_websocket_connect_failure", fake_emit) + monkeypatch.setattr(proxy_service.ProxyService, "_select_websocket_connect_account", fake_select) + + prepared = await service._prepare_websocket_response_create_request( + json.loads(_create_frame("gpt-5-high")), + headers={}, + codex_session_affinity=False, + openai_cache_affinity=False, + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=0, + api_key=api_key, + ) + assert prepared.request_state.model == "gpt-5", "enforcement is expected to normalize the alias" + + account, upstream = await service._connect_proxy_websocket( + {}, + sticky_key=None, + sticky_kind=None, + prefer_earlier_reset=False, + routing_strategy="capacity_weighted", + # Exactly what the session loop passes: the normalized model. + model=prepared.request_state.model, + request_state=prepared.request_state, + api_key=api_key, + client_send_lock=anyio.Lock(), + websocket=AsyncMock(), + ) + + assert account is None + assert upstream is None + assert selection_calls == 0, "the connect guard must short-circuit before account selection" + assert emitted.get("error_code") == "model_source_requires_http_transport" + assert emitted.get("status_code") == 503 + assert "gpt-5-high" in catalog.seen_candidates, "the raw alias must reach source selection on the connect path" + + +def _create_frame_with_input(model: str, input_items: list[dict[str, object]]) -> str: + return json.dumps( + { + "type": "response.create", + "model": model, + "instructions": "", + "input": input_items, + "stream": True, + }, + separators=(",", ":"), + ) + + +def _input_file_frame(model: str, file_id: str) -> str: + return _create_frame_with_input( + model, + [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "summarize the attachment"}, + {"type": "input_file", "file_id": file_id}, + ], + } + ], + ) + + +def _compaction_trigger_frame(model: str) -> str: + return _create_frame_with_input( + model, + [ + {"role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + {"type": "compaction_trigger"}, + ], + ) + + +class _TurnSerializedDownstream: + """A downstream that sends each next frame only after the prior turn ends. + + ``_Downstream`` hands the session loop the next frame as soon as it asks, + so a second turn can be dispatched while the first turn's events are still + in flight; the first turn's terminal frame then arrives with no pending + client frames left and ends the session before the second turn's events + come back. The rejection tests never notice — the guard fails the second + turn synchronously inside the message loop — but the forwarding + regressions below need the second turn's scripted upstream events to reach + the client, so this downstream serializes turns the way a real Codex + client does: it waits for a terminal frame before sending the next + ``response.create``. + """ + + def __init__(self, request_texts: list[str]) -> None: + self.pending = list(request_texts) + self.done = asyncio.Event() + self.sent_text: list[str] = [] + self.turn_completed = asyncio.Event() + self._dispatched_any = False + + async def receive(self) -> dict[str, object]: + if self.pending: + if self._dispatched_any: + await self.turn_completed.wait() + self.turn_completed.clear() + self._dispatched_any = True + return {"type": "websocket.receive", "text": self.pending.pop(0)} + await self.done.wait() + return {"type": "websocket.disconnect"} + + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + payload = json.loads(text) + if payload.get("type") in {"response.completed", "response.failed", "error"}: + self.turn_completed.set() + if not self.pending: + self.done.set() + + async def send_bytes(self, _data: bytes) -> None: + return None + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + del code, reason + self.done.set() + + +@pytest.mark.asyncio +async def test_reuse_guard_forwards_a_pinned_input_file_turn(db_setup, monkeypatch: pytest.MonkeyPatch) -> None: + """A later source-owned turn that references an uploaded file must be forwarded. + + The HTTP route skips source selection whenever the input references an + ``input_file`` — the upload is account-scoped, so the request is pinned to + the subscription account that received it. The equivalent WebSocket turn + must reach that account through the owner-routing path instead of being + failed by the reuse guard (regression for the source-routing-exclusions + P2). + """ + settings = _make_proxy_settings() + settings.stream_idle_timeout_seconds = 300.0 + settings.proxy_downstream_websocket_idle_timeout_seconds = 120.0 + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_source_guard_file_pin") + upstream = _TurnDrivenUpstream([_completed_turn("resp_turn_one"), _completed_turn("resp_turn_two")]) + + async def fake_connect(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + return account, upstream + + # The second turn's model is source-owned, like the reuse-guard rejection test. + async def source_owned_for_qwen(model, _api_key, *, raw_model=None): # noqa: ANN001 + return model == "qwen3.8-max" + + monkeypatch.setattr(ws_mixin, "responses_model_is_source_owned", source_owned_for_qwen) + monkeypatch.setattr(proxy_service.ProxyService, "_connect_proxy_websocket", fake_connect) + monkeypatch.setattr(service, "_resolve_compact_turn_state_owner", AsyncMock(return_value=None)) + + await service._pin_file_account("file_ws_guard_pin", account.id) + downstream = _TurnSerializedDownstream( + [_create_frame("gpt-5.6-sol"), _input_file_frame("qwen3.8-max", "file_ws_guard_pin")] + ) + + await service.proxy_responses_websocket( + cast(WebSocket, downstream), + {}, + codex_session_affinity=False, + openai_cache_affinity=False, + api_key=None, + ) + + assert not any("model_source_requires_http_transport" in text for text in downstream.sent_text), ( + "HTTP excludes file-referencing requests from source routing, so the " + "reuse guard must not fail the file-pinned turn" + ) + assert len(upstream.sent_text) == 2, "the file-pinned turn must be forwarded to the pinned subscription account" + assert any("resp_turn_two" in text for text in downstream.sent_text), "the file-pinned turn must complete" + + +@pytest.mark.asyncio +async def test_reuse_guard_forwards_a_terminal_compaction_trigger_turn( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A terminal compaction-trigger turn must stay on the subscription upstream. + + The HTTP route serves terminal compaction triggers through the upstream + compact flow on the turn's owner account and never source-routes them, so + the reuse guard must not fail the equivalent WebSocket turn even when its + model is also exposed by a source (regression for the + source-routing-exclusions P2). + """ + settings = _make_proxy_settings() + settings.stream_idle_timeout_seconds = 300.0 + settings.proxy_downstream_websocket_idle_timeout_seconds = 120.0 + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_source_guard_compact") + upstream = _TurnDrivenUpstream([_completed_turn("resp_turn_one"), _completed_turn("resp_turn_two")]) + + async def fake_connect(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + return account, upstream + + async def source_owned_for_qwen(model, _api_key, *, raw_model=None): # noqa: ANN001 + return model == "qwen3.8-max" + + monkeypatch.setattr(ws_mixin, "responses_model_is_source_owned", source_owned_for_qwen) + monkeypatch.setattr(proxy_service.ProxyService, "_connect_proxy_websocket", fake_connect) + monkeypatch.setattr(service, "_resolve_compact_turn_state_owner", AsyncMock(return_value=None)) + + downstream = _TurnSerializedDownstream([_create_frame("gpt-5.6-sol"), _compaction_trigger_frame("qwen3.8-max")]) + + await service.proxy_responses_websocket( + cast(WebSocket, downstream), + {}, + codex_session_affinity=False, + openai_cache_affinity=False, + api_key=None, + ) + + assert not any("model_source_requires_http_transport" in text for text in downstream.sent_text), ( + "HTTP excludes terminal compaction triggers from source routing, so " + "the reuse guard must not fail the compaction turn" + ) + assert len(upstream.sent_text) == 2, "the compaction turn must be forwarded to the subscription upstream" + assert any("resp_turn_two" in text for text in downstream.sent_text), "the compaction turn must complete" + + +async def _drive_prepared_request_into_connect_guard( + monkeypatch: pytest.MonkeyPatch, + frame: str, +): + """Prepare ``frame`` for real and drive it into the real connect guard. + + Mirrors ``test_connect_guard_sees_the_raw_model_alias``: the alias catalog + makes ``gpt-5-high`` genuinely source-owned, so before the exclusions fix + the guard demonstrably fires for these frames — the stubbed account + selector standing in for the failover loop proves the guard was skipped. + """ + settings = _make_proxy_settings() + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + catalog = _AliasSourceCatalog({"gpt-5-high"}) + catalog.install(monkeypatch) + + api_key = _alias_allowlist_api_key() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + + emitted: dict[str, object] = {} + selection_calls = 0 + + async def fake_refresh(key): # noqa: ANN001, ANN202 + return api_key + + async def fake_emit(self, websocket, **kwargs): # noqa: ANN001, ANN003, ANN202 + emitted.update(kwargs) + + async def fake_select(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + nonlocal selection_calls + selection_calls += 1 + return None + + monkeypatch.setattr(service, "_refresh_websocket_api_key_policy", fake_refresh) + monkeypatch.setattr(service, "_reserve_websocket_api_key_usage", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_service.ProxyService, "_emit_websocket_connect_failure", fake_emit) + monkeypatch.setattr(proxy_service.ProxyService, "_select_websocket_connect_account", fake_select) + + prepared = await service._prepare_websocket_response_create_request( + json.loads(frame), + headers={}, + codex_session_affinity=False, + openai_cache_affinity=False, + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=0, + api_key=api_key, + ) + + account, upstream = await service._connect_proxy_websocket( + {}, + sticky_key=None, + sticky_kind=None, + prefer_earlier_reset=False, + routing_strategy="capacity_weighted", + model=prepared.request_state.model, + request_state=prepared.request_state, + api_key=api_key, + client_send_lock=anyio.Lock(), + websocket=AsyncMock(), + ) + return prepared, account, upstream, emitted, lambda: selection_calls + + +@pytest.mark.asyncio +async def test_connect_guard_skips_input_file_requests(db_setup, monkeypatch: pytest.MonkeyPatch) -> None: + """The connect guard must not bounce a file-referencing request to HTTP. + + An ``input_file`` reference excludes the request from source routing on + the HTTP path — pinned or not, the upload lives on a subscription + account — so the connect path must proceed to (owner-required) account + selection instead of emitting the 503 fallback. + """ + prepared, account, upstream, emitted, selection_calls = await _drive_prepared_request_into_connect_guard( + monkeypatch, + _input_file_frame("gpt-5-high", "file_ws_connect_unpinned"), + ) + + assert emitted == {}, "the connect guard must not emit the 503 HTTP-fallback failure" + assert selection_calls() >= 1, "file-referencing requests must proceed to account selection" + assert account is None # the stubbed selector returns no account + assert upstream is None + assert prepared.request_state.source_route_excluded is True, ( + "preparation must record the HTTP source-route exclusion on the request state" + ) + + +@pytest.mark.asyncio +async def test_connect_guard_skips_terminal_compaction_trigger_requests( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The connect guard must not bounce a terminal compaction trigger to HTTP. + + HTTP serves these through the compact flow on the owner account and never + source-routes them; the WebSocket connect path must likewise proceed to + account selection. + """ + prepared, account, upstream, emitted, selection_calls = await _drive_prepared_request_into_connect_guard( + monkeypatch, + _compaction_trigger_frame("gpt-5-high"), + ) + + assert emitted == {}, "the connect guard must not emit the 503 HTTP-fallback failure" + assert selection_calls() >= 1, "compaction-trigger requests must proceed to account selection" + assert account is None # the stubbed selector returns no account + assert upstream is None + assert prepared.request_state.source_route_excluded is True, ( + "preparation must record the HTTP source-route exclusion on the request state" + ) diff --git a/tests/unit/test_rate_limit_reset_credits_scheduler.py b/tests/unit/test_rate_limit_reset_credits_scheduler.py index 42a0433f6f..3acf640df7 100644 --- a/tests/unit/test_rate_limit_reset_credits_scheduler.py +++ b/tests/unit/test_rate_limit_reset_credits_scheduler.py @@ -1,9 +1,11 @@ from __future__ import annotations import asyncio +import logging import random from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta +from types import SimpleNamespace from typing import Any from unittest.mock import AsyncMock @@ -821,3 +823,105 @@ async def _refresh_once(self: RateLimitResetCreditsRefreshScheduler) -> None: await scheduler.stop() assert refreshed is False + + +def _patch_dashboard_settings(monkeypatch: pytest.MonkeyPatch, *, auto_redeem: bool) -> None: + class _FakeSession: + def expunge_all(self) -> None: + return None + + @asynccontextmanager + async def _fake_background_session(): + yield _FakeSession() + + monkeypatch.setattr(scheduler_module, "get_background_session", _fake_background_session) + monkeypatch.setattr( + scheduler_module, + "SettingsRepository", + lambda session: _FakeSettingsRepository(auto_redeem_reset_credits_before_expiry=auto_redeem), + ) + + +@pytest.mark.asyncio +async def test_scheduler_start_is_noop_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_dashboard_settings(monkeypatch, auto_redeem=False) + scheduler = RateLimitResetCreditsRefreshScheduler(interval_seconds=60, enabled=False) + + await scheduler.start() + + assert scheduler._task is None + + +@pytest.mark.asyncio +async def test_disabled_start_warns_on_persisted_auto_redeem_conflict( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _patch_dashboard_settings(monkeypatch, auto_redeem=True) + scheduler = RateLimitResetCreditsRefreshScheduler(interval_seconds=60, enabled=False) + + with caplog.at_level(logging.WARNING): + await scheduler.start() + + assert scheduler._task is None + conflict_warnings = [ + record + for record in caplog.records + if record.levelno >= logging.WARNING + and "auto_redeem_reset_credits_before_expiry" in record.getMessage() + and "rate_limit_reset_credits_refresh_enabled" in record.getMessage() + ] + assert conflict_warnings + + +@pytest.mark.asyncio +async def test_disabled_start_stays_silent_without_auto_redeem_opt_in( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _patch_dashboard_settings(monkeypatch, auto_redeem=False) + scheduler = RateLimitResetCreditsRefreshScheduler(interval_seconds=60, enabled=False) + + with caplog.at_level(logging.WARNING): + await scheduler.start() + + assert scheduler._task is None + assert not [ + record + for record in caplog.records + if record.levelno >= logging.WARNING and "auto_redeem_reset_credits_before_expiry" in record.getMessage() + ] + + +@pytest.mark.asyncio +async def test_scheduler_start_creates_task_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + started = asyncio.Event() + + async def _fake_run_loop(self: RateLimitResetCreditsRefreshScheduler) -> None: + started.set() + await self._stop.wait() + + monkeypatch.setattr(RateLimitResetCreditsRefreshScheduler, "_run_loop", _fake_run_loop) + scheduler = RateLimitResetCreditsRefreshScheduler(interval_seconds=60, enabled=True) + + await scheduler.start() + await asyncio.wait_for(started.wait(), timeout=1.0) + + assert scheduler._task is not None + assert not scheduler._task.done() + await scheduler.stop() + assert scheduler._task is None + + +def test_build_scheduler_wires_enabled_setting(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + scheduler_module, + "get_settings", + lambda: SimpleNamespace( + rate_limit_reset_credits_refresh_enabled=False, + rate_limit_reset_credits_refresh_interval_seconds=123, + ), + ) + + scheduler = scheduler_module.build_rate_limit_reset_credits_scheduler() + + assert scheduler.enabled is False + assert scheduler.interval_seconds == 123 diff --git a/tests/unit/test_replay_safety.py b/tests/unit/test_replay_safety.py index 76d03e0e3a..e17cf26c36 100644 --- a/tests/unit/test_replay_safety.py +++ b/tests/unit/test_replay_safety.py @@ -1,12 +1,5 @@ from __future__ import annotations -import copy -import hashlib -import json -from pathlib import Path -from typing import Any, cast -from uuid import UUID - import pytest from app.core.openai.requests import ResponsesRequest @@ -16,17 +9,11 @@ make_http_bridge_account_neutral_replay_key, ) from app.modules.proxy.replay_safety import ( - abandoned_pending_agent_boundary_rejection_reason, - account_neutral_codex_turn_metadata_identity, - project_responses_input_for_abandoned_pending_fresh_replay, project_responses_input_for_account_neutral_fresh_replay, responses_input_suffix_matches_pending_tool_calls, - responses_input_suffix_matches_transition_manifest, - responses_input_suffix_proves_abandoned_pending_agent_boundary, responses_input_suffix_retains_prior_output, responses_payload_is_account_neutral_fresh_replay, ) -from app.modules.proxy.response_transition_manifest import build_response_transition_manifest @pytest.mark.parametrize( @@ -164,493 +151,6 @@ def test_account_neutral_fresh_replay_accepts_self_contained_payloads( assert responses_payload_is_account_neutral_fresh_replay(payload) is True -def test_account_neutral_fresh_replay_accepts_identity_bound_responses_lite_0149_schema() -> None: - task_id = "01a0322c-0c11-7780-b68e-061ace9161a4" - payload: dict[str, JsonValue] = { - "reasoning": {"context": "all_turns", "effort": "high", "summary": "auto"}, - "input": [ - { - "type": "additional_tools", - "role": "developer", - "tools": [ - { - "type": "namespace", - "name": "functions", - "description": "", - "tools": [ - { - "type": "function", - "name": "lookup", - "description": "Lookup a fixture.", - "strict": False, - "defer_loading": True, - "parameters": {"type": "object", "properties": {}}, - }, - { - "type": "custom", - "name": "shell", - "description": "Run a shell command.", - "format": { - "type": "grammar", - "syntax": "lark", - "definition": "start: /.+/", - }, - }, - ], - }, - { - "type": "tool_search", - "execution": "client", - "description": "Search deferred tools.", - "parameters": { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "Search query for deferred tools.", - }, - "limit": { - "type": "number", - "description": "Maximum number of tools to return. Defaults to 8.", - }, - }, - "required": ["query"], - "additionalProperties": False, - }, - }, - ], - }, - {"role": "developer", "content": "Use the declared tools."}, - {"role": "user", "content": "Continue."}, - ], - "client_metadata": { - "session_id": task_id, - "thread_id": task_id, - "turn_id": "019f-turn-id", - "root_turn_id": "019f-turn-id", - "x-codex-installation-id": "installation-a", - "x-codex-window-id": "window-a", - "x-codex-turn-metadata": json.dumps( - { - "installation_id": "installation-a", - "session_id": task_id, - "thread_id": task_id, - "turn_id": "019f-turn-id", - "root_turn_id": "019f-turn-id", - "window_id": "window-a", - "workspace_kind": "projectless", - "request_kind": "turn", - } - ), - }, - } - - assert responses_payload_is_account_neutral_fresh_replay( - payload, - expected_session_identity=task_id, - expected_task_identity=task_id, - ) - assert not responses_payload_is_account_neutral_fresh_replay(payload) - assert not responses_payload_is_account_neutral_fresh_replay( - payload, - expected_session_identity=task_id, - expected_task_identity="different-task", - ) - - -@pytest.mark.parametrize("context", [None, "", "last_turn", "ALL_TURNS", 7, {"mode": "all_turns"}]) -def test_account_neutral_responses_lite_rejects_noncanonical_reasoning_context( - context: JsonValue, -) -> None: - assert not responses_payload_is_account_neutral_fresh_replay( - { - "input": [{"role": "user", "content": "Continue."}], - "reasoning": {"context": context, "effort": "high", "summary": "auto"}, - } - ) - - -@pytest.mark.parametrize( - "mutation", - [ - {"type": "namespace", "name": "functions", "description": "", "tools": []}, - { - "type": "namespace", - "name": "functions", - "description": "", - "tools": [{"type": "function", "name": "lookup", "container_id": "ctr-owner-a"}], - }, - { - "type": "namespace", - "name": "functions", - "description": "", - "tools": [{"type": "function", "name": "lookup"}], - }, - { - "type": "namespace", - "name": "functions", - "description": "", - "tools": [{"type": "custom", "name": "shell"}], - }, - { - "type": "namespace", - "name": "functions", - "description": "", - "tools": [ - { - "type": "custom", - "name": "shell", - "description": "Run a shell command.", - "format": {"type": "text"}, - } - ], - }, - { - "type": "namespace", - "name": "functions", - "description": "", - "tools": [ - { - "type": "function", - "name": "lookup", - "description": "Lookup a fixture.", - "strict": False, - "defer_loading": "yes", - "parameters": {}, - } - ], - }, - { - "type": "namespace", - "name": "functions", - "description": "", - "tools": [ - { - "type": "function", - "name": "lookup", - "description": "Lookup a fixture.", - "strict": False, - "defer_loading": False, - "parameters": {}, - } - ], - }, - { - "type": "tool_search", - "execution": "server", - "description": "Search deferred tools.", - "parameters": {}, - }, - { - "type": "tool_search", - "execution": "client", - "description": "Search deferred tools.", - "parameters": {"container_id": "ctr-owner-a"}, - }, - ], -) -def test_account_neutral_responses_lite_tools_reject_drift_and_owner_state( - mutation: dict[str, JsonValue], -) -> None: - assert not responses_payload_is_account_neutral_fresh_replay( - { - "input": [ - { - "type": "additional_tools", - "role": "developer", - "tools": [mutation], - }, - {"role": "developer", "content": "Use the declared tools."}, - {"role": "user", "content": "Continue."}, - ] - } - ) - - -@pytest.mark.parametrize( - "client_metadata", - [ - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "x-codex-parent-thread-id": "parent-task", - "x-codex-turn-metadata": json.dumps( - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "request_kind": "turn", - } - ), - }, - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "x-codex-turn-metadata": json.dumps( - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "request_kind": "turn", - "tool_namespaces_info": { - "functions": { - "name": "functions", - "functions": {}, - } - }, - } - ), - }, - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "root_turn_id": "turn-1", - "x-codex-turn-metadata": json.dumps( - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "request_kind": "turn", - } - ), - }, - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "x-codex-installation-id": "installation-flat", - "x-codex-turn-metadata": json.dumps( - { - "installation_id": "installation-nested", - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "request_kind": "turn", - } - ), - }, - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "x-codex-window-id": "window-flat", - "x-codex-turn-metadata": json.dumps( - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "window_id": "window-nested", - "request_kind": "turn", - } - ), - }, - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "x-openai-subagent": "review", - "x-codex-turn-metadata": json.dumps( - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "request_kind": "turn", - } - ), - }, - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "x-codex-turn-metadata": json.dumps({"request_kind": "turn"}), - }, - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "x-codex-turn-metadata": json.dumps( - { - "session_id": "root-task", - "thread_id": "root-task", - "turn_id": "turn-1", - "request_kind": "turn", - "container_id": "ctr-owner-a", - } - ), - }, - ], -) -def test_account_neutral_responses_lite_metadata_rejects_lineage_missing_identity_and_owner_state( - client_metadata: dict[str, JsonValue], -) -> None: - assert not responses_payload_is_account_neutral_fresh_replay( - { - "input": [{"role": "user", "content": "Continue."}], - "client_metadata": client_metadata, - }, - expected_session_identity="root-task", - expected_task_identity="root-task", - ) - - -@pytest.mark.parametrize( - "workspace_kind", - [None, "", " " * 4, "x" * 129, "界" * 43, 7, {"kind": "projectless"}], -) -def test_account_neutral_responses_lite_metadata_rejects_invalid_workspace_kind( - workspace_kind: JsonValue, -) -> None: - task_id = "root-task" - assert not responses_payload_is_account_neutral_fresh_replay( - { - "input": [{"role": "user", "content": "Continue."}], - "client_metadata": { - "session_id": task_id, - "thread_id": task_id, - "turn_id": "turn-1", - "x-codex-turn-metadata": json.dumps( - { - "session_id": task_id, - "thread_id": task_id, - "turn_id": "turn-1", - "request_kind": "turn", - "workspace_kind": workspace_kind, - } - ), - }, - }, - expected_session_identity=task_id, - expected_task_identity=task_id, - ) - - -def test_account_neutral_responses_lite_metadata_accepts_workspace_kind_at_utf8_byte_limit() -> None: - task_id = "root-task" - assert responses_payload_is_account_neutral_fresh_replay( - { - "input": [{"role": "user", "content": "Continue."}], - "client_metadata": { - "session_id": task_id, - "thread_id": task_id, - "turn_id": "turn-1", - "x-codex-turn-metadata": json.dumps( - { - "session_id": task_id, - "thread_id": task_id, - "turn_id": "turn-1", - "request_kind": "turn", - "workspace_kind": "界" * 42 + "ab", - } - ), - }, - }, - expected_session_identity=task_id, - expected_task_identity=task_id, - ) - - -def test_account_neutral_responses_lite_accepts_large_body_only_tool_inventory() -> None: - task_id = "root-task" - turn_metadata: dict[str, JsonValue] = { - "installation_id": "installation-a", - "session_id": task_id, - "thread_id": task_id, - "turn_id": "turn-1", - "window_id": "window-a", - "request_kind": "turn", - "tool_namespaces_info": { - "functions": { - "name": "functions", - "functions": { - f"tool_{index:03d}": { - "name": f"tool_{index:03d}", - "direct": False, - "code_mode_name": None, - "deferred": True, - "source": {"kind": "harness"}, - } - for index in range(160) - }, - } - }, - } - serialized = json.dumps(turn_metadata, separators=(",", ":")) - assert len(serialized.encode()) > 16 * 1024 - assert responses_payload_is_account_neutral_fresh_replay( - { - "input": [{"role": "user", "content": "Continue."}], - "client_metadata": { - "session_id": task_id, - "thread_id": task_id, - "turn_id": "turn-1", - "x-codex-turn-metadata": serialized, - }, - }, - expected_session_identity=task_id, - expected_task_identity=task_id, - ) - assert ( - account_neutral_codex_turn_metadata_identity( - serialized, - carrier="body", - expected_session_identity=task_id, - expected_task_identity=task_id, - expected_turn_identity="turn-1", - ) - is not None - ) - assert ( - account_neutral_codex_turn_metadata_identity( - serialized, - carrier="direct", - expected_session_identity=task_id, - expected_task_identity=task_id, - expected_turn_identity="turn-1", - ) - is None - ) - small_inventory = dict(turn_metadata) - small_inventory["tool_namespaces_info"] = { - "functions": { - "name": "functions", - "functions": { - "tool_000": { - "name": "tool_000", - "direct": False, - "code_mode_name": None, - "deferred": True, - "source": {"kind": "harness"}, - } - }, - } - } - small_serialized = json.dumps(small_inventory, separators=(",", ":")) - assert len(small_serialized.encode()) < 16 * 1024 - assert ( - account_neutral_codex_turn_metadata_identity( - small_serialized, - carrier="body", - expected_session_identity=task_id, - expected_task_identity=task_id, - expected_turn_identity="turn-1", - ) - is not None - ) - assert ( - account_neutral_codex_turn_metadata_identity( - small_serialized, - carrier="direct", - expected_session_identity=task_id, - expected_task_identity=task_id, - expected_turn_identity="turn-1", - ) - is None - ) - - def test_account_neutral_replay_projection_removes_response_owned_bookkeeping() -> None: metadata = {"turn_id": "turn_owner_a"} input_items: list[JsonValue] = [ @@ -2235,32 +1735,32 @@ def test_full_resend_retained_output_rejects_unverified_stored_developer_without ) -def test_full_resend_retained_output_accepts_exact_settled_same_session_prefix() -> None: - stored_input: list[JsonValue] = [ - {"role": "user", "content": "first question"}, - { - "type": "custom_tool_call", - "call_id": "call_historical", - "name": "shell", - "input": "pwd", - }, - { - "type": "custom_tool_call_output", - "call_id": "call_historical", - "output": "/workspace", - }, - {"type": "message", "role": "developer", "content": "later stored control"}, - ] +def test_full_resend_retained_output_rejects_response_owned_fresh_developer() -> None: + stored_input: list[JsonValue] = [{"role": "user", "content": "first question"}] suffix: list[JsonValue] = [ { "type": "message", + "id": "msg_answer", "role": "assistant", "phase": "final_answer", "status": "completed", "content": [{"type": "output_text", "text": "prior answer"}], }, - {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "next question"}]}, + { + "type": "message", + "id": "msg_user", + "role": "user", + "content": [{"type": "input_text", "text": "next question"}], + }, + { + "type": "message", + "id": "msg_response_owned", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_current"}, + "content": [{"type": "input_text", "text": "new control message"}], + }, ] + projection = project_responses_input_for_account_neutral_fresh_replay( [*stored_input, *suffix], stored_count=len(stored_input), @@ -2272,1439 +1772,10 @@ def test_full_resend_retained_output_accepts_exact_settled_same_session_prefix() projection.input_items, stored_count=projection.stored_prefix_count, ) - assert responses_input_suffix_retains_prior_output( - projection.input_items, - stored_count=projection.stored_prefix_count, - exact_stored_prefix_without_pending_manifest=True, - ) - - -def _canonical_agent_message() -> dict[str, JsonValue]: - return { - "type": "agent_message", - "id": "amsg_01a02b33-3b30-7742-bdb3-091f07cf2ea0", - "author": "/root/episode_identity_final_audit", - "recipient": "/root", - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - "create_time": 1787431172.912141, - }, - "content": [{"type": "input_text", "text": "verified inter-agent result"}], - } - - -def _canonical_response_owned_user_message( - *, - message_id: str = "msg_01a02c43-4980-7afb-97f5-2e2d30aa73de", - turn_id: str = "01a02c43-4980-7afb-97f5-2e2d30aa73de", - text: str = "retry", -) -> dict[str, JsonValue]: - return { - "type": "message", - "id": message_id, - "role": "user", - "content": [{"type": "input_text", "text": text}], - "internal_chat_message_metadata_passthrough": { - "turn_id": turn_id, - "create_time": 1787433402.605, - }, - } - - -def _canonical_response_owned_developer_message() -> dict[str, JsonValue]: - return { - "type": "message", - "id": "msg_01a02a78-d2b5-71e3-a33e-fab25a40b322", - "role": "developer", - "content": [ - {"type": "input_text", "text": "stored permissions"}, - {"type": "input_text", "text": "stored app context"}, - {"type": "input_text", "text": "stored collaboration mode"}, - {"type": "input_text", "text": "stored skills"}, - ], - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02a78-d2b4-7a73-b03f-6d85db9cf496", - }, - } - - -def _rehydrate_sanitized_pending_settlement_shapes() -> list[tuple[list[JsonValue], int, dict[str, str]]]: - fixture_path = ( - Path(__file__).parents[1] / "fixtures" / "http_responses" / "pending_settlement_real_transport_shapes_v1.json" - ) - fixture_bytes = fixture_path.read_bytes() - assert ( - hashlib.sha256(fixture_bytes).hexdigest() == "c9a5eedac589f973c30ed100cb429d60d8e2b9db439dd75f747b4081c09f2630" - ) - fixture = json.loads(fixture_bytes) - assert fixture["schema"] == "qk_http_responses_pending_settlement_shape_fixture_v1" - assert fixture["provenance"] == { - "contains_call_ids": False, - "contains_credentials": False, - "contains_message_text": False, - "contains_raw_ids": False, - "sanitized": True, - "source_kind": "production_request_structure", - } - hydrated: list[tuple[list[JsonValue], int, dict[str, str]]] = [] - for case_index, case in enumerate(fixture["cases"]): - stored_count = case["stored_count"] - input_items: list[JsonValue] = [ - {"role": "user", "content": f"sanitized-stored-{index}"} for index in range(stored_count) - ] - call_id = f"fixture-pending-{case_index}" - for shape in case["suffix_shape"]: - if shape == "reasoning": - input_items.append( - { - "type": "reasoning", - "id": f"rs_fixture_{case_index}", - "content": None, - "encrypted_content": "fixture-ciphertext", - "summary": [], - } - ) - elif shape == "custom_tool_call": - input_items.append( - { - "type": "custom_tool_call", - "id": f"ctc_fixture_{case_index}", - "call_id": call_id, - "name": "fixture_tool", - "input": "fixture-input", - "status": "completed", - } - ) - elif shape == "custom_tool_call_output": - input_items.append( - { - "type": "custom_tool_call_output", - "id": f"ctco_fixture_{case_index}", - "call_id": call_id, - "output": "fixture-output", - } - ) - elif shape == "function_call": - input_items.append( - { - "type": "function_call", - "id": f"fc_fixture_{case_index}", - "call_id": call_id, - "name": "fixture_tool", - "arguments": "{}", - } - ) - elif shape == "function_call_output": - input_items.append( - { - "type": "function_call_output", - "id": f"fco_fixture_{case_index}", - "call_id": call_id, - "output": "fixture-output", - } - ) - elif shape == "agent_message": - input_items.append(_canonical_agent_message()) - elif shape == "user_message": - input_items.append( - _canonical_response_owned_user_message( - message_id=f"msg_{UUID(int=case_index + 100)}", - turn_id=f"{UUID(int=case_index + 100)}", - text="sanitized-followup", - ) - ) - else: # pragma: no cover - fixture schema is closed above - raise AssertionError(f"unknown pending-settlement fixture shape: {shape}") - hydrated.append((input_items, stored_count, {call_id: case["pending_type"]})) - return hydrated - - -def test_real_sanitized_pending_settlement_shapes_accept_exact_manifest_then_bounded_followups() -> None: - cases = _rehydrate_sanitized_pending_settlement_shapes() - - assert len(cases) == 3 - for input_items, stored_count, pending_tool_calls in cases: - projection = project_responses_input_for_account_neutral_fresh_replay( - input_items, - stored_count=stored_count, - preserve_developer_message_ids=True, - preserve_response_owned_agent_message_ids=True, - ) - assert projection is not None - assert responses_input_suffix_matches_pending_tool_calls( - projection.input_items, - stored_count=projection.stored_prefix_count, - pending_tool_calls=pending_tool_calls, - ) - - -def test_pending_settlement_followup_never_treats_unrelated_call_as_the_pending_call() -> None: - input_items, stored_count, pending_tool_calls = _rehydrate_sanitized_pending_settlement_shapes()[0] - pending_call_id = next(iter(pending_tool_calls)) - output_index = next( - index - for index, item in enumerate(input_items) - if isinstance(item, dict) - and item.get("call_id") == pending_call_id - and str(item.get("type", "")).endswith("_output") - ) - input_items[output_index] = { - "type": "custom_tool_call_output", - "call_id": "fixture-unrelated-call", - "output": "fixture-output", - } - projection = project_responses_input_for_account_neutral_fresh_replay( - input_items, - stored_count=stored_count, - preserve_developer_message_ids=True, - preserve_response_owned_agent_message_ids=True, - ) - - assert projection is not None - assert not responses_input_suffix_matches_pending_tool_calls( - projection.input_items, - stored_count=projection.stored_prefix_count, - pending_tool_calls=pending_tool_calls, - ) - - -def test_pending_settlement_followup_rejects_a_second_tool_loop_after_exact_settlement() -> None: - input_items, stored_count, pending_tool_calls = _rehydrate_sanitized_pending_settlement_shapes()[0] - input_items.extend( - [ - { - "type": "custom_tool_call", - "call_id": "fixture-unrelated-call", - "name": "fixture_tool", - "input": "fixture-input", - }, - { - "type": "custom_tool_call_output", - "call_id": "fixture-unrelated-call", - "output": "fixture-output", - }, - ] - ) - projection = project_responses_input_for_account_neutral_fresh_replay( - input_items, - stored_count=stored_count, - preserve_developer_message_ids=True, - preserve_response_owned_agent_message_ids=True, - ) - - assert projection is not None - assert not responses_input_suffix_matches_pending_tool_calls( - projection.input_items, - stored_count=projection.stored_prefix_count, - pending_tool_calls=pending_tool_calls, - ) @pytest.mark.parametrize( - "mutation", - [ - "missing-output", - "duplicate-output", - "output-before-call", - "pending-type-drift", - "call-status-drift", - "malformed-agent-message", - "developer-before-user", - ], -) -def test_pending_settlement_followup_rejects_inexact_or_unbounded_shapes(mutation: str) -> None: - original, stored_count, original_manifest = _rehydrate_sanitized_pending_settlement_shapes()[0] - input_items = cast(list[JsonValue], json.loads(json.dumps(original))) - pending_tool_calls = dict(original_manifest) - call_id = next(iter(pending_tool_calls)) - call_index = next( - index - for index, item in enumerate(input_items) - if isinstance(item, dict) and item.get("call_id") == call_id and item.get("type") == "custom_tool_call" - ) - output_index = next( - index - for index, item in enumerate(input_items) - if isinstance(item, dict) and item.get("call_id") == call_id and item.get("type") == "custom_tool_call_output" - ) - if mutation == "missing-output": - input_items.pop(output_index) - elif mutation == "duplicate-output": - input_items.insert( - output_index + 1, cast(JsonValue, dict(cast(dict[str, JsonValue], input_items[output_index]))) - ) - elif mutation == "output-before-call": - input_items[call_index], input_items[output_index] = input_items[output_index], input_items[call_index] - elif mutation == "pending-type-drift": - pending_tool_calls[call_id] = "function_call" - elif mutation == "call-status-drift": - cast(dict[str, JsonValue], input_items[call_index])["status"] = "in_progress" - elif mutation == "malformed-agent-message": - input_items[-1] = _canonical_agent_message() - input_items[-1]["extra"] = "unbound" - input_items.append({"role": "user", "content": "sanitized-followup"}) - elif mutation == "developer-before-user": - input_items[-1] = {"role": "developer", "content": "sanitized-control"} - input_items.append({"role": "user", "content": "sanitized-followup"}) - else: # pragma: no cover - parametrization is closed above - raise AssertionError(mutation) - projection = project_responses_input_for_account_neutral_fresh_replay( - input_items, - stored_count=stored_count, - preserve_developer_message_ids=True, - preserve_response_owned_agent_message_ids=True, - ) - - assert projection is not None - assert not responses_input_suffix_matches_pending_tool_calls( - projection.input_items, - stored_count=projection.stored_prefix_count, - pending_tool_calls=pending_tool_calls, - ) - - -def _rehydrate_sanitized_abandoned_pending_fixture() -> tuple[list[JsonValue], int, dict[str, str]]: - fixture_path = ( - Path(__file__).parents[1] / "fixtures" / "http_responses" / "abandoned_pending_real_transport_shape_v1.json" - ) - fixture_bytes = fixture_path.read_bytes() - assert ( - hashlib.sha256(fixture_bytes).hexdigest() == "272d9d38876e409ca0ddf5e1c1990bd603347dffb3a33a3e43be5d82d77a9932" - ) - fixture = json.loads(fixture_bytes) - assert fixture["schema"] == "qk_http_responses_sanitized_shape_fixture_v1" - assert fixture["provenance"] == { - "contains_call_ids": False, - "contains_credentials": False, - "contains_message_text": False, - "contains_raw_ids": False, - "sanitized": True, - "source_kind": "production_request_structure", - } - request = fixture["request"] - assert request["top_keys"] == [ - "client_metadata", - "include", - "input", - "model", - "parallel_tool_calls", - "prompt_cache_key", - "reasoning", - "store", - "stream", - "text", - "tool_choice", - ] - catalog = request["item_shape_catalog"] - sequence = request["item_shape_sequence"] - assert len(sequence) == request["item_count"] == 143 - - items: list[JsonValue] = [] - preceding_call_id: str | None = None - for index, shape_id in enumerate(sequence): - shape = catalog[shape_id] - item_type = shape["type"] - role = shape["role"] - item: dict[str, Any] - if item_type == "additional_tools": - item = { - "type": "additional_tools", - "role": "developer", - "tools": [ - { - "type": "custom", - "name": "fixture_tool", - "description": "fixture-only tool declaration", - "format": {"type": "text"}, - } - ], - } - elif item_type == "reasoning": - item = { - "type": "reasoning", - "id": f"rs_fixture_{index}", - "content": None, - "encrypted_content": f"fixture-ciphertext-{index}", - "summary": [], - } - elif item_type == "agent_message": - item = { - "type": "agent_message", - "id": f"amsg_{UUID(int=index + 1)}", - "author": "/root/fixture_worker", - "recipient": "/root", - "content": [{"type": "input_text", "text": f"fixture-item-{index}-part-0"}], - } - elif item_type == "message": - content_types = shape["content_types"] - assert isinstance(content_types, list) - item = { - "type": "message", - "role": role, - "content": [ - {"type": content_type, "text": f"fixture-item-{index}-part-{part_index}"} - for part_index, content_type in enumerate(content_types) - ], - } - if "id" in shape["known_keys"]: - item["id"] = f"msg_{UUID(int=index + 1)}" - if shape["phase"] is not None: - item["phase"] = shape["phase"] - elif item_type in {"function_call", "custom_tool_call"}: - preceding_call_id = f"fixture-call-{index}" - item = { - "type": item_type, - "id": f"fixture-call-item-{index}", - "call_id": preceding_call_id, - "name": "fixture_tool", - } - if item_type == "function_call": - item["namespace"] = "fixture_namespace" - item["arguments"] = "{}" - else: - item["input"] = f"fixture-input-{index}" - if shape["status"] is not None: - item["status"] = shape["status"] - elif item_type in {"function_call_output", "custom_tool_call_output"}: - assert preceding_call_id is not None - item = { - "type": item_type, - "id": f"fixture-output-item-{index}", - "call_id": preceding_call_id, - "output": f"fixture-output-{index}", - } - preceding_call_id = None - else: # pragma: no cover - fixture schema is closed above - raise AssertionError(f"unknown sanitized shape: {item_type}") - assert set(item) == set(shape["known_keys"]) - items.append(cast(JsonValue, item)) - - assert preceding_call_id is None - pending_tool_calls = {"fixture-call-undelivered": fixture["pending_tool_calls"]["types"][0]} - return items, request["stored_count"], pending_tool_calls - - -def test_real_sanitized_transport_shape_recovers_once_without_duplicate_or_context_loss() -> None: - input_items, stored_count, pending_tool_calls = _rehydrate_sanitized_abandoned_pending_fixture() - - # This exact production shape was rejected by the legacy proof because - # the HTTP transport retained response-owned IDs but stripped their - # internal metadata, and because one developer refresh followed the user - # retry run. The fixture intentionally contains neither message bodies nor - # raw production identifiers. - response_owned_suffix = input_items[stored_count:] - assert all( - not isinstance(item, dict) or "internal_chat_message_metadata_passthrough" not in item - for item in response_owned_suffix - ) - assert any(isinstance(item, dict) and item.get("role") == "developer" for item in response_owned_suffix) - legacy_metadata_required_items = [input_items[index] for index in (111, 112, 113, 140)] - legacy_boundary_proof_would_accept = all( - isinstance(item, dict) and isinstance(item.get("internal_chat_message_metadata_passthrough"), dict) - for item in legacy_metadata_required_items - ) - assert legacy_boundary_proof_would_accept is False - - parsed_payload = ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "", "input": input_items}) - assert isinstance(parsed_payload.input, list) - parsed_input = parsed_payload.input - assert len(parsed_input) == 143 - assert isinstance(parsed_input[140], dict) and parsed_input[140].get("role") == "developer" - assert ( - abandoned_pending_agent_boundary_rejection_reason( - parsed_input, - stored_count=stored_count, - pending_tool_calls=pending_tool_calls, - ) - is None - ) - projection = project_responses_input_for_abandoned_pending_fresh_replay( - parsed_input, - stored_count=stored_count, - pending_tool_calls=pending_tool_calls, - ) - assert projection is not None - - def marker_sequence(items: list[JsonValue], *, roles: set[str]) -> list[str]: - markers: list[str] = [] - for item in items: - if not isinstance(item, dict) or item.get("type") not in (None, "message") or item.get("role") not in roles: - continue - content = item.get("content") - assert isinstance(content, list) - for part in content: - if not isinstance(part, dict): - continue - text = part.get("text") - if isinstance(text, str): - markers.append(text) - return markers - - expected_message_markers = marker_sequence(parsed_input, roles={"assistant", "developer", "user"}) - projected_message_markers = marker_sequence( - projection.input_items, - roles={"assistant", "developer", "user"}, - ) - assert projected_message_markers == expected_message_markers - - expected_call_ids = [ - item["call_id"] - for item in parsed_input[:stored_count] - if isinstance(item, dict) and item.get("type") in {"function_call", "custom_tool_call"} - ] - projected_call_ids = [ - item["call_id"] - for item in projection.input_items - if isinstance(item, dict) and item.get("type") in {"function_call", "custom_tool_call"} - ] - projected_output_ids = [ - item["call_id"] - for item in projection.input_items - if isinstance(item, dict) and item.get("type") in {"function_call_output", "custom_tool_call_output"} - ] - assert projected_call_ids == expected_call_ids - assert projected_output_ids == expected_call_ids - assert len(projected_call_ids) == len(set(projected_call_ids)) - assert pending_tool_calls.keys().isdisjoint(projected_call_ids) - assert all(not isinstance(item, dict) or item.get("type") != "reasoning" for item in projection.input_items) - - -def test_full_resend_exact_prefix_accepts_canonical_agent_message_before_user_retries() -> None: - stored_input: list[JsonValue] = [{"role": "user", "content": "first question"}] - full_input: list[JsonValue] = [ - *stored_input, - { - "type": "reasoning", - "id": "reasoning_previous", - "encrypted_content": "opaque", - "summary": [], - }, - _canonical_agent_message(), - {"type": "message", "role": "user", "content": "first retry"}, - {"type": "message", "role": "user", "content": "second retry"}, - ] - classification = project_responses_input_for_account_neutral_fresh_replay( - full_input, - stored_count=len(stored_input), - preserve_response_owned_agent_message_ids=True, - ) - serialized = project_responses_input_for_account_neutral_fresh_replay( - full_input, - stored_count=len(stored_input), - ) - - assert classification is not None - assert responses_input_suffix_retains_prior_output( - classification.input_items, - stored_count=classification.stored_prefix_count, - exact_stored_prefix_without_pending_manifest=True, - ) - assert serialized is not None - assert not responses_input_suffix_retains_prior_output( - serialized.input_items, - stored_count=serialized.stored_prefix_count, - exact_stored_prefix_without_pending_manifest=True, - ) - assert not responses_payload_is_account_neutral_fresh_replay({"input": serialized.input_items}) - - -@pytest.mark.parametrize( - "mutate", - [ - pytest.param(lambda item: item.pop("id"), id="missing-id"), - pytest.param(lambda item: item.__setitem__("id", "msg_not_agent_owned"), id="wrong-id-prefix"), - pytest.param(lambda item: item.__setitem__("id", "amsg_not-a-uuid"), id="invalid-id-uuid"), - pytest.param(lambda item: item.__setitem__("author", "root/no-leading-slash"), id="invalid-author"), - pytest.param(lambda item: item.__setitem__("author", "/foo"), id="non-root-author"), - pytest.param(lambda item: item.__setitem__("author", "/root/.."), id="path-traversal-author"), - pytest.param(lambda item: item.__setitem__("author", "/root/UPPER"), id="uppercase-author"), - pytest.param(lambda item: item.__setitem__("recipient", "/root/with-hyphen"), id="invalid-recipient-segment"), - pytest.param(lambda item: item.__setitem__("recipient", item["author"]), id="self-delivery"), - pytest.param( - lambda item: item["internal_chat_message_metadata_passthrough"].__setitem__("extra", True), - id="extra-metadata", - ), - pytest.param( - lambda item: item["internal_chat_message_metadata_passthrough"].__setitem__("turn_id", "bad-turn"), - id="invalid-turn-id", - ), - pytest.param( - lambda item: item["internal_chat_message_metadata_passthrough"].__setitem__("create_time", float("inf")), - id="nonfinite-create-time", - ), - pytest.param( - lambda item: item["internal_chat_message_metadata_passthrough"].__setitem__( - "create_time", - 10**400, - ), - id="oversized-integer-create-time", - ), - pytest.param( - lambda item: item.__setitem__("content", [{"type": "output_text", "text": "wrong direction"}]), - id="output-content", - ), - pytest.param(lambda item: item.__setitem__("extra", "unbound"), id="extra-field"), - ], -) -def test_full_resend_agent_message_proof_rejects_malformed_lookalikes(mutate) -> None: - stored_input: list[JsonValue] = [{"role": "user", "content": "first question"}] - agent_message = _canonical_agent_message() - mutate(agent_message) - projection = project_responses_input_for_account_neutral_fresh_replay( - [*stored_input, agent_message, {"role": "user", "content": "follow-up"}], - stored_count=len(stored_input), - preserve_response_owned_agent_message_ids=True, - ) - - assert projection is not None - assert not responses_input_suffix_retains_prior_output( - projection.input_items, - stored_count=projection.stored_prefix_count, - exact_stored_prefix_without_pending_manifest=True, - ) - - -def test_full_resend_agent_message_after_fresh_user_is_not_a_prior_output_boundary() -> None: - stored_input: list[JsonValue] = [{"role": "user", "content": "first question"}] - projection = project_responses_input_for_account_neutral_fresh_replay( - [ - *stored_input, - {"role": "user", "content": "follow-up"}, - _canonical_agent_message(), - ], - stored_count=len(stored_input), - preserve_response_owned_agent_message_ids=True, - ) - - assert projection is not None - assert not responses_input_suffix_retains_prior_output( - projection.input_items, - stored_count=projection.stored_prefix_count, - exact_stored_prefix_without_pending_manifest=True, - ) - - -def test_full_resend_agent_message_proves_client_abandoned_undelivered_pending_call() -> None: - stored_input: list[JsonValue] = [ - {"role": "user", "content": "first question"}, - _canonical_agent_message(), - ] - input_items: list[JsonValue] = [ - *stored_input, - { - "type": "reasoning", - "id": "rs_old", - "encrypted_content": "opaque", - "summary": [], - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - }, - }, - _canonical_agent_message(), - {"role": "user", "content": "first retry"}, - {"role": "user", "content": "second retry"}, - ] - - assert responses_input_suffix_proves_abandoned_pending_agent_boundary( - input_items, - stored_count=len(stored_input), - pending_tool_calls={"call_undelivered": "custom_tool_call"}, - ) - - -def test_full_resend_real_codex_user_bookkeeping_is_stripped_after_boundary_proof() -> None: - stored_input: list[JsonValue] = [ - _canonical_response_owned_user_message( - message_id="msg_01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - turn_id="01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - text="first question", - ) - ] - first_retry = _canonical_response_owned_user_message( - message_id="msg_01a02c31-2f60-7dd2-9f22-d7ef316596b1", - turn_id="01a02c31-2f60-7dd2-9f22-d7ef316596b1", - text="scheduled retry", - ) - second_retry = _canonical_response_owned_user_message(text="continue") - input_items: list[JsonValue] = [ - *stored_input, - { - "type": "reasoning", - "id": "rs_08639659bba14680016a8a0902e9a887d090946ff27b898f05", - "encrypted_content": "opaque", - "summary": [], - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - }, - }, - _canonical_agent_message(), - first_retry, - second_retry, - ] - - assert responses_input_suffix_proves_abandoned_pending_agent_boundary( - input_items, - stored_count=len(stored_input), - pending_tool_calls={"call_undelivered": "custom_tool_call"}, - ) - projection = project_responses_input_for_account_neutral_fresh_replay( - input_items, - stored_count=len(stored_input), - omit_response_owned_agent_messages_from_stored_prefix=True, - ) - assert projection is not None - for item in projection.input_items: - if not isinstance(item, dict) or item.get("role") != "user": - continue - assert "id" not in item - metadata = item.get("internal_chat_message_metadata_passthrough") - assert isinstance(metadata, dict) - assert set(metadata) == {"turn_id"} - - -def test_abandoned_pending_boundary_projects_response_owned_developer_in_exact_stored_prefix() -> None: - developer_message = _canonical_response_owned_developer_message() - stored_input: list[JsonValue] = [ - _canonical_response_owned_user_message(text="compacted user context"), - developer_message, - { - "type": "custom_tool_call", - "call_id": "call_settled", - "name": "shell", - "input": "pwd", - "status": "completed", - }, - { - "type": "custom_tool_call_output", - "call_id": "call_settled", - "output": "/workspace", - }, - ] - input_items: list[JsonValue] = [ - *stored_input, - { - "type": "reasoning", - "id": "rs_08639659bba14680016a8a0902e9a887d090946ff27b898f05", - "encrypted_content": "opaque", - "summary": [], - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - }, - }, - _canonical_agent_message(), - _canonical_response_owned_user_message(text="scheduled retry"), - _canonical_response_owned_user_message( - message_id="msg_01a02c59-7810-75fc-a7ee-ea7db5a66b6e", - turn_id="01a02c59-7810-75fc-a7ee-ea7db5a66b6e", - text="continue", - ), - ] - pending_tool_calls = {"call_undelivered": "custom_tool_call"} - - assert responses_input_suffix_proves_abandoned_pending_agent_boundary( - input_items, - stored_count=len(stored_input), - pending_tool_calls=pending_tool_calls, - ) - projection = project_responses_input_for_abandoned_pending_fresh_replay( - input_items, - stored_count=len(stored_input), - pending_tool_calls=pending_tool_calls, - ) - assert projection is not None - projected_developer = next( - item - for item in projection.input_items[: projection.stored_prefix_count] - if isinstance(item, dict) and item.get("role") == "developer" - ) - assert "id" not in projected_developer - assert projected_developer["content"] == developer_message["content"] - assert projected_developer["internal_chat_message_metadata_passthrough"] == { - "turn_id": "01a02a78-d2b4-7a73-b03f-6d85db9cf496" - } - - -def test_abandoned_pending_boundary_accepts_exact_http_transport_normalized_bookkeeping() -> None: - """Codex strips internal metadata, but not response IDs, before HTTP.""" - - stored_developer = _canonical_response_owned_developer_message() - stored_developer.pop("internal_chat_message_metadata_passthrough") - stored_user = _canonical_response_owned_user_message(text="stored user") - stored_user.pop("internal_chat_message_metadata_passthrough") - stored_input: list[JsonValue] = [ - { - "type": "additional_tools", - "role": "developer", - "tools": [ - { - "type": "custom", - "name": "shell", - "description": "execute a bounded shell command", - "format": {"type": "text"}, - } - ], - }, - { - "type": "message", - "role": "developer", - "content": [{"type": "input_text", "text": "request-scoped instructions"}], - }, - stored_user, - stored_developer, - { - "type": "custom_tool_call", - "id": "ctc_transport_settled", - "call_id": "call_transport_settled", - "name": "shell", - "input": "pwd", - "status": "completed", - }, - { - "type": "custom_tool_call_output", - "id": "ctco_transport_settled", - "call_id": "call_transport_settled", - "output": "/workspace", - }, - ] - reasoning = { - "type": "reasoning", - "id": "rs_08639659bba14680016a8a0902e9a887d090946ff27b898f05", - "content": None, - "encrypted_content": "opaque", - "summary": [], - } - agent_message = _canonical_agent_message() - agent_message.pop("internal_chat_message_metadata_passthrough") - first_user = _canonical_response_owned_user_message(text="first retry") - first_user.pop("internal_chat_message_metadata_passthrough") - developer_followup = _canonical_response_owned_developer_message() - developer_followup["id"] = "msg_01a02d3a-0319-76f1-9fd0-b28e9b9bc2d7" - developer_followup.pop("internal_chat_message_metadata_passthrough") - second_user = _canonical_response_owned_user_message( - message_id="msg_01a02d3a-0321-7631-8a0a-7b90517b4bb2", - turn_id="01a02d3a-0321-7631-8a0a-7b90517b4bb2", - text="second retry", - ) - second_user.pop("internal_chat_message_metadata_passthrough") - input_items: list[JsonValue] = [ - *stored_input, - reasoning, - agent_message, - first_user, - developer_followup, - second_user, - ] - pending_tool_calls = {"call_undelivered": "custom_tool_call"} - - assert responses_input_suffix_proves_abandoned_pending_agent_boundary( - input_items, - stored_count=len(stored_input), - pending_tool_calls=pending_tool_calls, - ) - parsed_reasoning = dict(reasoning) - parsed_reasoning.pop("content") - parsed_input_items = [*stored_input, parsed_reasoning, *input_items[len(stored_input) + 1 :]] - assert responses_input_suffix_proves_abandoned_pending_agent_boundary( - parsed_input_items, - stored_count=len(stored_input), - pending_tool_calls=pending_tool_calls, - ) - projection = project_responses_input_for_abandoned_pending_fresh_replay( - input_items, - stored_count=len(stored_input), - pending_tool_calls=pending_tool_calls, - ) - assert projection is not None - assert all( - not isinstance(item, dict) or item.get("role") not in {"developer", "user"} or "id" not in item - for item in projection.input_items - ) - assert all(not isinstance(item, dict) or item.get("type") != "reasoning" for item in projection.input_items) - assert projection.input_items[-2] == { - "type": "message", - "role": "developer", - "content": developer_followup["content"], - } - - -def test_abandoned_pending_boundary_reports_first_content_free_rejection_branch() -> None: - stored: list[JsonValue] = [{"role": "user", "content": "stored"}] - boundary = _canonical_agent_message() - followup: dict[str, JsonValue] = {"role": "user", "content": "follow-up"} - pending = {"call_undelivered": "custom_tool_call"} - - assert ( - abandoned_pending_agent_boundary_rejection_reason( - [*stored, boundary, followup], - stored_count=len(stored), - pending_tool_calls=pending, - ) - is None - ) - assert ( - abandoned_pending_agent_boundary_rejection_reason( - [*stored, boundary, followup], - stored_count=0, - pending_tool_calls=pending, - ) - == "stored_prefix_invalid" - ) - assert ( - abandoned_pending_agent_boundary_rejection_reason( - [*stored, boundary, followup], - stored_count=len(stored), - pending_tool_calls={}, - ) - == "pending_call_manifest_missing" - ) - malformed_reasoning: dict[str, JsonValue] = { - "type": "reasoning", - "id": "rs_fixture", - "encrypted_content": "opaque", - "summary": [], - "extra": True, - } - assert ( - abandoned_pending_agent_boundary_rejection_reason( - [*stored, malformed_reasoning, boundary, followup], - stored_count=len(stored), - pending_tool_calls=pending, - ) - == "boundary_reasoning_shape_invalid" - ) - assert ( - abandoned_pending_agent_boundary_rejection_reason( - [*stored, {"role": "assistant", "content": "lookalike"}, followup], - stored_count=len(stored), - pending_tool_calls=pending, - ) - == "boundary_agent_message_shape_invalid" - ) - assert ( - abandoned_pending_agent_boundary_rejection_reason( - [*stored, boundary], - stored_count=len(stored), - pending_tool_calls=pending, - ) - == "followup_missing" - ) - assert ( - abandoned_pending_agent_boundary_rejection_reason( - [*stored, boundary, 7], - stored_count=len(stored), - pending_tool_calls=pending, - ) - == "followup_shape_invalid" - ) - assert ( - abandoned_pending_agent_boundary_rejection_reason( - [*stored, boundary, followup, {"type": "message", "role": "developer", "content": 7}, followup], - stored_count=len(stored), - pending_tool_calls=pending, - ) - == "developer_message_shape_invalid" - ) - assert ( - abandoned_pending_agent_boundary_rejection_reason( - [*stored, boundary, followup, _canonical_response_owned_developer_message()], - stored_count=len(stored), - pending_tool_calls=pending, - ) - == "developer_message_sequence_invalid" - ) - assert ( - abandoned_pending_agent_boundary_rejection_reason( - [{"role": "user", "content": "stored", "call_id": "call_undelivered"}, boundary, followup], - stored_count=1, - pending_tool_calls=pending, - ) - == "pending_call_conflict" - ) - assert ( - abandoned_pending_agent_boundary_rejection_reason( - [ - { - "type": "custom_tool_call_output", - "call_id": "call_orphan", - "output": "orphaned", - }, - boundary, - followup, - ], - stored_count=1, - pending_tool_calls=pending, - ) - == "projection_failed" - ) - unmatched_call: dict[str, JsonValue] = { - "type": "custom_tool_call", - "call_id": "call_historical_unmatched", - "name": "shell", - "input": "pwd", - "status": "completed", - } - assert ( - abandoned_pending_agent_boundary_rejection_reason( - [unmatched_call, boundary, followup], - stored_count=1, - pending_tool_calls=pending, - ) - == "direct_call_prefix_state_invalid" - ) - - -@pytest.mark.parametrize( - "mutate", - [ - pytest.param(lambda items: items[0].__setitem__("content", "not-null"), id="reasoning-content"), - pytest.param(lambda items: items[1].__setitem__("extra", True), id="agent-extra-field"), - pytest.param(lambda items: items[2].__setitem__("id", "msg_not-a-uuid"), id="user-invalid-id"), - pytest.param(lambda items: items[3].__setitem__("extra", True), id="developer-extra-field"), - pytest.param( - lambda items: items[3].__setitem__("internal_chat_message_metadata_passthrough", {}), - id="developer-empty-metadata", - ), - ], -) -def test_abandoned_pending_boundary_rejects_malformed_transport_normalized_bookkeeping(mutate) -> None: - reasoning: dict[str, JsonValue] = { - "type": "reasoning", - "id": "rs_08639659bba14680016a8a0902e9a887d090946ff27b898f05", - "content": None, - "encrypted_content": "opaque", - "summary": [], - } - agent_message = _canonical_agent_message() - agent_message.pop("internal_chat_message_metadata_passthrough") - user_message = _canonical_response_owned_user_message() - user_message.pop("internal_chat_message_metadata_passthrough") - developer_message = _canonical_response_owned_developer_message() - developer_message.pop("internal_chat_message_metadata_passthrough") - suffix = [reasoning, agent_message, user_message, developer_message] - mutate(suffix) - - assert not responses_input_suffix_proves_abandoned_pending_agent_boundary( - [{"role": "user", "content": "stored"}, *suffix], - stored_count=1, - pending_tool_calls={"call_undelivered": "custom_tool_call"}, - ) - - -def test_abandoned_pending_boundary_requires_a_user_after_transport_developer_followup() -> None: - reasoning: dict[str, JsonValue] = { - "type": "reasoning", - "id": "rs_08639659bba14680016a8a0902e9a887d090946ff27b898f05", - "content": None, - "encrypted_content": "opaque", - "summary": [], - } - agent_message = _canonical_agent_message() - agent_message.pop("internal_chat_message_metadata_passthrough") - developer_message = _canonical_response_owned_developer_message() - developer_message.pop("internal_chat_message_metadata_passthrough") - - assert not responses_input_suffix_proves_abandoned_pending_agent_boundary( - [{"role": "user", "content": "stored"}, reasoning, agent_message, developer_message], - stored_count=1, - pending_tool_calls={"call_undelivered": "custom_tool_call"}, - ) - - -def test_abandoned_pending_boundary_rejects_unbounded_transport_developer_followups() -> None: - reasoning: dict[str, JsonValue] = { - "type": "reasoning", - "id": "rs_08639659bba14680016a8a0902e9a887d090946ff27b898f05", - "content": None, - "encrypted_content": "opaque", - "summary": [], - } - agent_message = _canonical_agent_message() - agent_message.pop("internal_chat_message_metadata_passthrough") - first_user = _canonical_response_owned_user_message(text="first retry") - first_user.pop("internal_chat_message_metadata_passthrough") - second_user = _canonical_response_owned_user_message( - message_id="msg_01a02d3a-0321-7631-8a0a-7b90517b4bb2", - turn_id="01a02d3a-0321-7631-8a0a-7b90517b4bb2", - text="second retry", - ) - second_user.pop("internal_chat_message_metadata_passthrough") - first_developer = _canonical_response_owned_developer_message() - first_developer.pop("internal_chat_message_metadata_passthrough") - second_developer = _canonical_response_owned_developer_message() - second_developer["id"] = "msg_01a02d3a-0319-76f1-9fd0-b28e9b9bc2d7" - second_developer.pop("internal_chat_message_metadata_passthrough") - - for followups in ( - [first_developer, first_user], - [first_user, first_developer], - [first_user, first_developer, second_developer, second_user], - ): - assert not responses_input_suffix_proves_abandoned_pending_agent_boundary( - [{"role": "user", "content": "stored"}, reasoning, agent_message, *followups], - stored_count=1, - pending_tool_calls={"call_undelivered": "custom_tool_call"}, - ) - - -@pytest.mark.parametrize( - "mutate", - [ - pytest.param(lambda item: item.__setitem__("id", "msg_not-a-uuid"), id="invalid-message-id"), - pytest.param( - lambda item: item["internal_chat_message_metadata_passthrough"].__setitem__("extra", True), - id="extra-metadata", - ), - pytest.param( - lambda item: item.__setitem__("content", [{"type": "output_text", "text": "wrong authority"}]), - id="output-content", - ), - pytest.param(lambda item: item.__setitem__("extra", "unbound"), id="extra-field"), - ], -) -def test_abandoned_pending_boundary_rejects_malformed_response_owned_developer_messages(mutate) -> None: - developer_message = _canonical_response_owned_developer_message() - mutate(developer_message) - stored_input: list[JsonValue] = [ - _canonical_response_owned_user_message(text="compacted user context"), - developer_message, - ] - - assert not responses_input_suffix_proves_abandoned_pending_agent_boundary( - [*stored_input, _canonical_agent_message(), _canonical_response_owned_user_message()], - stored_count=len(stored_input), - pending_tool_calls={"call_undelivered": "custom_tool_call"}, - ) - - -def test_abandoned_pending_boundary_drops_only_exact_leading_orphan_output() -> None: - orphan_call_id = "call_clipped_before_retained_window" - stored_input: list[JsonValue] = [ - { - "type": "custom_tool_call_output", - "call_id": orphan_call_id, - "output": "historical result", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-clipped"}, - }, - { - "type": "custom_tool_call", - "id": "ctc_response_owned_retained", - "call_id": "call_retained", - "name": "shell", - "input": "pwd", - "status": "completed", - "internal_chat_message_metadata_passthrough": { - "turn_id": "turn-retained", - "create_time": 1787433300.0, - }, - }, - { - "type": "custom_tool_call_output", - "id": "ctco_response_owned_retained", - "call_id": "call_retained", - "output": "done", - "internal_chat_message_metadata_passthrough": { - "turn_id": "turn-retained", - "create_time": 1787433301.0, - }, - }, - { - "type": "message", - "role": "assistant", - "phase": "final_answer", - "content": [{"type": "output_text", "text": "historical task completed"}], - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-retained"}, - }, - ] - input_items = [ - *stored_input, - _canonical_agent_message(), - _canonical_response_owned_user_message(text="continue"), - ] - pending_tool_calls = {"call_undelivered": "custom_tool_call"} - - assert responses_input_suffix_proves_abandoned_pending_agent_boundary( - input_items, - stored_count=len(stored_input), - pending_tool_calls=pending_tool_calls, - ) - projection = project_responses_input_for_abandoned_pending_fresh_replay( - input_items, - stored_count=len(stored_input), - pending_tool_calls=pending_tool_calls, - ) - assert projection is not None - assert projection.stored_prefix_count == len(stored_input) - 1 - projected_call = projection.input_items[0] - projected_output = projection.input_items[1] - assert isinstance(projected_call, dict) - assert isinstance(projected_output, dict) - assert projected_call["call_id"] == "call_retained" - assert projected_call["internal_chat_message_metadata_passthrough"] == {"turn_id": "turn-retained"} - assert projected_output["internal_chat_message_metadata_passthrough"] == {"turn_id": "turn-retained"} - assert all(not isinstance(item, dict) or item.get("call_id") != orphan_call_id for item in projection.input_items) - - -@pytest.mark.parametrize( - "mutate", - [ - pytest.param( - lambda items: items[0].__setitem__("call_id", "call_undelivered"), - id="orphan-is-abandoned-pending-call", - ), - pytest.param( - lambda items: items.append( - { - "type": "custom_tool_call", - "call_id": "call_clipped_before_retained_window", - "name": "shell", - "input": "pwd", - "status": "completed", - } - ), - id="orphan-call-id-reused-later", - ), - pytest.param( - lambda items: items.__setitem__( - 3, - {"type": "message", "role": "user", "content": "no retained assistant boundary"}, - ), - id="no-retained-assistant-boundary", - ), - pytest.param( - lambda items: items.insert( - 2, - { - "type": "custom_tool_call_output", - "call_id": "call_nonleading_orphan", - "output": "ambiguous", - }, - ), - id="nonleading-orphan-remains-invalid", - ), - ], -) -def test_abandoned_pending_boundary_rejects_ambiguous_orphan_output(mutate) -> None: - stored_input: list[JsonValue] = [ - { - "type": "custom_tool_call_output", - "call_id": "call_clipped_before_retained_window", - "output": "historical result", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-clipped"}, - }, - { - "type": "custom_tool_call", - "call_id": "call_retained", - "name": "shell", - "input": "pwd", - "status": "completed", - }, - {"type": "custom_tool_call_output", "call_id": "call_retained", "output": "done"}, - { - "type": "message", - "role": "assistant", - "phase": "final_answer", - "content": "historical task completed", - }, - ] - mutate(stored_input) - input_items = [*stored_input, _canonical_agent_message(), {"role": "user", "content": "continue"}] - - assert not responses_input_suffix_proves_abandoned_pending_agent_boundary( - input_items, - stored_count=len(stored_input), - pending_tool_calls={"call_undelivered": "custom_tool_call"}, - ) - - -@pytest.mark.parametrize( - "mutate", - [ - pytest.param(lambda item: item.__setitem__("id", "msg_not-a-uuid"), id="invalid-message-id"), - pytest.param( - lambda item: item["internal_chat_message_metadata_passthrough"].__setitem__("extra", True), - id="extra-metadata", - ), - pytest.param( - lambda item: item["internal_chat_message_metadata_passthrough"].__setitem__("create_time", -1), - id="negative-create-time", - ), - pytest.param(lambda item: item.__setitem__("extra", "unbound"), id="extra-field"), - ], -) -def test_abandoned_pending_boundary_rejects_malformed_response_owned_user_messages(mutate) -> None: - stored_input: list[JsonValue] = [{"role": "user", "content": "first question"}] - user_message = _canonical_response_owned_user_message() - mutate(user_message) - - assert not responses_input_suffix_proves_abandoned_pending_agent_boundary( - [*stored_input, _canonical_agent_message(), user_message], - stored_count=len(stored_input), - pending_tool_calls={"call_undelivered": "custom_tool_call"}, - ) - - -@pytest.mark.parametrize( - "suffix", - [ - pytest.param([_canonical_agent_message()], id="no-fresh-user"), - pytest.param([{"role": "user", "content": "retry"}], id="no-agent-boundary"), - pytest.param( - [ - _canonical_agent_message(), - { - "type": "custom_tool_call_output", - "call_id": "call_undelivered", - "output": "forged", - }, - ], - id="pending-output-present", - ), - pytest.param( - [ - _canonical_agent_message(), - {"role": "user", "content": "retry"}, - { - "type": "custom_tool_call", - "call_id": "call_other", - "name": "shell", - "input": "pwd", - }, - ], - id="new-call-after-user", - ), - ], -) -def test_abandoned_pending_agent_boundary_rejects_incomplete_or_tool_bearing_suffix( - suffix: list[JsonValue], -) -> None: - stored_input: list[JsonValue] = [{"role": "user", "content": "first question"}] - assert not responses_input_suffix_proves_abandoned_pending_agent_boundary( - [*stored_input, *suffix], - stored_count=len(stored_input), - pending_tool_calls={"call_undelivered": "custom_tool_call"}, - ) - - -@pytest.mark.parametrize( - "suffix", - [ - pytest.param( - [ - {"type": "reasoning", "id": "rs_missing_encrypted_content", "summary": []}, - _canonical_agent_message(), - {"role": "user", "content": "retry"}, - ], - id="reasoning-missing-encrypted-content", - ), - pytest.param( - [ - _canonical_agent_message(), - { - "type": "reasoning", - "id": "rs_after_boundary", - "encrypted_content": "opaque", - "summary": [], - "internal_chat_message_metadata_passthrough": { - "turn_id": "01a02b31-bc02-70b0-a09e-0dedbc2e2da9", - }, - }, - {"role": "user", "content": "retry"}, - ], - id="reasoning-after-agent-boundary", - ), - ], -) -def test_abandoned_pending_agent_boundary_rejects_unproven_or_misordered_reasoning( - suffix: list[JsonValue], -) -> None: - assert not responses_input_suffix_proves_abandoned_pending_agent_boundary( - [{"role": "user", "content": "first question"}, *suffix], - stored_count=1, - pending_tool_calls={"call_undelivered": "custom_tool_call"}, - ) - - -def test_full_resend_exact_settled_prefix_rejects_historical_call_id_reuse() -> None: - input_items: list[JsonValue] = [ - { - "type": "custom_tool_call", - "call_id": "call_historical", - "name": "shell", - "input": "pwd", - }, - { - "type": "custom_tool_call_output", - "call_id": "call_historical", - "output": "/workspace", - }, - { - "type": "custom_tool_call", - "call_id": "call_historical", - "name": "shell", - "input": "git status --short", - }, - { - "type": "custom_tool_call_output", - "call_id": "call_historical", - "output": "", - }, - { - "type": "message", - "role": "assistant", - "phase": "final_answer", - "content": [{"type": "output_text", "text": "prior answer"}], - }, - {"type": "message", "role": "user", "content": "next question"}, - ] - - assert not responses_input_suffix_retains_prior_output( - input_items, - stored_count=2, - exact_stored_prefix_without_pending_manifest=True, - ) - - -def test_full_resend_retained_output_rejects_response_owned_fresh_developer() -> None: - stored_input: list[JsonValue] = [{"role": "user", "content": "first question"}] - suffix: list[JsonValue] = [ - { - "type": "message", - "id": "msg_answer", - "role": "assistant", - "phase": "final_answer", - "status": "completed", - "content": [{"type": "output_text", "text": "prior answer"}], - }, - { - "type": "message", - "id": "msg_user", - "role": "user", - "content": [{"type": "input_text", "text": "next question"}], - }, - { - "type": "message", - "id": "msg_response_owned", - "role": "developer", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn_current"}, - "content": [{"type": "input_text", "text": "new control message"}], - }, - ] - - projection = project_responses_input_for_account_neutral_fresh_replay( - [*stored_input, *suffix], - stored_count=len(stored_input), - preserve_developer_message_ids=True, - ) - - assert projection is not None - assert not responses_input_suffix_retains_prior_output( - projection.input_items, - stored_count=projection.stored_prefix_count, - ) - - -@pytest.mark.parametrize( - "suffix", + "suffix", [ pytest.param( [ @@ -4542,174 +2613,3 @@ def test_account_neutral_replay_marker_requires_tagged_existing_hard_kind() -> N def test_account_neutral_replay_marker_rejects_empty_nonce() -> None: with pytest.raises(ValueError, match="nonce"): make_http_bridge_account_neutral_replay_key("") - - -def _manifest_bound_fourcam_retry() -> tuple[ - list[JsonValue], - int, - dict[str, str], - Any, -]: - stored_input: list[JsonValue] = [ - {"type": "message", "role": "user", "content": "sealed stored input"}, - ] - response_output: list[JsonValue] = [ - { - "type": "reasoning", - "id": "rs_manifest_fourcam", - "encrypted_content": "opaque", - "summary": [], - "status": "completed", - }, - { - "type": "message", - "id": "msg_manifest_fourcam", - "role": "assistant", - "phase": "commentary", - "content": [{"type": "output_text", "text": "running checks"}], - }, - { - "type": "custom_tool_call", - "id": "ctc_manifest_fourcam", - "call_id": "call_manifest_fourcam", - "name": "shell", - "input": "rustfmt --check", - "status": "completed", - }, - ] - pending = {"call_manifest_fourcam": "custom_tool_call"} - manifest = build_response_transition_manifest( - { - "response": { - "id": "resp_manifest_fourcam", - "status": "completed", - "output": response_output, - } - }, - pending_tool_calls=pending, - ) - assert manifest is not None - input_items: list[JsonValue] = [ - *stored_input, - *response_output, - { - "type": "custom_tool_call_output", - "id": "ctco_manifest_fourcam", - "call_id": "call_manifest_fourcam", - "output": "verified", - "status": "completed", - "internal_chat_message_metadata_passthrough": { - "turn_id": "00000000-0000-4000-8000-000000000301", - "create_time": 1.0, - }, - }, - { - "type": "message", - "id": "msg_00000000-0000-4000-8000-000000000302", - "role": "developer", - "content": [{"type": "input_text", "text": "first retry context"}], - "internal_chat_message_metadata_passthrough": {"turn_id": "00000000-0000-4000-8000-000000000303"}, - }, - { - "type": "message", - "id": "msg_00000000-0000-4000-8000-000000000304", - "role": "user", - "content": [{"type": "input_text", "text": "first retry"}], - "internal_chat_message_metadata_passthrough": { - "turn_id": "00000000-0000-4000-8000-000000000303", - "create_time": 2.0, - }, - }, - { - "type": "message", - "id": "msg_00000000-0000-4000-8000-000000000305", - "role": "developer", - "content": [{"type": "input_text", "text": "second retry context"}], - "internal_chat_message_metadata_passthrough": { - "turn_id": "00000000-0000-4000-8000-000000000306", - "create_time": 3.0, - }, - }, - { - "type": "message", - "id": "msg_00000000-0000-4000-8000-000000000307", - "role": "user", - "content": [{"type": "input_text", "text": "second retry"}], - "internal_chat_message_metadata_passthrough": { - "turn_id": "00000000-0000-4000-8000-000000000306", - "create_time": 4.0, - }, - }, - ] - return input_items, len(stored_input), pending, manifest - - -def test_transition_manifest_accepts_completed_output_settlement_and_grouped_retries() -> None: - input_items, stored_count, pending, manifest = _manifest_bound_fourcam_retry() - - assert responses_input_suffix_matches_transition_manifest( - input_items, - stored_count=stored_count, - response_id="resp_manifest_fourcam", - pending_tool_calls=pending, - transition_manifest=manifest, - ) - - -def test_transition_manifest_accepts_one_plain_responses_user_followup() -> None: - input_items, stored_count, pending, manifest = _manifest_bound_fourcam_retry() - plain_followup = [*input_items[:5], {"role": "user", "content": "retry"}] - - assert responses_input_suffix_matches_transition_manifest( - plain_followup, - stored_count=stored_count, - response_id="resp_manifest_fourcam", - pending_tool_calls=pending, - transition_manifest=manifest, - ) - - -@pytest.mark.parametrize( - "mutation", - [ - "changed-manifest-item", - "reordered-manifest-items", - "missing-output", - "wrong-output-call", - "duplicate-item-id", - "orphan-developer-turn", - "duplicate-user-in-turn", - "unknown-suffix-item", - ], -) -def test_transition_manifest_rejects_unproven_or_ambiguous_retry(mutation: str) -> None: - input_items, stored_count, pending, manifest = _manifest_bound_fourcam_retry() - if mutation == "changed-manifest-item": - cast(dict[str, JsonValue], input_items[2])["content"] = [{"type": "output_text", "text": "changed"}] - elif mutation == "reordered-manifest-items": - input_items[1], input_items[2] = input_items[2], input_items[1] - elif mutation == "missing-output": - input_items.pop(4) - elif mutation == "wrong-output-call": - cast(dict[str, JsonValue], input_items[4])["call_id"] = "call_other" - elif mutation == "duplicate-item-id": - cast(dict[str, JsonValue], input_items[4])["id"] = "ctc_manifest_fourcam" - elif mutation == "orphan-developer-turn": - cast(dict[str, JsonValue], input_items[6])["internal_chat_message_metadata_passthrough"] = { - "turn_id": "00000000-0000-4000-8000-000000000399", - "create_time": 2.0, - } - elif mutation == "duplicate-user-in-turn": - duplicate = copy.deepcopy(input_items[8]) - cast(dict[str, JsonValue], duplicate)["id"] = "msg_00000000-0000-4000-8000-000000000308" - input_items.append(duplicate) - elif mutation == "unknown-suffix-item": - input_items.append({"type": "computer_call", "id": "computer_unknown"}) - - assert not responses_input_suffix_matches_transition_manifest( - input_items, - stored_count=stored_count, - response_id="resp_manifest_fourcam", - pending_tool_calls=pending, - transition_manifest=manifest, - ) diff --git a/tests/unit/test_reports_service.py b/tests/unit/test_reports_service.py index 8a8690b4f5..ebfe92b596 100644 --- a/tests/unit/test_reports_service.py +++ b/tests/unit/test_reports_service.py @@ -53,6 +53,8 @@ async def test_get_reports_averages_use_inclusive_local_calendar_days( total_cost_usd=60.0, total_input_tokens=0, total_output_tokens=0, + total_reasoning_tokens=0, + reasoning_usage_known_requests=0, total_cached_tokens=0, total_requests=30, conversation_count=0, @@ -144,6 +146,8 @@ async def test_get_reports_serializes_conversation_and_breakdown_request_counts( total_cost_usd=1.2, total_input_tokens=12, total_output_tokens=6, + total_reasoning_tokens=4, + reasoning_usage_known_requests=2, total_cached_tokens=2, total_requests=2, conversation_count=1, @@ -155,6 +159,8 @@ async def test_get_reports_serializes_conversation_and_breakdown_request_counts( total_cost_usd=0.4, total_input_tokens=4, total_output_tokens=2, + total_reasoning_tokens=2, + reasoning_usage_known_requests=1, total_cached_tokens=0, total_requests=1, conversation_count=0, @@ -172,6 +178,7 @@ async def test_get_reports_serializes_conversation_and_breakdown_request_counts( conversation_count=1, input_tokens=12, output_tokens=6, + reasoning_tokens=None, cached_input_tokens=2, cost_usd=1.2, active_accounts=1, @@ -206,6 +213,7 @@ async def test_get_reports_serializes_conversation_and_breakdown_request_counts( None, None, "opencode", + None, ) repo.aggregate_daily_rows.assert_awaited_once_with( date(2026, 6, 1), @@ -214,6 +222,7 @@ async def test_get_reports_serializes_conversation_and_breakdown_request_counts( None, None, "opencode", + None, ) repo.aggregate_by_model.assert_awaited_once_with( datetime(2026, 6, 1, 0, 0, 0), @@ -221,6 +230,7 @@ async def test_get_reports_serializes_conversation_and_breakdown_request_counts( None, None, "opencode", + None, ) repo.aggregate_by_account.assert_awaited_once_with( datetime(2026, 6, 1, 0, 0, 0), @@ -228,6 +238,7 @@ async def test_get_reports_serializes_conversation_and_breakdown_request_counts( None, None, "opencode", + None, ) repo.aggregate_by_useragent.assert_awaited_once_with( datetime(2026, 6, 1, 0, 0, 0), @@ -235,15 +246,20 @@ async def test_get_reports_serializes_conversation_and_breakdown_request_counts( None, None, "opencode", + None, ) - repo.earliest_report_activity_at.assert_awaited_once_with(None, None, "opencode") + repo.earliest_report_activity_at.assert_awaited_once_with(None, None, "opencode", None) assert result.daily[0].median_ttft_ms == 123.46 assert result.daily[0].conversations == 1 assert result.daily[0].median_tps == 78.9 assert result.daily[0].median_queue_ms == 45.68 + assert result.daily[0].reasoning_tokens is None assert result.by_model[0].model == "gpt-5.1" assert result.summary.total_conversations == 1 + assert result.summary.total_reasoning_tokens == 4 + assert result.summary.reasoning_usage_known_requests == 2 + assert result.comparison.previous.total_tokens == 6 assert result.by_model[0].requests == 2 assert result.by_useragent[0].useragent == "opencode" assert result.by_useragent[0].requests == 2 diff --git a/tests/unit/test_request_body_limit_middleware.py b/tests/unit/test_request_body_limit_middleware.py index cc6956b897..3206c9878a 100644 --- a/tests/unit/test_request_body_limit_middleware.py +++ b/tests/unit/test_request_body_limit_middleware.py @@ -8,14 +8,16 @@ import pytest from fastapi import Body, Depends, FastAPI, HTTPException from httpx import ASGITransport, AsyncByteStream, AsyncClient -from starlette.middleware.base import BaseHTTPMiddleware from starlette.types import Message, Receive, Scope, Send from app.core.config.settings import get_settings from app.core.handlers import add_exception_handlers from app.core.middleware.path_rewrite import BackendApiCodexV1AliasMiddleware from app.core.middleware.request_body_limit import RequestBodyLimitMiddleware, add_request_body_limit_middleware -from app.core.middleware.request_decompression import add_request_decompression_middleware +from app.core.middleware.request_decompression import ( + RequestDecompressionMiddleware, + add_request_decompression_middleware, +) from app.main import create_app pytestmark = pytest.mark.unit @@ -495,9 +497,7 @@ def test_production_middleware_order_keeps_alias_and_admission_outside_body_read alias_index = next(index for index, item in enumerate(middleware) if item.cls is BackendApiCodexV1AliasMiddleware) limit_index = next(index for index, item in enumerate(middleware) if item.cls is RequestBodyLimitMiddleware) decompression_index = next( - index - for index, item in enumerate(middleware) - if item.cls is BaseHTTPMiddleware and item.kwargs.get("dispatch").__name__ == "request_decompression_middleware" + index for index, item in enumerate(middleware) if item.cls is RequestDecompressionMiddleware ) assert alias_index < limit_index < decompression_index diff --git a/tests/unit/test_request_decompression_middleware.py b/tests/unit/test_request_decompression_middleware.py index e1240a28c5..e5978d3a25 100644 --- a/tests/unit/test_request_decompression_middleware.py +++ b/tests/unit/test_request_decompression_middleware.py @@ -1,25 +1,27 @@ from __future__ import annotations +import asyncio import gzip import json import zlib -from collections.abc import Awaitable, Callable -from typing import cast +from collections.abc import AsyncIterator import pytest import zstandard as zstd from fastapi import FastAPI, Request -from fastapi.responses import Response -from httpx import ASGITransport, AsyncClient +from fastapi.responses import StreamingResponse +from httpx import ASGITransport, AsyncByteStream, AsyncClient from starlette.requests import ClientDisconnect +from starlette.types import Message, Receive, Scope, Send from app.core.middleware.request_body_limit import add_request_body_limit_middleware -from app.core.middleware.request_decompression import add_request_decompression_middleware +from app.core.middleware.request_decompression import ( + RequestDecompressionMiddleware, + add_request_decompression_middleware, +) pytestmark = pytest.mark.unit -_Dispatch = Callable[[Request, Callable[[Request], Awaitable[Response]]], Awaitable[Response]] - def _build_echo_app(*, touch_headers: bool = False) -> FastAPI: app = FastAPI() @@ -442,67 +444,131 @@ async def test_request_decompression_keeps_default_limit_for_other_routes(monkey assert response_data["error"]["code"] == "payload_too_large" +def _encoded_post_scope() -> Scope: + return { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/echo", + "raw_path": b"/echo", + "query_string": b"", + "root_path": "", + "headers": [(b"content-encoding", b"gzip"), (b"content-type", b"application/json")], + "client": ("testclient", 50000), + "server": ("testserver", 80), + "state": {}, + } + + +async def _unused_send(message: Message) -> None: + raise AssertionError(f"send should not be reached, got {message['type']}") + + @pytest.mark.asyncio async def test_request_decompression_propagates_client_disconnect(): - app = FastAPI() - add_request_decompression_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) + async def downstream(scope: Scope, receive: Receive, send: Send) -> None: + raise AssertionError("downstream app should not run after client disconnect") - async def receive() -> dict[str, object]: - return {"type": "http.disconnect"} + middleware = RequestDecompressionMiddleware(downstream) - request = Request( - { - "type": "http", - "http_version": "1.1", - "method": "POST", - "scheme": "http", - "path": "/echo", - "raw_path": b"/echo", - "query_string": b"", - "root_path": "", - "headers": [(b"content-encoding", b"gzip"), (b"content-type", b"application/json")], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=receive, - ) - - async def call_next(_: Request): - raise AssertionError("call_next should not run after client disconnect") + async def receive() -> Message: + return {"type": "http.disconnect"} with pytest.raises(ClientDisconnect): - await dispatch(request, call_next) + await middleware(_encoded_post_scope(), receive, _unused_send) @pytest.mark.asyncio async def test_request_decompression_propagates_body_read_failures(): + async def downstream(scope: Scope, receive: Receive, send: Send) -> None: + raise AssertionError("downstream app should not run when body read fails") + + middleware = RequestDecompressionMiddleware(downstream) + + async def receive() -> Message: + raise RuntimeError("receive failed") + + with pytest.raises(RuntimeError, match="receive failed"): + await middleware(_encoded_post_scope(), receive, _unused_send) + + +class _ChunkedBody(AsyncByteStream): + def __init__(self, *chunks: bytes) -> None: + self._chunks = chunks + + async def __aiter__(self) -> AsyncIterator[bytes]: + for chunk in self._chunks: + yield chunk + + +@pytest.mark.asyncio +async def test_request_decompression_supports_chunked_compressed_upload(): + app = _build_echo_app() + + payload = {"hello": "chunked"} + compressed = gzip.compress(json.dumps(payload).encode("utf-8")) + assert len(compressed) > 10 + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + resp = await client.post( + "/echo", + content=_ChunkedBody(compressed[:10], compressed[10:]), + headers={"Content-Encoding": "gzip", "Content-Type": "application/json"}, + ) + + assert resp.status_code == 200 + response_data = resp.json() + assert response_data["content_encoding"] is None + assert response_data["data"] == payload + + +@pytest.mark.asyncio +async def test_client_disconnect_mid_sse_stops_stream_after_replayed_body(): + """Regression: without BaseHTTPMiddleware's receive wrapper, http.disconnect + must still reach StreamingResponse's disconnect listener through the + decompression middleware's replay receive.""" app = FastAPI() add_request_decompression_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) + add_request_body_limit_middleware(app) - async def receive() -> dict[str, object]: - raise RuntimeError("receive failed") + chunks_seen = asyncio.Event() + chunk_count = 0 - request = Request( - { - "type": "http", - "http_version": "1.1", - "method": "POST", - "scheme": "http", - "path": "/echo", - "raw_path": b"/echo", - "query_string": b"", - "root_path": "", - "headers": [(b"content-encoding", b"gzip"), (b"content-type", b"application/json")], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=receive, - ) - - async def call_next(_: Request): - raise AssertionError("call_next should not run when body read fails") + @app.post("/stream") + async def stream(request: Request) -> StreamingResponse: + data = await request.json() + assert data == {"hello": "sse"} - with pytest.raises(RuntimeError, match="receive failed"): - await dispatch(request, call_next) + async def event_stream() -> AsyncIterator[bytes]: + while True: + yield b"data: tick\n\n" + await asyncio.sleep(0) + + return StreamingResponse(event_stream(), media_type="text/event-stream") + + compressed = gzip.compress(json.dumps({"hello": "sse"}).encode("utf-8")) + scope = _encoded_post_scope() + scope["path"] = "/stream" + scope["raw_path"] = b"/stream" + + body_messages = iter([{"type": "http.request", "body": compressed, "more_body": False}]) + + async def receive() -> Message: + for message in body_messages: + return message + # Simulate the client hanging up once a few SSE chunks have streamed. + await chunks_seen.wait() + return {"type": "http.disconnect"} + + async def send(message: Message) -> None: + nonlocal chunk_count + if message["type"] == "http.response.body" and message.get("body"): + chunk_count += 1 + if chunk_count >= 3: + chunks_seen.set() + + await asyncio.wait_for(app(scope, receive, send), timeout=5) + assert chunk_count >= 3 diff --git a/tests/unit/test_request_id_middleware.py b/tests/unit/test_request_id_middleware.py index 3a48b4c47b..17f952b615 100644 --- a/tests/unit/test_request_id_middleware.py +++ b/tests/unit/test_request_id_middleware.py @@ -1,98 +1,86 @@ from __future__ import annotations import asyncio -from collections.abc import Awaitable, Callable -from typing import cast import pytest -from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse, Response -from starlette.types import Message +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient from app.core.middleware.request_id import add_request_id_middleware from app.core.utils.request_id import get_request_id, get_request_scope_id pytestmark = pytest.mark.unit -_Dispatch = Callable[[Request, Callable[[Request], Awaitable[Response]]], Awaitable[Response]] + +def _build_app(request_ids: list[str | None], scope_ids: list[str | None]) -> FastAPI: + app = FastAPI() + add_request_id_middleware(app) + + @app.get("/health") + async def health() -> dict[str, bool]: + request_ids.append(get_request_id()) + scope_ids.append(get_request_scope_id()) + return {"ok": True} + + return app @pytest.mark.asyncio async def test_request_id_middleware_resets_context_on_success(): - app = FastAPI() - add_request_id_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) - - request = Request( - { - "type": "http", - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/health", - "raw_path": b"/health", - "query_string": b"", - "root_path": "", - "headers": [(b"x-request-id", b"req-test-123")], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=_empty_receive, - ) - - async def call_next(_: Request) -> JSONResponse: - assert get_request_id() == "req-test-123" - assert get_request_scope_id() not in {None, "req-test-123"} - return JSONResponse({"ok": True}) - - response = await dispatch(request, call_next) + request_ids: list[str | None] = [] + scope_ids: list[str | None] = [] + transport = ASGITransport(app=_build_app(request_ids, scope_ids)) + + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + response = await client.get("/health", headers={"x-request-id": "req-test-123"}) assert response.headers["x-request-id"] == "req-test-123" + assert request_ids == ["req-test-123"] + assert scope_ids[0] not in {None, "req-test-123"} assert get_request_id() is None assert get_request_scope_id() is None @pytest.mark.asyncio -async def test_request_id_middleware_uses_distinct_server_scopes_for_duplicate_client_ids(): - app = FastAPI() - add_request_id_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) - scopes: list[str] = [] - - def make_request() -> Request: - return Request( - { - "type": "http", - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/health", - "raw_path": b"/health", - "query_string": b"", - "root_path": "", - "headers": [(b"x-request-id", b"duplicate-client-id")], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=_empty_receive, - ) +async def test_request_id_middleware_generates_id_when_missing(): + request_ids: list[str | None] = [] + scope_ids: list[str | None] = [] + transport = ASGITransport(app=_build_app(request_ids, scope_ids)) - async def call_next(_: Request) -> JSONResponse: - assert get_request_id() == "duplicate-client-id" - scope = get_request_scope_id() - assert scope is not None - scopes.append(scope) - return JSONResponse({"ok": True}) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + response = await client.get("/health") - first, second = await asyncio.gather( - dispatch(make_request(), call_next), - dispatch(make_request(), call_next), - ) + generated = response.headers["x-request-id"] + assert generated + assert request_ids == [generated] - assert first.headers["x-request-id"] == "duplicate-client-id" - assert second.headers["x-request-id"] == "duplicate-client-id" - assert len(set(scopes)) == 2 + +@pytest.mark.asyncio +async def test_request_id_middleware_falls_back_to_request_id_header(): + request_ids: list[str | None] = [] + scope_ids: list[str | None] = [] + transport = ASGITransport(app=_build_app(request_ids, scope_ids)) + + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + response = await client.get("/health", headers={"request-id": "legacy-456"}) + + assert response.headers["x-request-id"] == "legacy-456" + assert request_ids == ["legacy-456"] -async def _empty_receive() -> Message: - return {"type": "http.request", "body": b"", "more_body": False} +@pytest.mark.asyncio +async def test_request_id_middleware_uses_distinct_server_scopes_for_duplicate_client_ids(): + request_ids: list[str | None] = [] + scope_ids: list[str | None] = [] + transport = ASGITransport(app=_build_app(request_ids, scope_ids)) + + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + first, second = await asyncio.gather( + client.get("/health", headers={"x-request-id": "duplicate-client-id"}), + client.get("/health", headers={"x-request-id": "duplicate-client-id"}), + ) + + assert first.headers["x-request-id"] == "duplicate-client-id" + assert second.headers["x-request-id"] == "duplicate-client-id" + assert request_ids == ["duplicate-client-id", "duplicate-client-id"] + assert len(set(scope_ids)) == 2 diff --git a/tests/unit/test_request_logs_repository.py b/tests/unit/test_request_logs_repository.py index 784705922d..ff56ef296a 100644 --- a/tests/unit/test_request_logs_repository.py +++ b/tests/unit/test_request_logs_repository.py @@ -22,7 +22,12 @@ def _clear_recent_count_cache_between_tests(): @pytest.mark.asyncio -async def test_add_log_ignores_closed_transaction(monkeypatch) -> None: +async def test_add_log_ignores_closed_transaction(monkeypatch, db_setup) -> None: + # The insert now executes eagerly (Core insert instead of a unit-of-work + # flush inside commit), so the schema must exist; the contract under test + # is unchanged: a ResourceClosedError commit is swallowed and the built + # log row is still returned. + del db_setup async with SessionLocal() as session: repo = RequestLogsRepository(session) diff --git a/tests/unit/test_request_policy.py b/tests/unit/test_request_policy.py index 93f5ffaefa..4b113bb288 100644 --- a/tests/unit/test_request_policy.py +++ b/tests/unit/test_request_policy.py @@ -5,11 +5,19 @@ import pytest -from app.core.exceptions import ProxyModelNotAllowed +from app.core.exceptions import ProxyModelNotAllowed, ProxyReasoningEffortNotAllowed +from app.core.openai.exceptions import ClientPayloadError from app.core.openai.model_registry import ModelRegistry -from app.core.openai.requests import ResponsesRequest +from app.core.openai.requests import ResponsesCompactRequest, ResponsesRequest +from app.core.types import JsonValue from app.modules.api_keys.service import ApiKeyData -from app.modules.proxy.request_policy import apply_api_key_enforcement, validate_model_access +from app.modules.proxy.request_policy import ( + apply_api_key_enforcement, + apply_api_key_enforcement_to_chat_payload, + normalize_source_reasoning_aliases, + responses_source_route_excluded, + validate_model_access, +) @pytest.mark.parametrize( @@ -243,3 +251,650 @@ def test_model_access_rejects_alias_when_canonical_model_not_allowed() -> None: with pytest.raises(ProxyModelNotAllowed): validate_model_access(api_key, "gpt-5.5-extra") + + +def test_reasoning_effort_allowlist_rejects_max_before_wire_normalization() -> None: + request = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [], + "reasoning": {"effort": "max"}, + } + ) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-reasoning-policy", + enforced_model=None, + enforced_reasoning_effort=None, + allowed_reasoning_efforts=["minimal", "low", "medium", "high", "xhigh"], + enforced_service_tier=None, + ), + ) + + with pytest.raises(ProxyReasoningEffortNotAllowed, match="max") as raised: + apply_api_key_enforcement(request, api_key) + + assert raised.value.code == "reasoning_effort_not_allowed" + assert raised.value.param == "reasoning.effort" + assert request.reasoning is not None + assert request.reasoning.effort == "max" + + +def test_reasoning_effort_allowlist_uses_client_plane_model_alias() -> None: + request = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol-xhigh", + "instructions": "", + "input": [], + } + ) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-alias-reasoning-policy", + enforced_model=None, + enforced_reasoning_effort=None, + allowed_reasoning_efforts=["xhigh"], + enforced_service_tier=None, + ), + ) + + apply_api_key_enforcement(request, api_key) + + assert request.model == "gpt-5.6-sol" + assert request.reasoning is not None + assert request.reasoning.effort == "high" + + +def test_reasoning_effort_allowlist_preserves_client_alias_when_model_is_enforced() -> None: + request = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol-xhigh", + "instructions": "", + "input": [], + } + ) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-enforced-plain-model-reasoning-policy", + enforced_model="gpt-5.6-sol", + enforced_reasoning_effort=None, + allowed_reasoning_efforts=["xhigh"], + enforced_service_tier=None, + ), + ) + + apply_api_key_enforcement(request, api_key) + + assert request.model == "gpt-5.6-sol" + assert request.reasoning is not None + assert request.reasoning.effort == "high" + + +def test_reasoning_effort_allowlist_is_idempotent_after_wire_normalization() -> None: + request = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol-xhigh", + "instructions": "", + "input": [], + } + ) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-idempotent-reasoning-policy", + enforced_model=None, + enforced_reasoning_effort=None, + allowed_reasoning_efforts=["xhigh"], + enforced_service_tier=None, + ), + ) + + apply_api_key_enforcement(request, api_key) + apply_api_key_enforcement(request, api_key) + + +@pytest.mark.parametrize( + ("field", "value", "expected_effort"), + [ + ("reasoningEffort", "max", "max"), + ("reasoning_effort", "max", "max"), + ("thinking", "minimal", "minimal"), + ("thinking", {"effort": "max", "summary": "auto"}, "max"), + ("enable_thinking", True, "medium"), + ], +) +def test_reasoning_effort_allowlist_checks_responses_alias_fields( + field: str, + value: JsonValue, + expected_effort: str, +) -> None: + request = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [], + field: value, + } + ) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-responses-alias-reasoning-policy", + enforced_model=None, + enforced_reasoning_effort=None, + allowed_reasoning_efforts=["low"], + enforced_service_tier=None, + ), + ) + + with pytest.raises(ProxyReasoningEffortNotAllowed, match=expected_effort): + apply_api_key_enforcement(request, api_key) + + +@pytest.mark.parametrize( + "request_type", + [ + pytest.param(ResponsesRequest, id="responses-and-websocket"), + pytest.param(ResponsesCompactRequest, id="compact"), + ], +) +def test_provider_reasoning_alias_runs_subscription_wire_fallback(request_type) -> None: + request = request_type.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [], + "thinking": "minimal", + } + ) + + apply_api_key_enforcement(request, None) + + assert request.reasoning is not None + assert request.reasoning.effort == "low" + assert request.to_payload()["reasoning"] == {"effort": "low"} + + +def test_reasoning_effort_allowlist_ignores_blank_alias_before_thinking_effort() -> None: + request = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [], + "reasoningEffort": " ", + "thinking": "max", + } + ) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-blank-reasoning-alias-policy", + enforced_model=None, + enforced_reasoning_effort=None, + allowed_reasoning_efforts=["low"], + enforced_service_tier=None, + ), + ) + + with pytest.raises(ProxyReasoningEffortNotAllowed, match="max"): + apply_api_key_enforcement(request, api_key) + + +@pytest.mark.parametrize( + "disabled_thinking", + [False, "disabled", "false", "off", {"type": "disabled"}, {"enabled": False}], +) +def test_reasoning_effort_allowlist_checks_enabled_alias_after_disabled_thinking( + disabled_thinking: JsonValue, +) -> None: + request = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [], + "thinking": disabled_thinking, + "enable_thinking": True, + } + ) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-conflicting-thinking-alias-policy", + enforced_model=None, + enforced_reasoning_effort=None, + allowed_reasoning_efforts=["low"], + enforced_service_tier=None, + ), + ) + + with pytest.raises(ProxyReasoningEffortNotAllowed, match="medium"): + apply_api_key_enforcement(request, api_key) + + +@pytest.mark.parametrize( + ("thinking", "enable_thinking"), + [ + ({"summary": "auto", "enabled": True}, None), + ({"summary": "auto", "type": "enabled"}, None), + ({"summary": "auto"}, True), + ], +) +def test_reasoning_effort_allowlist_checks_enabled_thinking_with_metadata( + thinking: JsonValue, + enable_thinking: bool | None, +) -> None: + request_payload: dict[str, JsonValue] = { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [], + "thinking": thinking, + } + if enable_thinking is not None: + request_payload["enable_thinking"] = enable_thinking + request = ResponsesRequest.model_validate(request_payload) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-thinking-metadata-policy", + enforced_model=None, + enforced_reasoning_effort=None, + allowed_reasoning_efforts=["low"], + enforced_service_tier=None, + ), + ) + + with pytest.raises(ProxyReasoningEffortNotAllowed, match="medium"): + apply_api_key_enforcement(request, api_key) + + +@pytest.mark.parametrize("request_type", [ResponsesRequest, ResponsesCompactRequest]) +@pytest.mark.parametrize(("alias_effort", "wire_effort"), [("minimal", "low"), ("ultra", "max")]) +def test_reasoning_aliases_receive_wire_normalization_after_allowlist( + request_type, + alias_effort: str, + wire_effort: str, +) -> None: + request = request_type.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "hi", + "input": [], + "reasoningEffort": alias_effort, + } + ) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-reasoning-alias-wire-normalization", + enforced_model=None, + enforced_reasoning_effort=None, + allowed_reasoning_efforts=[alias_effort], + enforced_service_tier=None, + ), + ) + + apply_api_key_enforcement(request, api_key) + + assert request.reasoning is not None + assert request.reasoning.effort == wire_effort + + +@pytest.mark.parametrize( + "request_type", + [ + pytest.param(ResponsesRequest, id="responses-and-websocket"), + pytest.param(ResponsesCompactRequest, id="compact"), + ], +) +def test_allowed_canonical_reasoning_effort_is_normalized_for_subscription_wire(request_type) -> None: + request = request_type.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "hi", + "input": [], + "reasoning": {"effort": " LOW "}, + } + ) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-canonical-reasoning-wire-normalization", + enforced_model=None, + enforced_reasoning_effort=None, + allowed_reasoning_efforts=["low"], + enforced_service_tier=None, + ), + ) + + apply_api_key_enforcement(request, api_key) + + assert request.reasoning is not None + assert request.reasoning.effort == "low" + assert request.to_payload()["reasoning"] == {"effort": "low"} + + +def test_reasoning_effort_allowlist_checks_explicit_effort_with_fast_model_alias() -> None: + request = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol-fast", + "instructions": "", + "input": [], + "reasoning": {"effort": "max"}, + } + ) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-fast-alias-reasoning-policy", + enforced_model=None, + enforced_reasoning_effort=None, + allowed_reasoning_efforts=["low"], + enforced_service_tier=None, + ), + ) + + with pytest.raises(ProxyReasoningEffortNotAllowed, match="max"): + apply_api_key_enforcement(request, api_key) + + +def test_reasoning_effort_allowlist_checks_alias_effort_from_enforced_model() -> None: + request = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [], + } + ) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-enforced-model-reasoning-policy", + enforced_model="gpt-5.6-sol-xhigh", + enforced_reasoning_effort=None, + allowed_reasoning_efforts=["low"], + enforced_service_tier=None, + ), + ) + + with pytest.raises(ProxyReasoningEffortNotAllowed, match="xhigh"): + apply_api_key_enforcement(request, api_key) + + +def test_reasoning_effort_allowlist_allows_alias_effort_from_enforced_model() -> None: + request = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [], + } + ) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-enforced-model-allowed-reasoning-policy", + enforced_model="gpt-5.6-sol-xhigh", + enforced_reasoning_effort=None, + allowed_reasoning_efforts=["xhigh"], + enforced_service_tier=None, + ), + ) + + apply_api_key_enforcement(request, api_key) + + assert request.model == "gpt-5.6-sol" + assert request.reasoning is not None + assert request.reasoning.effort == "high" + + +@pytest.mark.parametrize( + ("requested_effort", "allowed_effort"), + [ + ("xhigh", "high"), + ("high", "xhigh"), + ("ultra", "max"), + ("max", "ultra"), + ], +) +def test_reasoning_effort_allowlist_keeps_client_plane_efforts_distinct( + requested_effort: str, + allowed_effort: str, +) -> None: + request = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [], + "reasoning": {"effort": requested_effort}, + } + ) + api_key = cast( + ApiKeyData, + SimpleNamespace( + id="key-ultra-reasoning-policy", + enforced_model=None, + enforced_reasoning_effort=None, + allowed_reasoning_efforts=[allowed_effort], + enforced_service_tier=None, + ), + ) + + with pytest.raises(ProxyReasoningEffortNotAllowed, match=requested_effort): + apply_api_key_enforcement(request, api_key) + + +def test_source_chat_reasoning_aliases_remain_unchanged_without_policy() -> None: + payload: dict[str, JsonValue] = { + "reasoning_effort": "ultra", + "reasoningEffort": "ultra", + "thinking": {"effort": "ultra", "summary": "auto"}, + "reasoning": {"effort": "ultra", "summary": "auto"}, + } + + apply_api_key_enforcement_to_chat_payload(payload, None) + + assert payload == { + "reasoning_effort": "ultra", + "reasoningEffort": "ultra", + "thinking": {"effort": "ultra", "summary": "auto"}, + "reasoning": {"effort": "ultra", "summary": "auto"}, + } + + +def test_source_chat_reasoning_policy_aligns_conflicting_aliases() -> None: + payload: dict[str, JsonValue] = { + "reasoning_effort": "low", + "reasoningEffort": "low", + "thinking": {"effort": "ultra", "summary": "auto", "type": "budget", "budget": 4096}, + "enable_thinking": True, + "reasoning": {"effort": "ultra", "summary": "auto"}, + } + + apply_api_key_enforcement_to_chat_payload(payload, None, allowed_reasoning_effort="low") + + assert payload == { + "reasoning_effort": "low", + "reasoningEffort": "low", + "thinking": {"effort": "low", "summary": "auto", "type": "budget", "budget": 4096}, + "reasoning": {"effort": "low", "summary": "auto"}, + } + + +@pytest.mark.parametrize("inactive_selector", [{"type": "disabled"}, {"enabled": False}]) +def test_source_chat_reasoning_policy_removes_inactive_selector_from_explicit_thinking( + inactive_selector: dict[str, JsonValue], +) -> None: + payload: dict[str, JsonValue] = { + "thinking": {"effort": "low", **inactive_selector, "vendor_hint": "keep"}, + } + + apply_api_key_enforcement_to_chat_payload(payload, None, allowed_reasoning_effort="low") + + assert payload == {"thinking": {"effort": "low", "vendor_hint": "keep"}} + + +@pytest.mark.parametrize("effort", ["minimal", "xhigh"]) +def test_source_chat_reasoning_policy_preserves_client_plane_effort(effort: str) -> None: + payload: dict[str, JsonValue] = {"reasoning_effort": "low"} + + apply_api_key_enforcement_to_chat_payload(payload, None, allowed_reasoning_effort=effort) + + assert payload == { + "reasoning_effort": effort, + } + + +def test_source_chat_reasoning_policy_preserves_caller_alias_set() -> None: + payload: dict[str, JsonValue] = {"thinking": "ultra"} + + apply_api_key_enforcement_to_chat_payload(payload, None, allowed_reasoning_effort="ultra") + + assert payload == {"thinking": "max"} + + +def test_source_chat_reasoning_policy_preserves_authorized_enable_thinking() -> None: + payload: dict[str, JsonValue] = {"enable_thinking": True} + + apply_api_key_enforcement_to_chat_payload(payload, None, allowed_reasoning_effort="medium") + + assert payload == {"enable_thinking": True} + + +@pytest.mark.parametrize( + "thinking", + [ + {"type": "enabled", "budget_tokens": 2048}, + {"enabled": True, "summary": "auto", "vendor_hint": "keep"}, + ], +) +def test_source_chat_reasoning_policy_preserves_implicit_thinking_object(thinking: dict[str, JsonValue]) -> None: + payload: dict[str, JsonValue] = {"thinking": thinking} + + apply_api_key_enforcement_to_chat_payload(payload, None, allowed_reasoning_effort="medium") + + assert payload == {"thinking": thinking} + + +def test_source_chat_reasoning_policy_strips_blank_effort_from_implicit_thinking_object() -> None: + payload: dict[str, JsonValue] = { + "thinking": {"effort": " ", "enabled": True, "budget_tokens": 2048, "vendor_hint": "keep"} + } + + apply_api_key_enforcement_to_chat_payload(payload, None, allowed_reasoning_effort="medium") + + assert payload == {"thinking": {"enabled": True, "budget_tokens": 2048, "vendor_hint": "keep"}} + + +@pytest.mark.parametrize("thinking", [{"enabled": False}, {"type": "disabled"}]) +def test_source_chat_reasoning_policy_drops_inactive_thinking_object_beside_enable_alias( + thinking: dict[str, JsonValue], +) -> None: + payload: dict[str, JsonValue] = {"thinking": thinking, "enable_thinking": True} + + apply_api_key_enforcement_to_chat_payload(payload, None, allowed_reasoning_effort="medium") + + assert payload == {"enable_thinking": True} + + +def test_source_chat_reasoning_policy_drops_conflicting_implicit_thinking_object() -> None: + payload: dict[str, JsonValue] = { + "reasoning_effort": "low", + "thinking": {"type": "enabled", "budget_tokens": 2048}, + } + + apply_api_key_enforcement_to_chat_payload(payload, None, allowed_reasoning_effort="low") + + assert payload == {"reasoning_effort": "low"} + + +def test_source_reasoning_policy_preserves_effortless_thinking_beside_enable_alias() -> None: + thinking: dict[str, JsonValue] = {"type": "adaptive", "budget_tokens": 2048} + payload: dict[str, JsonValue] = { + "reasoning": {"effort": "low"}, + "thinking": thinking, + "enable_thinking": True, + } + + normalize_source_reasoning_aliases(payload) + + assert payload == {"reasoning": {"effort": "low"}, "thinking": thinking} + + +def test_source_reasoning_policy_strips_blank_effort_from_preserved_thinking() -> None: + payload: dict[str, JsonValue] = { + "reasoning": {"effort": "low"}, + "thinking": {"effort": " ", "type": "adaptive", "vendor_hint": "keep"}, + } + + normalize_source_reasoning_aliases(payload) + + assert payload == { + "reasoning": {"effort": "low"}, + "thinking": {"type": "adaptive", "vendor_hint": "keep"}, + } + + +@pytest.mark.parametrize("thinking", [False, {"type": "disabled"}, {"enabled": False}]) +def test_source_reasoning_policy_drops_inactive_thinking_beside_enable_alias(thinking: JsonValue) -> None: + payload: dict[str, JsonValue] = {"thinking": thinking, "enable_thinking": True} + + normalize_source_reasoning_aliases(payload) + + assert payload == {"reasoning": {"effort": "medium"}} + + +def test_source_chat_reasoning_policy_materializes_model_alias_effort_for_canonical_source() -> None: + payload: dict[str, JsonValue] = {} + + apply_api_key_enforcement_to_chat_payload( + payload, + None, + allowed_reasoning_effort="xhigh", + materialize_allowed_reasoning_effort=True, + ) + + assert payload == {"reasoning_effort": "xhigh"} + + +def _responses_request_with_input(input_value: object) -> ResponsesRequest: + return ResponsesRequest.model_validate({"model": "gpt-5", "instructions": "", "input": input_value}) + + +def test_source_route_excluded_is_false_for_plain_turns() -> None: + request = _responses_request_with_input([{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}]) + + assert responses_source_route_excluded(request) is False + + +def test_source_route_excluded_for_input_file_references() -> None: + request = _responses_request_with_input( + [{"role": "user", "content": [{"type": "input_file", "file_id": "file_123"}]}] + ) + + assert responses_source_route_excluded(request) is True + + +def test_source_route_excluded_for_terminal_compaction_trigger() -> None: + request = _responses_request_with_input( + [ + {"role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + {"type": "compaction_trigger"}, + ] + ) + + assert responses_source_route_excluded(request) is True + + +def test_source_route_excluded_raises_for_malformed_compaction_trigger() -> None: + request = _responses_request_with_input( + [ + {"type": "compaction_trigger"}, + {"role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + ] + ) + + with pytest.raises(ClientPayloadError): + responses_source_route_excluded(request) diff --git a/tests/unit/test_response_transition_manifest.py b/tests/unit/test_response_transition_manifest.py deleted file mode 100644 index ae6e81b14d..0000000000 --- a/tests/unit/test_response_transition_manifest.py +++ /dev/null @@ -1,220 +0,0 @@ -from __future__ import annotations - -import copy - -from app.core.openai.public_output import ( - MAX_PUBLIC_RESPONSE_OUTPUT_ITEMS, - collect_public_output_item_event, - merge_public_response_output_items, -) -from app.core.types import JsonValue -from app.modules.proxy.response_transition_manifest import ( - build_response_transition_manifest, - decode_response_transition_manifest, - encode_response_transition_manifest, - match_response_transition_manifest_prefix, - response_transition_manifest_matches_context, -) - - -def _completed_payload() -> dict[str, JsonValue]: - return { - "type": "response.completed", - "response": { - "id": "resp_manifest_1", - "status": "completed", - "output": [ - { - "type": "reasoning", - "id": "rs_manifest_1", - "encrypted_content": "secret-reasoning-ciphertext", - "summary": [], - "status": "completed", - }, - { - "type": "message", - "id": "msg_manifest_1", - "role": "assistant", - "phase": "commentary", - "content": [{"type": "output_text", "text": "secret commentary"}], - }, - { - "type": "custom_tool_call", - "id": "ctc_manifest_1", - "call_id": "call_manifest_1", - "name": "shell", - "input": "secret command", - "status": "completed", - }, - ], - }, - } - - -def test_streamed_output_item_collection_is_bounded_and_ordered() -> None: - output_items: dict[int, dict[str, JsonValue]] = {} - second_item: dict[str, JsonValue] = {"type": "message", "role": "assistant", "content": []} - first_item: dict[str, JsonValue] = {"type": "reasoning", "summary": []} - - assert collect_public_output_item_event( - { - "type": "response.output_item.added", - "output_index": 1, - "item": second_item, - }, - output_items, - ) - assert collect_public_output_item_event( - { - "type": "response.output_item.done", - "output_index": 0, - "item": first_item, - }, - output_items, - ) - assert merge_public_response_output_items( - {"id": "resp_streamed", "status": "completed", "output": []}, - output_items, - )["output"] == [first_item, second_item] - - assert not collect_public_output_item_event( - { - "type": "response.output_item.done", - "output_index": MAX_PUBLIC_RESPONSE_OUTPUT_ITEMS, - "item": first_item, - }, - output_items, - ) - assert not collect_public_output_item_event( - { - "type": "response.output_item.done", - "output_index": True, - "item": first_item, - }, - output_items, - ) - - -def test_response_transition_manifest_round_trip_is_content_free() -> None: - manifest = build_response_transition_manifest( - _completed_payload(), - pending_tool_calls={"call_manifest_1": "custom_tool_call"}, - ) - - assert manifest is not None - encoded = encode_response_transition_manifest(manifest) - assert encoded is not None - assert "secret" not in encoded - assert "call_manifest_1" not in encoded - assert "resp_manifest_1" not in encoded - assert decode_response_transition_manifest(encoded) == manifest - assert response_transition_manifest_matches_context( - manifest, - response_id="resp_manifest_1", - pending_tool_calls={"call_manifest_1": "custom_tool_call"}, - ) - assert not response_transition_manifest_matches_context( - manifest, - response_id="resp_other", - pending_tool_calls={"call_manifest_1": "custom_tool_call"}, - ) - assert decode_response_transition_manifest(encoded.replace('"kind":"reasoning"', '"kind":"unknown"')) is None - - -def test_response_transition_manifest_matches_only_exact_ordered_output_prefix() -> None: - payload = _completed_payload() - manifest = build_response_transition_manifest( - payload, - pending_tool_calls={"call_manifest_1": "custom_tool_call"}, - ) - assert manifest is not None - response = payload["response"] - assert isinstance(response, dict) - output = response["output"] - assert isinstance(output, list) - input_items: list[JsonValue] = [ - {"type": "message", "role": "user", "content": "stored"}, - *copy.deepcopy(output), - ] - - assert match_response_transition_manifest_prefix(input_items, stored_count=1, manifest=manifest) == 4 - - reordered = copy.deepcopy(input_items) - reordered[1], reordered[2] = reordered[2], reordered[1] - assert match_response_transition_manifest_prefix(reordered, stored_count=1, manifest=manifest) is None - - mutated = copy.deepcopy(input_items) - assert isinstance(mutated[2], dict) - mutated[2]["content"] = [{"type": "output_text", "text": "changed"}] - assert match_response_transition_manifest_prefix(mutated, stored_count=1, manifest=manifest) is None - - -def test_response_transition_manifest_rejects_inconsistent_or_unsupported_output() -> None: - payload = _completed_payload() - - assert ( - build_response_transition_manifest( - payload, - pending_tool_calls={"call_other": "custom_tool_call"}, - ) - is None - ) - - unsupported = copy.deepcopy(payload) - response = unsupported["response"] - assert isinstance(response, dict) - output = response["output"] - assert isinstance(output, list) - assert isinstance(output[0], dict) - output[0]["type"] = "computer_call" - assert ( - build_response_transition_manifest( - unsupported, - pending_tool_calls={"call_manifest_1": "custom_tool_call"}, - ) - is None - ) - - -def test_response_transition_manifest_uses_public_retry_representation() -> None: - payload: dict[str, JsonValue] = { - "type": "response.completed", - "response": { - "id": "resp_manifest_public_1", - "status": "completed", - "output": [ - { - "type": "agent_message", - "id": "amsg_01a02b33-3b30-7742-bdb3-091f07cf2ea0", - "author": "/root/worker", - "recipient": "/root", - "content": [{"type": "input_text", "text": "completed result"}], - } - ], - }, - } - manifest = build_response_transition_manifest( - payload, - pending_tool_calls={}, - normalize_for_public_contract=True, - ) - - assert manifest is not None - assert manifest.item_kinds == ("message:assistant",) - assert ( - match_response_transition_manifest_prefix( - [ - {"role": "user", "content": "stored"}, - { - "type": "message", - "id": "amsg_01a02b33-3b30-7742-bdb3-091f07cf2ea0", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "completed result"}], - }, - ], - stored_count=1, - manifest=manifest, - ) - == 2 - ) diff --git a/tests/unit/test_rowless_recovery.py b/tests/unit/test_rowless_recovery.py deleted file mode 100644 index 425426725e..0000000000 --- a/tests/unit/test_rowless_recovery.py +++ /dev/null @@ -1,2166 +0,0 @@ -from __future__ import annotations - -import asyncio -import copy -from collections.abc import AsyncIterator, Callable -from dataclasses import replace -from datetime import datetime, timedelta, timezone -from typing import cast - -import pytest -from sqlalchemy import select, update -from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine -from starlette.requests import Request - -import app.modules.proxy.rowless_recovery_api as rowless_recovery_api -from app.core.auth.dashboard_access import admin_principal, guest_principal -from app.core.auth.dashboard_mode import DashboardAuthMode -from app.core.clients.proxy_websocket import UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE -from app.core.openai.requests import ResponsesRequest -from app.db.models import ( - AuditLog, - Base, - HttpBridgeRecoveryAttemptRecord, - HttpBridgeRecoveryAttemptState, - HttpBridgeRowlessRecoveryAuthority, - HttpBridgeRowlessRecoveryState, - HttpBridgeSessionRecord, -) -from app.modules.proxy.durable_bridge_repository import DurableBridgeRepository, durable_bridge_hash -from app.modules.proxy.rowless_recovery import ( - ROWLESS_AUTHORIZATION_MODE_AUTOMATIC, - ROWLESS_AUTHORIZATION_MODE_OPERATOR, - ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - RowlessRecoveryCaptureFacts, - approved_rowless_recovery_projection, - build_rowless_recovery_capture_facts, - canonical_json_sha256, - rowless_strong_session_hash, - rowless_task_authority_digest, -) -from app.modules.proxy.rowless_recovery_repository import ( - ROWLESS_RECOVERY_CAPTURED_RETENTION_SECONDS, - RowlessCheckpointReceipt, - RowlessRecoveryAuthoritySnapshot, - RowlessRecoveryRepository, - RowlessRecoveryStateError, -) - -pytestmark = pytest.mark.unit - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "principal", - ( - admin_principal(auth_mode=DashboardAuthMode.DISABLED), - admin_principal(auth_mode=DashboardAuthMode.STANDARD, actor="password-admin"), - admin_principal(auth_mode=DashboardAuthMode.TRUSTED_HEADER, actor=""), - guest_principal(), - ), -) -async def test_rowless_admin_dependency_rejects_non_trusted_operator(monkeypatch, principal) -> None: - async def resolve(_request): - return principal - - monkeypatch.setattr(rowless_recovery_api, "require_dashboard_admin_access", resolve) - request = Request({"type": "http", "method": "GET", "path": "/", "headers": []}) - with pytest.raises(Exception) as exc_info: - await rowless_recovery_api.require_authenticated_rebase_admin(request) - assert getattr(exc_info.value, "status_code", None) == 403 - - -@pytest.fixture -async def async_session_factory() -> AsyncIterator[Callable[[], AsyncSession]]: - engine = create_async_engine("sqlite+aiosqlite:///:memory:") - async with engine.begin() as connection: - await connection.run_sync(Base.metadata.create_all) - session_maker = async_sessionmaker(engine, expire_on_commit=False) - yield session_maker - await engine.dispose() - - -def _facts(*, input_count: int, account_neutral: bool = True) -> RowlessRecoveryCaptureFacts: - return RowlessRecoveryCaptureFacts( - input_item_count=input_count, - input_fingerprint=canonical_json_sha256([{"shape": input_count}]), - contract_fingerprint=canonical_json_sha256({"model": "gpt-5.6"}), - direct_call_ledger_digest=canonical_json_sha256({"retained": input_count}), - projected_payload_fingerprint=canonical_json_sha256({"projected": input_count}), - actual_wire_fingerprint=canonical_json_sha256({"wire": input_count}), - unresolved_count=0, - projected_input=[{"role": "user", "content": "redacted"}], - self_contained=True, - account_neutral=account_neutral, - retains_prior_output=True, - ) - - -def _receipt( - *, - task_id: str, - strong_session_hash: str, - full_ledger_pairs: int, - unresolved_count: int = 0, - captured_input_item_count: int = 85, -) -> RowlessCheckpointReceipt: - facts = _facts(input_count=captured_input_item_count) - return RowlessCheckpointReceipt( - schema="qk_http_bridge_rowless_checkpoint_receipt_v1", - remote_session_jsonl_sha256=canonical_json_sha256({"task": task_id}), - remote_session_jsonl_size_bytes=1000 + full_ledger_pairs, - remote_session_jsonl_last_offset=1000 + full_ledger_pairs, - full_checkpoint_tool_ledger_digest=canonical_json_sha256( - {"domain": "full_checkpoint_tool_ledger_v1", "pairs": full_ledger_pairs} - ), - unresolved_count=unresolved_count, - task_identity=task_id, - session_identity=task_id, - strong_session_hash=strong_session_hash, - task_authority_digest=rowless_task_authority_digest( - session_id=task_id, - prompt_cache_key=task_id, - thread_id=task_id, - ), - captured_input_item_count=captured_input_item_count, - captured_input_fingerprint=facts.input_fingerprint, - non_input_contract_fingerprint=facts.contract_fingerprint, - retained_request_direct_call_ledger_digest=facts.direct_call_ledger_digest, - captured_projected_payload_fingerprint=facts.projected_payload_fingerprint, - captured_actual_wire_fingerprint=facts.actual_wire_fingerprint, - captured_request_binding_provenance="server_challenge", - ) - - -async def _capture_and_challenge( - session: AsyncSession, - *, - task_id: str, - input_count: int, - stale_anchor_hash: str | None = None, - origin_marker_session_id: str | None = None, -) -> tuple[RowlessRecoveryRepository, RowlessRecoveryAuthoritySnapshot, str]: - repository = RowlessRecoveryRepository(session) - task_authority_digest = rowless_task_authority_digest( - session_id=task_id, - prompt_cache_key=task_id, - thread_id=task_id, - ) - strong_hash = rowless_strong_session_hash("task_authority", task_authority_digest) - captured = await repository.capture( - api_key_scope="key-scope", - session_key_kind="session_header", - strong_session_hash=strong_hash, - stale_anchor_hash=stale_anchor_hash or canonical_json_sha256({"anchor": task_id}), - selected_account_intent="account-a", - task_identity=task_id, - session_identity=task_id, - task_authority_digest=task_authority_digest, - facts=_facts(input_count=input_count), - origin_marker_session_id=origin_marker_session_id, - ) - challenge = await repository.issue_challenge( - authority_id=captured.id, - generation=captured.generation, - ) - return repository, captured, challenge.challenge - - -@pytest.mark.asyncio -async def test_rowless_capture_rejects_account_scoped_request( - async_session_factory: Callable[[], AsyncSession], -) -> None: - async with async_session_factory() as session: - repository = RowlessRecoveryRepository(session) - with pytest.raises(RowlessRecoveryStateError, match="rowless_request_not_account_neutral"): - await repository.capture( - api_key_scope="key-scope", - session_key_kind="session_header", - strong_session_hash=rowless_strong_session_hash("session_header", "task-a"), - stale_anchor_hash="a" * 64, - selected_account_intent="account-b-that-rejected", - task_identity="task-a", - session_identity="task-a", - task_authority_digest=rowless_task_authority_digest( - session_id="task-a", - prompt_cache_key="task-a", - thread_id="task-a", - ), - facts=_facts(input_count=85, account_neutral=False), - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("task_id", "retained_input_count", "full_ledger_pairs"), - ( - ("01a02f21-77a1-7cc2-a892-b6abac317deb", 85, 93), - ("01a02edf-376c-7c13-a7de-08715e492fab", 288, 110), - ("01a0287a-6709-7cc3-9bfd-b1967f4af3f5", 194, 1186), - ), -) -async def test_rowless_approval_separates_full_checkpoint_and_retained_request_ledger_domains( - async_session_factory: Callable[[], AsyncSession], - task_id: str, - retained_input_count: int, - full_ledger_pairs: int, -) -> None: - async with async_session_factory() as session: - repository, captured, challenge = await _capture_and_challenge( - session, - task_id=task_id, - input_count=retained_input_count, - ) - receipt = _receipt( - task_id=task_id, - strong_session_hash=captured.strong_session_hash, - full_ledger_pairs=full_ledger_pairs, - captured_input_item_count=retained_input_count, - ) - assert receipt.full_checkpoint_tool_ledger_digest != captured.settled_direct_call_ledger_digest - - approved = await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id="admin-request", - ) - - assert approved.state == HttpBridgeRowlessRecoveryState.APPROVED - assert approved.captured_input_item_count == retained_input_count - assert approved.authorization_mode == ROWLESS_AUTHORIZATION_MODE_OPERATOR - assert approved.authorization_proof_sha256 == receipt.sha256() - - -@pytest.mark.asyncio -async def test_automatic_live_request_claims_new_authority_without_operator_approval( - async_session_factory: Callable[[], AsyncSession], -) -> None: - task_id = "task-automatic-live" - task_authority_digest = rowless_task_authority_digest( - session_id=task_id, - prompt_cache_key=task_id, - thread_id=task_id, - ) - strong_hash = rowless_strong_session_hash("task_authority", task_authority_digest) - facts = _facts(input_count=122) - - async with async_session_factory() as session: - repository = RowlessRecoveryRepository(session) - claimed = await repository.capture_and_claim_automatic_preflight( - api_key_scope="key-scope", - session_key_kind="session_header", - strong_session_hash=strong_hash, - stale_anchor_hash="a" * 64, - selected_account_intent="account-a", - task_identity=task_id, - session_identity=task_id, - task_authority_digest=task_authority_digest, - facts=facts, - request_id="request-automatic", - wire_request_fingerprint=facts.actual_wire_fingerprint, - ) - - assert claimed.generation == 1 - assert claimed.state == HttpBridgeRowlessRecoveryState.UNKNOWN - assert claimed.authorization_mode == ROWLESS_AUTHORIZATION_MODE_AUTOMATIC - assert claimed.authorization_proof_sha256 is not None - assert claimed.checkpoint_receipt_sha256 is None - assert claimed.dispatch_request_id == "request-automatic" - assert await repository.active_automatic_authority_count() == 1 - assert await repository.rollback_preflight_setup_failure( - authority_id=claimed.id, - generation=claimed.generation, - request_id="request-automatic", - wire_request_fingerprint=facts.actual_wire_fingerprint, - ) - restored = await repository.get(claimed.id) - assert restored is not None - assert restored.state == HttpBridgeRowlessRecoveryState.APPROVED - assert restored.authorization_mode == ROWLESS_AUTHORIZATION_MODE_AUTOMATIC - reclaimed = await repository.claim_dispatch_preflight( - authority_id=restored.id, - generation=restored.generation, - request_id="request-automatic-retry", - wire_request_fingerprint=facts.actual_wire_fingerprint, - task_authority_digest=restored.captured_task_authority_digest, - ) - assert reclaimed.state == HttpBridgeRowlessRecoveryState.UNKNOWN - assert await repository.active_automatic_authority_count() == 1 - - -@pytest.mark.asyncio -async def test_automatic_live_request_maps_flush_uniqueness_conflict_to_stable_rejection( - async_session_factory: Callable[[], AsyncSession], - monkeypatch: pytest.MonkeyPatch, -) -> None: - task_id = "task-automatic-flush-conflict" - task_authority_digest = rowless_task_authority_digest( - session_id=task_id, - prompt_cache_key=task_id, - thread_id=task_id, - ) - facts = _facts(input_count=124) - - async with async_session_factory() as session: - repository = RowlessRecoveryRepository(session) - - async def raise_uniqueness_conflict(*args, **kwargs) -> None: - del args, kwargs - raise IntegrityError("INSERT", {}, RuntimeError("duplicate task contract")) - - monkeypatch.setattr(session, "flush", raise_uniqueness_conflict) - with pytest.raises(RowlessRecoveryStateError, match="automatic_live_request_claim_conflict"): - await repository.capture_and_claim_automatic_preflight( - api_key_scope="key-scope", - session_key_kind="session_header", - strong_session_hash=rowless_strong_session_hash("task_authority", task_authority_digest), - stale_anchor_hash="d" * 64, - selected_account_intent="account-a", - task_identity=task_id, - session_identity=task_id, - task_authority_digest=task_authority_digest, - facts=facts, - request_id="request-flush-conflict", - wire_request_fingerprint=facts.actual_wire_fingerprint, - ) - assert await session.scalar(select(HttpBridgeRowlessRecoveryAuthority.id)) is None - - -@pytest.mark.asyncio -async def test_automatic_live_request_cancellation_after_commit_restores_unsent_claim( - async_session_factory: Callable[[], AsyncSession], - monkeypatch: pytest.MonkeyPatch, -) -> None: - task_id = "task-automatic-cancel-after-commit" - task_authority_digest = rowless_task_authority_digest( - session_id=task_id, - prompt_cache_key=task_id, - thread_id=task_id, - ) - strong_hash = rowless_strong_session_hash("task_authority", task_authority_digest) - facts = _facts(input_count=123) - - async with async_session_factory() as session: - repository = RowlessRecoveryRepository(session) - real_commit = session.commit - commit_finished = asyncio.Event() - release_commit = asyncio.Event() - - async def commit_then_pause() -> None: - await real_commit() - commit_finished.set() - await release_commit.wait() - - monkeypatch.setattr(session, "commit", commit_then_pause) - claim_task = asyncio.create_task( - repository.capture_and_claim_automatic_preflight( - api_key_scope="key-scope", - session_key_kind="session_header", - strong_session_hash=strong_hash, - stale_anchor_hash="c" * 64, - selected_account_intent="account-a", - task_identity=task_id, - session_identity=task_id, - task_authority_digest=task_authority_digest, - facts=facts, - request_id="request-cancel-after-commit", - wire_request_fingerprint=facts.actual_wire_fingerprint, - ) - ) - await asyncio.wait_for(commit_finished.wait(), timeout=2.0) - claim_task.cancel() - release_commit.set() - - with pytest.raises(asyncio.CancelledError): - await claim_task - - restored = await session.scalar(select(HttpBridgeRowlessRecoveryAuthority)) - assert restored is not None - assert restored.state == HttpBridgeRowlessRecoveryState.APPROVED - assert restored.dispatch_request_id is None - assert restored.wire_request_fingerprint is None - assert restored.dispatch_send_started_at is None - - -@pytest.mark.asyncio -async def test_automatic_live_request_supersedes_only_an_unsent_operator_generation( - async_session_factory: Callable[[], AsyncSession], -) -> None: - task_id = "task-automatic-supersede" - async with async_session_factory() as session: - repository, captured, challenge = await _capture_and_challenge( - session, - task_id=task_id, - input_count=122, - stale_anchor_hash="b" * 64, - ) - receipt = _receipt( - task_id=task_id, - strong_session_hash=captured.strong_session_hash, - full_ledger_pairs=120, - captured_input_item_count=122, - ) - approved = await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id="approval-request", - ) - changed_facts = _facts(input_count=124) - claimed = await repository.capture_and_claim_automatic_preflight( - api_key_scope=approved.api_key_scope, - session_key_kind=approved.session_key_kind, - strong_session_hash=approved.strong_session_hash, - stale_anchor_hash=approved.stale_anchor_hash, - selected_account_intent=approved.selected_account_intent, - task_identity=task_id, - session_identity=task_id, - task_authority_digest=approved.captured_task_authority_digest, - facts=changed_facts, - request_id="automatic-request", - wire_request_fingerprint=changed_facts.actual_wire_fingerprint, - expected_authority_id=approved.id, - expected_generation=approved.generation, - ) - - assert claimed.id == approved.id - assert claimed.generation == approved.generation + 1 - assert claimed.state == HttpBridgeRowlessRecoveryState.UNKNOWN - assert claimed.captured_input_item_count == 124 - assert claimed.authorization_mode == ROWLESS_AUTHORIZATION_MODE_AUTOMATIC - assert claimed.checkpoint_receipt_sha256 is None - assert claimed.authorization_proof_sha256 is not None - audit_actions = list(await session.scalars(select(AuditLog.action))) - assert "http_bridge_rowless_automatic_generation_superseded" in audit_actions - assert "http_bridge_rowless_automatic_live_request_claimed" in audit_actions - - with pytest.raises(RowlessRecoveryStateError, match="automatic_unsent_generation_fence_rejected"): - await repository.capture_and_claim_automatic_preflight( - api_key_scope=claimed.api_key_scope, - session_key_kind=claimed.session_key_kind, - strong_session_hash=claimed.strong_session_hash, - stale_anchor_hash=claimed.stale_anchor_hash, - selected_account_intent=claimed.selected_account_intent, - task_identity=task_id, - session_identity=task_id, - task_authority_digest=claimed.captured_task_authority_digest, - facts=_facts(input_count=126), - request_id="unsafe-second-request", - wire_request_fingerprint=_facts(input_count=126).actual_wire_fingerprint, - expected_authority_id=claimed.id, - expected_generation=claimed.generation, - ) - - -@pytest.mark.asyncio -async def test_automatic_live_request_ignores_historical_replayed_marker_journal( - async_session_factory: Callable[[], AsyncSession], -) -> None: - task_id = "task-automatic-replayed-journal" - rejected_anchor = "resp-automatic-replayed-journal" - task_authority_digest = rowless_task_authority_digest( - session_id=task_id, - prompt_cache_key=task_id, - thread_id=task_id, - ) - strong_hash = rowless_strong_session_hash("task_authority", task_authority_digest) - - async with async_session_factory() as session: - marker = HttpBridgeSessionRecord( - session_key_kind="session_header", - session_key_value=task_id, - session_key_hash=canonical_json_sha256(task_id), - api_key_scope="key-scope", - account_id="account-a", - owner_instance_id="instance-a", - owner_epoch=3, - latest_response_id=rejected_anchor, - recovery_required_anchor_hash=durable_bridge_hash(rejected_anchor), - recovery_required_account_id="account-a", - recovery_required_at=datetime.now(timezone.utc), - ) - session.add(marker) - await session.commit() - - repository = RowlessRecoveryRepository(session) - captured = await repository.capture( - api_key_scope="key-scope", - session_key_kind="session_header", - strong_session_hash=strong_hash, - stale_anchor_hash=durable_bridge_hash(rejected_anchor), - selected_account_intent="account-a", - task_identity=task_id, - session_identity=task_id, - task_authority_digest=task_authority_digest, - facts=_facts(input_count=122), - origin_marker_session_id=marker.id, - ) - challenge = await repository.issue_challenge( - authority_id=captured.id, - generation=captured.generation, - ) - receipt = _receipt( - task_id=task_id, - strong_session_hash=strong_hash, - full_ledger_pairs=120, - captured_input_item_count=122, - ) - approved = await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge.challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id="approval-request", - ) - session.add( - HttpBridgeRecoveryAttemptRecord( - session_id=marker.id, - request_fingerprint="e" * 64, - request_id="historical-request", - account_id="account-a", - model="gpt-5.6", - replay_safe=False, - state=HttpBridgeRecoveryAttemptState.REPLAYED, - response_id="resp-historical-complete", - ) - ) - await session.commit() - - changed_facts = _facts(input_count=124) - claimed = await repository.capture_and_claim_automatic_preflight( - api_key_scope=approved.api_key_scope, - session_key_kind=approved.session_key_kind, - strong_session_hash=approved.strong_session_hash, - stale_anchor_hash=approved.stale_anchor_hash, - selected_account_intent=approved.selected_account_intent, - task_identity=task_id, - session_identity=task_id, - task_authority_digest=approved.captured_task_authority_digest, - facts=changed_facts, - request_id="automatic-request", - wire_request_fingerprint=changed_facts.actual_wire_fingerprint, - origin_marker_session_id=marker.id, - expected_authority_id=approved.id, - expected_generation=approved.generation, - ) - - assert claimed.generation == approved.generation + 1 - assert claimed.state == HttpBridgeRowlessRecoveryState.UNKNOWN - - -@pytest.mark.asyncio -async def test_automatic_live_request_concurrent_claim_has_one_winner( - async_session_factory: Callable[[], AsyncSession], -) -> None: - task_id = "task-automatic-concurrent" - task_authority_digest = rowless_task_authority_digest( - session_id=task_id, - prompt_cache_key=task_id, - thread_id=task_id, - ) - strong_hash = rowless_strong_session_hash("task_authority", task_authority_digest) - facts = _facts(input_count=128) - - async def claim(request_id: str) -> RowlessRecoveryAuthoritySnapshot | Exception: - try: - async with async_session_factory() as session: - return await RowlessRecoveryRepository(session).capture_and_claim_automatic_preflight( - api_key_scope="key-scope", - session_key_kind="session_header", - strong_session_hash=strong_hash, - stale_anchor_hash="d" * 64, - selected_account_intent="account-a", - task_identity=task_id, - session_identity=task_id, - task_authority_digest=task_authority_digest, - facts=facts, - request_id=request_id, - wire_request_fingerprint=facts.actual_wire_fingerprint, - ) - except Exception as exc: - return exc - - outcomes = await asyncio.gather(claim("concurrent-a"), claim("concurrent-b")) - winners = [outcome for outcome in outcomes if isinstance(outcome, RowlessRecoveryAuthoritySnapshot)] - losers = [outcome for outcome in outcomes if isinstance(outcome, Exception)] - - assert len(winners) == 1 - assert winners[0].state == HttpBridgeRowlessRecoveryState.UNKNOWN - assert len(losers) == 1 - assert isinstance(losers[0], RowlessRecoveryStateError) - assert "automatic_unsent_generation_fence_rejected" in str(losers[0]) - - -@pytest.mark.asyncio -async def test_rowless_approval_rejects_cross_task_receipt_and_unresolved_checkpoint( - async_session_factory: Callable[[], AsyncSession], -) -> None: - async with async_session_factory() as session: - repository, captured, challenge = await _capture_and_challenge( - session, - task_id="task-a", - input_count=85, - ) - cross_task = _receipt( - task_id="task-b", - strong_session_hash=captured.strong_session_hash, - full_ledger_pairs=93, - ) - with pytest.raises(RowlessRecoveryStateError, match="checkpoint_receipt_contract_mismatch"): - await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge, - declared_receipt_sha256=cross_task.sha256(), - receipt=cross_task, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id=None, - ) - - unresolved = _receipt( - task_id="task-a", - strong_session_hash=captured.strong_session_hash, - full_ledger_pairs=93, - unresolved_count=1, - ) - with pytest.raises(RowlessRecoveryStateError, match="checkpoint_receipt_invalid"): - await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge, - declared_receipt_sha256=unresolved.sha256(), - receipt=unresolved, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id=None, - ) - - exact = _receipt( - task_id="task-a", - strong_session_hash=captured.strong_session_hash, - full_ledger_pairs=93, - ) - for drifted in ( - replace(exact, captured_input_item_count=exact.captured_input_item_count + 1), - replace(exact, captured_input_fingerprint="c" * 64), - replace(exact, non_input_contract_fingerprint="d" * 64), - replace(exact, retained_request_direct_call_ledger_digest="e" * 64), - ): - with pytest.raises(RowlessRecoveryStateError, match="checkpoint_receipt_contract_mismatch"): - await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge, - declared_receipt_sha256=drifted.sha256(), - receipt=drifted, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id=None, - ) - - -@pytest.mark.asyncio -async def test_rowless_dispatch_binds_replacement_session_and_send_started_fences_rollback( - async_session_factory: Callable[[], AsyncSession], -) -> None: - async with async_session_factory() as session: - repository, captured, challenge = await _capture_and_challenge( - session, - task_id="task-a", - input_count=85, - ) - receipt = _receipt( - task_id="task-a", - strong_session_hash=captured.strong_session_hash, - full_ledger_pairs=93, - ) - await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id=None, - ) - wrong_session = HttpBridgeSessionRecord( - session_key_kind="session_header", - session_key_value="task-b", - session_key_hash=canonical_json_sha256("task-b"), - api_key_scope="key-scope", - account_id="account-a", - ) - session.add(wrong_session) - await session.commit() - with pytest.raises(RowlessRecoveryStateError, match="approved_generation_dispatch_fence_rejected"): - await repository.claim_dispatch( - authority_id=captured.id, - generation=captured.generation, - replacement_session_id=wrong_session.id, - request_id="request-wrong", - wire_request_fingerprint="a" * 64, - model="gpt-5.6", - task_authority_digest=captured.captured_task_authority_digest, - ) - - replacement = HttpBridgeSessionRecord( - session_key_kind="session_header", - session_key_value="task-a", - session_key_hash=canonical_json_sha256("task-a"), - api_key_scope="key-scope", - account_id="account-a", - ) - session.add(replacement) - await session.commit() - replacement_id = replacement.id - await repository.claim_dispatch_preflight( - authority_id=captured.id, - generation=captured.generation, - request_id="request-a", - wire_request_fingerprint="b" * 64, - task_authority_digest=captured.captured_task_authority_digest, - ) - claimed = await repository.claim_dispatch( - authority_id=captured.id, - generation=captured.generation, - replacement_session_id=replacement_id, - request_id="request-a", - wire_request_fingerprint="b" * 64, - model="gpt-5.6", - task_authority_digest=captured.captured_task_authority_digest, - ) - assert claimed.state == HttpBridgeRowlessRecoveryState.UNKNOWN - assert claimed.dispatch_send_started_at is None - assert await repository.mark_dispatch_send_started( - authority_id=captured.id, - generation=captured.generation, - request_id="request-a", - wire_request_fingerprint="b" * 64, - ) - assert not await repository.rollback_proven_unsent( - authority_id=captured.id, - generation=captured.generation, - request_id="request-a", - wire_request_fingerprint="b" * 64, - ) - stored = await repository.get(captured.id) - assert stored is not None - assert stored.state == HttpBridgeRowlessRecoveryState.UNKNOWN - assert stored.dispatch_send_started_at is not None - assert stored.dispatch_send_started_at.tzinfo in (None, timezone.utc) - attempt = await session.scalar(select(HttpBridgeRecoveryAttemptRecord)) - assert attempt is not None - assert not await repository.rollback_physically_unsent_after_send_marker( - authority_id=captured.id, - generation=captured.generation, - request_id="request-a", - wire_request_fingerprint="b" * 64, - transport_proof_code="stream_incomplete", - ) - assert await repository.rollback_physically_unsent_after_send_marker( - authority_id=captured.id, - generation=captured.generation, - request_id="request-a", - wire_request_fingerprint="b" * 64, - transport_proof_code=UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - ) - restored = await repository.get(captured.id) - assert restored is not None - assert restored.state == HttpBridgeRowlessRecoveryState.APPROVED - assert restored.dispatch_send_started_at is None - assert await session.scalar(select(HttpBridgeRecoveryAttemptRecord)) is None - - -@pytest.mark.asyncio -async def test_marker_backed_admin_and_automatic_recovery_claims_are_mutually_exclusive( - async_session_factory: Callable[[], AsyncSession], -) -> None: - rejected_anchor = "resp-marker-exclusive" - task_id = "task-marker-exclusive" - task_authority_digest = rowless_task_authority_digest( - session_id=task_id, - prompt_cache_key=task_id, - thread_id=task_id, - ) - strong_hash = rowless_strong_session_hash("task_authority", task_authority_digest) - admin_wire_fingerprint = "a" * 64 - # Use the exact same raw wire fingerprint to cover the prior bug: the - # marker's legacy idempotency rule treated equal fingerprints as the same - # winner even when one belonged to the admin semantic-rebase authority. - automatic_wire_fingerprint = admin_wire_fingerprint - - async with async_session_factory() as session: - marker = HttpBridgeSessionRecord( - session_key_kind="session_header", - session_key_value=task_id, - session_key_hash=canonical_json_sha256(task_id), - api_key_scope="key-scope", - account_id="account-a", - owner_instance_id="instance-a", - owner_epoch=7, - latest_response_id=rejected_anchor, - recovery_required_anchor_hash=durable_bridge_hash(rejected_anchor), - recovery_required_account_id="account-a", - recovery_required_at=datetime.now(timezone.utc), - ) - session.add(marker) - await session.commit() - marker_id = marker.id - - repository = RowlessRecoveryRepository(session) - captured = await repository.capture( - api_key_scope="key-scope", - session_key_kind="session_header", - strong_session_hash=strong_hash, - stale_anchor_hash=durable_bridge_hash(rejected_anchor), - selected_account_intent="account-a", - task_identity=task_id, - session_identity=task_id, - task_authority_digest=task_authority_digest, - facts=_facts(input_count=85), - origin_marker_session_id=marker_id, - ) - challenge = await repository.issue_challenge( - authority_id=captured.id, - generation=captured.generation, - ) - receipt = _receipt( - task_id=task_id, - strong_session_hash=strong_hash, - full_ledger_pairs=93, - ) - approved = await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge.challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id="approval-request", - ) - assert approved.state == HttpBridgeRowlessRecoveryState.APPROVED - - await repository.claim_dispatch_preflight( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-dispatch", - wire_request_fingerprint=admin_wire_fingerprint, - task_authority_digest=task_authority_digest, - ) - await session.refresh(marker) - assert marker.recovery_required_attempt_fingerprint is not None - assert marker.recovery_required_attempt_fingerprint != admin_wire_fingerprint - assert not await DurableBridgeRepository(session).claim_recovery_required_attempt( - session_id=marker_id, - instance_id="instance-a", - owner_epoch=7, - account_id="account-a", - rejected_response_id=rejected_anchor, - attempt_fingerprint=automatic_wire_fingerprint, - request_id="generic-marker-loser", - ) - assert await repository.rollback_preflight_setup_failure( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-dispatch", - wire_request_fingerprint=admin_wire_fingerprint, - ) - await session.refresh(marker) - assert marker.recovery_required_attempt_fingerprint is None - - await repository.claim_dispatch_preflight( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-journal", - wire_request_fingerprint=admin_wire_fingerprint, - task_authority_digest=task_authority_digest, - ) - await repository.claim_dispatch( - authority_id=captured.id, - generation=captured.generation, - replacement_session_id=marker_id, - request_id="admin-journal", - wire_request_fingerprint=admin_wire_fingerprint, - model="gpt-5.6", - task_authority_digest=task_authority_digest, - ) - assert await repository.rollback_proven_unsent( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-journal", - wire_request_fingerprint=admin_wire_fingerprint, - ) - await session.refresh(marker) - assert marker.recovery_required_attempt_fingerprint is None - assert await session.scalar(select(HttpBridgeRecoveryAttemptRecord)) is None - - await repository.claim_dispatch_preflight( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-cancelled-before-send", - wire_request_fingerprint=admin_wire_fingerprint, - task_authority_digest=task_authority_digest, - ) - await repository.claim_dispatch( - authority_id=captured.id, - generation=captured.generation, - replacement_session_id=marker_id, - request_id="admin-cancelled-before-send", - wire_request_fingerprint=admin_wire_fingerprint, - model="gpt-5.6", - task_authority_digest=task_authority_digest, - ) - assert await repository.mark_dispatch_send_started( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-cancelled-before-send", - wire_request_fingerprint=admin_wire_fingerprint, - ) - assert await repository.rollback_before_send_primitive( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-cancelled-before-send", - wire_request_fingerprint=admin_wire_fingerprint, - ) - await session.refresh(marker) - assert marker.recovery_required_attempt_fingerprint is None - assert await session.scalar(select(HttpBridgeRecoveryAttemptRecord)) is None - - await repository.claim_dispatch_preflight( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-send-marker", - wire_request_fingerprint=admin_wire_fingerprint, - task_authority_digest=task_authority_digest, - ) - await repository.claim_dispatch( - authority_id=captured.id, - generation=captured.generation, - replacement_session_id=marker_id, - request_id="admin-send-marker", - wire_request_fingerprint=admin_wire_fingerprint, - model="gpt-5.6", - task_authority_digest=task_authority_digest, - ) - assert await repository.mark_dispatch_send_started( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-send-marker", - wire_request_fingerprint=admin_wire_fingerprint, - ) - assert not await repository.rollback_physically_unsent_after_send_marker( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-send-marker", - wire_request_fingerprint=admin_wire_fingerprint, - transport_proof_code="stream_incomplete", - ) - await session.refresh(marker) - assert marker.recovery_required_attempt_fingerprint is not None - ambiguous_authority = await repository.get(captured.id) - assert ambiguous_authority is not None - assert ambiguous_authority.state == HttpBridgeRowlessRecoveryState.UNKNOWN - ambiguous_attempt = await session.scalar(select(HttpBridgeRecoveryAttemptRecord)) - assert ambiguous_attempt is not None - assert ambiguous_attempt.state == HttpBridgeRecoveryAttemptState.UNKNOWN - assert await repository.rollback_physically_unsent_after_send_marker( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-send-marker", - wire_request_fingerprint=admin_wire_fingerprint, - transport_proof_code=UPSTREAM_WEBSOCKET_CLOSED_BEFORE_SEND_CODE, - ) - await session.refresh(marker) - assert marker.recovery_required_attempt_fingerprint is None - assert await session.scalar(select(HttpBridgeRecoveryAttemptRecord)) is None - - assert await DurableBridgeRepository(session).claim_recovery_required_attempt( - session_id=marker_id, - instance_id="instance-a", - owner_epoch=7, - account_id="account-a", - rejected_response_id=rejected_anchor, - attempt_fingerprint=automatic_wire_fingerprint, - request_id="generic-marker-owner", - ) - assert not await DurableBridgeRepository(session).claim_recovery_required_attempt( - session_id=marker_id, - instance_id="instance-a", - owner_epoch=7, - account_id="account-a", - rejected_response_id=rejected_anchor, - attempt_fingerprint=automatic_wire_fingerprint, - request_id="generic-marker-sibling", - ) - with pytest.raises(RowlessRecoveryStateError, match="approved_generation_preflight_fence_rejected"): - await repository.claim_dispatch_preflight( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-loser", - wire_request_fingerprint=admin_wire_fingerprint, - task_authority_digest=task_authority_digest, - ) - await session.refresh(marker) - assert marker.recovery_required_attempt_fingerprint == automatic_wire_fingerprint - marker_repository = DurableBridgeRepository(session) - assert not await marker_repository.rollback_recovery_required_attempt_before_dispatch( - session_id=marker_id, - api_key_scope="key-scope", - instance_id="instance-a", - owner_epoch=8, - account_id="account-a", - attempt_fingerprint=automatic_wire_fingerprint, - request_id="generic-marker-owner", - journal_request_id="generic-journal-owner", - ) - assert not await marker_repository.rollback_recovery_required_attempt_before_dispatch( - session_id=marker_id, - api_key_scope="key-scope", - instance_id="instance-a", - owner_epoch=7, - account_id="account-a", - attempt_fingerprint=automatic_wire_fingerprint, - request_id="generic-marker-sibling", - journal_request_id="generic-journal-owner", - ) - session.add( - HttpBridgeRecoveryAttemptRecord( - session_id=marker_id, - request_fingerprint=automatic_wire_fingerprint, - request_id="generic-journal-owner", - account_id="account-a", - model="gpt-5.6-sol", - replay_safe=True, - state=HttpBridgeRecoveryAttemptState.UNKNOWN, - ) - ) - await session.commit() - assert await marker_repository.rollback_recovery_required_attempt_before_dispatch( - session_id=marker_id, - api_key_scope="key-scope", - instance_id="instance-a", - owner_epoch=7, - account_id="account-a", - attempt_fingerprint=automatic_wire_fingerprint, - request_id="generic-marker-owner", - journal_request_id="generic-journal-owner", - ) - await session.refresh(marker) - assert marker.recovery_required_attempt_fingerprint is None - assert await session.scalar(select(HttpBridgeRecoveryAttemptRecord)) is None - - await session.delete(marker) - await session.commit() - with pytest.raises(RowlessRecoveryStateError, match="approved_generation_preflight_fence_rejected"): - await repository.claim_dispatch_preflight( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-missing-origin", - wire_request_fingerprint=admin_wire_fingerprint, - task_authority_digest=task_authority_digest, - ) - - -@pytest.mark.asyncio -async def test_marker_admin_and_automatic_claims_have_one_concurrent_winner( - async_session_factory: Callable[[], AsyncSession], -) -> None: - rejected_anchor = "resp-marker-concurrent-mixed" - task_id = "task-marker-concurrent-mixed" - task_authority_digest = rowless_task_authority_digest( - session_id=task_id, - prompt_cache_key=task_id, - thread_id=task_id, - ) - wire_fingerprint = "d" * 64 - async with async_session_factory() as session: - marker = HttpBridgeSessionRecord( - session_key_kind="session_header", - session_key_value=task_id, - session_key_hash=canonical_json_sha256(task_id), - api_key_scope="key-scope", - account_id="account-a", - owner_instance_id="instance-a", - owner_epoch=9, - latest_response_id=rejected_anchor, - recovery_required_anchor_hash=durable_bridge_hash(rejected_anchor), - recovery_required_account_id="account-a", - recovery_required_at=datetime.now(timezone.utc), - ) - session.add(marker) - await session.commit() - marker_id = marker.id - repository, captured, challenge = await _capture_and_challenge( - session, - task_id=task_id, - input_count=85, - stale_anchor_hash=durable_bridge_hash(rejected_anchor), - origin_marker_session_id=marker_id, - ) - receipt = _receipt( - task_id=task_id, - strong_session_hash=captured.strong_session_hash, - full_ledger_pairs=93, - ) - await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id="approval-request", - ) - - async def claim_admin() -> bool: - async with async_session_factory() as session: - try: - await RowlessRecoveryRepository(session).claim_dispatch_preflight( - authority_id=captured.id, - generation=captured.generation, - request_id="admin-concurrent", - wire_request_fingerprint=wire_fingerprint, - task_authority_digest=task_authority_digest, - ) - except RowlessRecoveryStateError: - return False - return True - - async def claim_automatic() -> bool: - async with async_session_factory() as session: - return await DurableBridgeRepository(session).claim_recovery_required_attempt( - session_id=marker_id, - instance_id="instance-a", - owner_epoch=9, - account_id="account-a", - rejected_response_id=rejected_anchor, - attempt_fingerprint=wire_fingerprint, - request_id="generic-concurrent", - ) - - winners = await asyncio.gather(claim_admin(), claim_automatic()) - assert winners.count(True) == 1 - async with async_session_factory() as session: - marker = await session.get(HttpBridgeSessionRecord, marker_id) - assert marker is not None - assert marker.recovery_required_attempt_fingerprint is not None - - -@pytest.mark.asyncio -async def test_marker_claim_snapshot_does_not_require_post_commit_refresh( - async_session_factory: Callable[[], AsyncSession], - monkeypatch, -) -> None: - rejected_anchor = "resp-marker-no-post-commit-refresh" - task_id = "task-marker-no-post-commit-refresh" - async with async_session_factory() as session: - marker = HttpBridgeSessionRecord( - session_key_kind="session_header", - session_key_value=task_id, - session_key_hash=canonical_json_sha256(task_id), - api_key_scope="key-scope", - account_id="account-a", - owner_instance_id="instance-a", - owner_epoch=11, - latest_response_id=rejected_anchor, - recovery_required_anchor_hash=durable_bridge_hash(rejected_anchor), - recovery_required_account_id="account-a", - recovery_required_at=datetime.now(timezone.utc), - ) - session.add(marker) - await session.commit() - - async def fail_refresh(*args, **kwargs): - del args, kwargs - raise AssertionError("claim must not await after a successful commit") - - monkeypatch.setattr(session, "refresh", fail_refresh) - claimed = await DurableBridgeRepository(session).claim_recovery_required_attempt_with_journal( - session_id=marker.id, - instance_id="instance-a", - owner_epoch=11, - account_id="account-a", - rejected_response_id=rejected_anchor, - attempt_fingerprint="e" * 64, - claim_request_id="claim-request", - journal_request_id="journal-request", - model="gpt-5.6", - ) - - assert claimed is not None - assert claimed.request_id == "journal-request" - assert claimed.state == HttpBridgeRecoveryAttemptState.UNKNOWN - - -@pytest.mark.asyncio -async def test_rowless_dispatch_can_rollback_only_before_send_started( - async_session_factory: Callable[[], AsyncSession], -) -> None: - async with async_session_factory() as session: - repository, captured, challenge = await _capture_and_challenge( - session, - task_id="task-a", - input_count=85, - ) - receipt = _receipt( - task_id="task-a", - strong_session_hash=captured.strong_session_hash, - full_ledger_pairs=93, - ) - await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id=None, - ) - replacement = HttpBridgeSessionRecord( - session_key_kind="session_header", - session_key_value="task-a", - session_key_hash=canonical_json_sha256("task-a"), - api_key_scope="key-scope", - account_id="account-a", - ) - session.add(replacement) - await session.commit() - replacement_id = replacement.id - await repository.claim_dispatch_preflight( - authority_id=captured.id, - generation=captured.generation, - request_id="request-a", - wire_request_fingerprint="c" * 64, - task_authority_digest=captured.captured_task_authority_digest, - ) - await repository.claim_dispatch( - authority_id=captured.id, - generation=captured.generation, - replacement_session_id=replacement_id, - request_id="request-a", - wire_request_fingerprint="c" * 64, - model="gpt-5.6", - task_authority_digest=captured.captured_task_authority_digest, - ) - assert await repository.rollback_proven_unsent( - authority_id=captured.id, - generation=captured.generation, - request_id="request-a", - wire_request_fingerprint="c" * 64, - ) - stored = await repository.get(captured.id) - assert stored is not None - assert stored.state == HttpBridgeRowlessRecoveryState.APPROVED - assert await session.scalar(select(HttpBridgeRecoveryAttemptRecord)) is None - - -@pytest.mark.asyncio -async def test_rowless_approval_requires_exact_explicit_acknowledgement( - async_session_factory: Callable[[], AsyncSession], -) -> None: - async with async_session_factory() as session: - repository, captured, challenge = await _capture_and_challenge( - session, - task_id="task-ack", - input_count=85, - ) - receipt = _receipt( - task_id="task-ack", - strong_session_hash=captured.strong_session_hash, - full_ledger_pairs=93, - ) - for acknowledgement in ("", "OPERATOR_ACKNOWLEDGED_SEMANTIC_REBASE", "operator_acknowledged"): - with pytest.raises(RowlessRecoveryStateError, match="semantic_rebase_acknowledgement_required"): - await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement=acknowledgement, - approved_actor="dashboard-admin", - request_id="admin-request", - ) - stored = await repository.get(captured.id) - assert stored is not None - assert stored.state == HttpBridgeRowlessRecoveryState.CAPTURED - assert await session.scalar(select(AuditLog)) is None - - -@pytest.mark.asyncio -async def test_rowless_duplicate_capture_is_stable_and_singleton( - async_session_factory: Callable[[], AsyncSession], -) -> None: - task_id = "task-duplicate" - digest = rowless_task_authority_digest( - session_id=task_id, - prompt_cache_key=task_id, - thread_id=task_id, - ) - - async def capture_with( - repository: RowlessRecoveryRepository, - *, - anchor: str, - ) -> RowlessRecoveryAuthoritySnapshot: - return await repository.capture( - api_key_scope="key-scope", - session_key_kind="session_header", - strong_session_hash=rowless_strong_session_hash("task_authority", digest), - stale_anchor_hash=canonical_json_sha256({"anchor": anchor}), - selected_account_intent="account-a", - task_identity=task_id, - session_identity=task_id, - task_authority_digest=digest, - facts=_facts(input_count=85), - ) - - async with async_session_factory() as session: - repository = RowlessRecoveryRepository(session) - first = await capture_with(repository, anchor="anchor-a") - second = await capture_with(repository, anchor="anchor-b") - assert second.id == first.id - rows = (await session.scalars(select(HttpBridgeRowlessRecoveryAuthority))).all() - assert [row.id for row in rows] == [first.id] - - async def capture_once(anchor: str) -> str: - async with async_session_factory() as session: - return (await capture_with(RowlessRecoveryRepository(session), anchor=anchor)).id - - assert await asyncio.gather(capture_once("anchor-c"), capture_once("anchor-d")) == [first.id, first.id] - - -@pytest.mark.asyncio -async def test_rowless_concurrent_first_captures_with_different_anchors_converge( - async_session_factory: Callable[[], AsyncSession], -) -> None: - task_id = "task-concurrent-different-anchors" - digest = rowless_task_authority_digest( - session_id=task_id, - prompt_cache_key=task_id, - thread_id=task_id, - ) - - async def capture(anchor: str) -> str: - async with async_session_factory() as session: - captured = await RowlessRecoveryRepository(session).capture( - api_key_scope="key-scope", - session_key_kind="session_header", - strong_session_hash=rowless_strong_session_hash("task_authority", digest), - stale_anchor_hash=canonical_json_sha256({"anchor": anchor}), - selected_account_intent="account-a", - task_identity=task_id, - session_identity=task_id, - task_authority_digest=digest, - facts=_facts(input_count=85), - ) - return captured.id - - authority_ids = await asyncio.gather(capture("anchor-a"), capture("anchor-b")) - assert authority_ids[0] == authority_ids[1] - async with async_session_factory() as session: - rows = (await session.scalars(select(HttpBridgeRowlessRecoveryAuthority))).all() - assert [row.id for row in rows] == [authority_ids[0]] - - -@pytest.mark.asyncio -async def test_rowless_preflight_cas_has_exactly_one_winner_and_crash_stays_unknown( - async_session_factory: Callable[[], AsyncSession], -) -> None: - async with async_session_factory() as session: - repository, captured, challenge = await _capture_and_challenge( - session, - task_id="task-concurrent", - input_count=85, - ) - receipt = _receipt( - task_id="task-concurrent", - strong_session_hash=captured.strong_session_hash, - full_ledger_pairs=93, - ) - await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id=None, - ) - - async def claim(request_id: str) -> str: - async with async_session_factory() as session: - try: - await RowlessRecoveryRepository(session).claim_dispatch_preflight( - authority_id=captured.id, - generation=captured.generation, - request_id=request_id, - wire_request_fingerprint="d" * 64, - task_authority_digest=captured.captured_task_authority_digest, - ) - except RowlessRecoveryStateError: - return "lost" - return "won" - - assert sorted(await asyncio.gather(claim("request-1"), claim("request-2"))) == ["lost", "won"] - async with async_session_factory() as session: - stored = await RowlessRecoveryRepository(session).get(captured.id) - assert stored is not None - assert stored.state == HttpBridgeRowlessRecoveryState.UNKNOWN - assert stored.replacement_session_id is None - assert stored.dispatch_send_started_at is None - assert await session.scalar(select(HttpBridgeRecoveryAttemptRecord)) is None - - -@pytest.mark.asyncio -async def test_rowless_local_setup_failure_can_restore_preflight_before_replacement( - async_session_factory: Callable[[], AsyncSession], -) -> None: - async with async_session_factory() as session: - repository, captured, challenge = await _capture_and_challenge( - session, - task_id="task-setup-failure", - input_count=85, - ) - receipt = _receipt( - task_id="task-setup-failure", - strong_session_hash=captured.strong_session_hash, - full_ledger_pairs=93, - ) - await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id=None, - ) - await repository.claim_dispatch_preflight( - authority_id=captured.id, - generation=captured.generation, - request_id="request-setup-failure", - wire_request_fingerprint="e" * 64, - task_authority_digest=captured.captured_task_authority_digest, - ) - assert await repository.rollback_preflight_setup_failure( - authority_id=captured.id, - generation=captured.generation, - request_id="request-setup-failure", - wire_request_fingerprint="e" * 64, - ) - stored = await repository.get(captured.id) - assert stored is not None - assert stored.state == HttpBridgeRowlessRecoveryState.APPROVED - assert stored.dispatch_request_id is None - - -@pytest.mark.asyncio -async def test_rowless_retention_purges_expired_capture_but_never_consumed_tombstone( - async_session_factory: Callable[[], AsyncSession], -) -> None: - task_ids = ("expired-captured", "expired-consumed", "approved-kept", "unknown-kept") - async with async_session_factory() as session: - repository = RowlessRecoveryRepository(session) - captured_ids: dict[str, str] = {} - for task_id in task_ids: - digest = rowless_task_authority_digest( - session_id=task_id, - prompt_cache_key=task_id, - thread_id=task_id, - ) - captured = await repository.capture( - api_key_scope="retention-scope", - session_key_kind="session_header", - strong_session_hash=rowless_strong_session_hash("task_authority", digest), - stale_anchor_hash=canonical_json_sha256({"anchor": task_id}), - selected_account_intent="account-a", - task_identity=task_id, - session_identity=task_id, - task_authority_digest=digest, - facts=_facts(input_count=85), - ) - captured_ids[task_id] = captured.id - - old = datetime.now(timezone.utc) - timedelta(seconds=ROWLESS_RECOVERY_CAPTURED_RETENTION_SECONDS + 60) - await session.execute( - update(HttpBridgeRowlessRecoveryAuthority) - .where(HttpBridgeRowlessRecoveryAuthority.id.in_(captured_ids.values())) - .values(updated_at=old) - ) - await session.execute( - update(HttpBridgeRowlessRecoveryAuthority) - .where(HttpBridgeRowlessRecoveryAuthority.id == captured_ids["expired-consumed"]) - .values(state=HttpBridgeRowlessRecoveryState.CONSUMED, consumed_at=old) - ) - await session.execute( - update(HttpBridgeRowlessRecoveryAuthority) - .where(HttpBridgeRowlessRecoveryAuthority.id == captured_ids["approved-kept"]) - .values(state=HttpBridgeRowlessRecoveryState.APPROVED) - ) - await session.execute( - update(HttpBridgeRowlessRecoveryAuthority) - .where(HttpBridgeRowlessRecoveryAuthority.id == captured_ids["unknown-kept"]) - .values(state=HttpBridgeRowlessRecoveryState.UNKNOWN) - ) - await session.commit() - - now = datetime.now(timezone.utc) - deleted = await repository.purge_expired_audit_rows( - captured_cutoff=now - timedelta(seconds=ROWLESS_RECOVERY_CAPTURED_RETENTION_SECONDS), - ) - assert deleted == {"captured": 1} - remaining = set((await session.scalars(select(HttpBridgeRowlessRecoveryAuthority.id))).all()) - assert remaining == { - captured_ids["expired-consumed"], - captured_ids["approved-kept"], - captured_ids["unknown-kept"], - } - consumed_task = "expired-consumed" - consumed_digest = rowless_task_authority_digest( - session_id=consumed_task, - prompt_cache_key=consumed_task, - thread_id=consumed_task, - ) - repeated = await repository.capture( - api_key_scope="retention-scope", - session_key_kind="session_header", - strong_session_hash=rowless_strong_session_hash("task_authority", consumed_digest), - stale_anchor_hash=canonical_json_sha256({"anchor": consumed_task}), - selected_account_intent="account-a", - task_identity=consumed_task, - session_identity=consumed_task, - task_authority_digest=consumed_digest, - facts=_facts(input_count=85), - ) - assert repeated.id == captured_ids["expired-consumed"] - assert repeated.state == HttpBridgeRowlessRecoveryState.CONSUMED - assert len((await session.scalars(select(HttpBridgeRowlessRecoveryAuthority))).all()) == 3 - - -def test_rowless_exact_three_sanitized_incident_shapes_are_self_contained_and_exact() -> None: - from tests.unit.test_replay_safety import _rehydrate_sanitized_pending_settlement_shapes - - cases = _rehydrate_sanitized_pending_settlement_shapes() - assert [stored_count for _, stored_count, _ in cases] == [85, 288, 194] - for items, _, _ in cases: - payload = ResponsesRequest.model_validate( - { - "model": "gpt-5.1", - "instructions": "sanitized fixture", - "previous_response_id": "resp_stale_fixture", - "prompt_cache_key": "fixture-task", - "input": items, - } - ) - facts = build_rowless_recovery_capture_facts(payload) - assert facts is not None - assert facts.self_contained - assert facts.account_neutral - assert facts.unresolved_count == 0 - assert ( - approved_rowless_recovery_projection( - payload, - captured_input_item_count=facts.input_item_count, - captured_input_fingerprint=facts.input_fingerprint, - non_input_contract_fingerprint=facts.contract_fingerprint, - direct_call_ledger_digest=facts.direct_call_ledger_digest, - projected_payload_fingerprint=facts.projected_payload_fingerprint, - ) - == facts.projected_input - ) - assert ( - approved_rowless_recovery_projection( - payload, - captured_input_item_count=facts.input_item_count, - captured_input_fingerprint=facts.input_fingerprint, - non_input_contract_fingerprint=facts.contract_fingerprint, - direct_call_ledger_digest=facts.direct_call_ledger_digest, - projected_payload_fingerprint="f" * 64, - ) - is None - ) - - -def test_automatic_rowless_proof_requires_retained_output_before_fresh_followup() -> None: - common = { - "model": "gpt-5.1", - "instructions": "sanitized fixture", - "previous_response_id": "resp_stale_fixture", - "prompt_cache_key": "fixture-task", - } - incremental = build_rowless_recovery_capture_facts( - ResponsesRequest.model_validate( - { - **common, - "input": [{"role": "user", "content": "continue"}], - } - ) - ) - complete = build_rowless_recovery_capture_facts( - ResponsesRequest.model_validate( - { - **common, - "input": [ - {"role": "user", "content": "original"}, - { - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "original result"}], - }, - {"role": "user", "content": "continue"}, - ], - } - ) - ) - compacted = build_rowless_recovery_capture_facts( - ResponsesRequest.model_validate( - { - **common, - "input": [ - { - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "compacted result"}], - }, - {"role": "user", "content": "continue"}, - ], - } - ) - ) - - assert incremental is not None - assert incremental.self_contained - assert not incremental.retains_prior_output - assert complete is not None - assert complete.self_contained - assert complete.retains_prior_output - assert compacted is not None - assert compacted.self_contained - assert compacted.retains_prior_output - - -def test_rowless_capture_normalizes_only_exact_empty_output_and_encrypted_agent_transport_parts() -> None: - items: list[object] = [ - { - "type": "custom_tool_call", - "call_id": "call-1", - "name": "fixture_tool", - "input": "{}", - "status": "completed", - }, - { - "type": "custom_tool_call_output", - "call_id": "call-1", - "output": [ - {"type": "input_text", "text": "settled"}, - {"type": "input_text", "text": ""}, - ], - }, - { - "type": "function_call", - "call_id": "call-2", - "namespace": "collaboration", - "name": "send_message", - "arguments": "{}", - }, - {"type": "function_call_output", "call_id": "call-2", "output": "settled"}, - { - "type": "agent_message", - "id": "amsg_00000000-0000-4000-8000-000000000001", - "author": "/root/worker", - "recipient": "/root", - "content": [ - {"type": "input_text", "text": "completed"}, - {"type": "encrypted_content", "encrypted_content": "opaque-response-state"}, - ], - "internal_chat_message_metadata_passthrough": { - "turn_id": "00000000-0000-4000-8000-000000000002", - "create_time": 1.0, - }, - }, - {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, - ] - payload = ResponsesRequest.model_validate( - { - "model": "gpt-5.6-sol", - "instructions": "sanitized fixture", - "previous_response_id": "resp_stale_fixture", - "prompt_cache_key": "fixture-task", - "input": items, - } - ) - - facts = build_rowless_recovery_capture_facts(payload) - - assert facts is not None - assert facts.self_contained - assert facts.account_neutral - assert facts.retains_prior_output - assert facts.unresolved_count == 0 - projected_output = next( - cast(dict[str, object], item) - for item in facts.projected_input - if isinstance(item, dict) and item.get("type") == "custom_tool_call_output" - ) - assert projected_output["output"] == [{"type": "input_text", "text": "settled"}] - projected_function_call = next( - cast(dict[str, object], item) - for item in facts.projected_input - if isinstance(item, dict) and item.get("type") == "function_call" - ) - assert projected_function_call["namespace"] == "collaboration" - projected_agent = next( - cast(dict[str, object], item) - for item in facts.projected_input - if isinstance(item, dict) and item.get("type") == "agent_message" - ) - assert "id" not in projected_agent - assert projected_agent["content"] == [{"type": "input_text", "text": "completed"}] - assert ( - approved_rowless_recovery_projection( - payload, - captured_input_item_count=facts.input_item_count, - captured_input_fingerprint=facts.input_fingerprint, - non_input_contract_fingerprint=facts.contract_fingerprint, - direct_call_ledger_digest=facts.direct_call_ledger_digest, - projected_payload_fingerprint=facts.projected_payload_fingerprint, - ) - == facts.projected_input - ) - - -def test_rowless_capture_accepts_compacted_agent_output_at_index_zero() -> None: - payload = ResponsesRequest.model_validate( - { - "model": "gpt-5.6-sol", - "instructions": "sanitized fixture", - "previous_response_id": "resp_stale_fixture", - "prompt_cache_key": "fixture-task", - "input": [ - { - "type": "agent_message", - "id": "amsg_00000000-0000-4000-8000-000000000001", - "author": "/root/worker", - "recipient": "/root", - "content": [{"type": "input_text", "text": "completed"}], - }, - {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, - ], - } - ) - - facts = build_rowless_recovery_capture_facts(payload) - - assert facts is not None - assert facts.self_contained - assert facts.account_neutral - assert facts.retains_prior_output - - -@pytest.mark.parametrize( - "mutate", - ( - lambda items: cast( - dict[str, object], cast(list[object], cast(dict[str, object], items[1])["output"])[1] - ).update({"unexpected": "drift"}), - lambda items: cast( - dict[str, object], cast(list[object], cast(dict[str, object], items[2])["content"])[1] - ).update({"unexpected": "drift"}), - lambda items: cast(dict[str, object], items[2]).update( - { - "content": [ - {"type": "encrypted_content", "encrypted_content": "opaque-response-state"}, - {"type": "input_text", "text": "completed"}, - ] - } - ), - ), -) -def test_rowless_transport_artifact_normalization_rejects_semantic_or_shape_drift(mutate) -> None: - items: list[object] = [ - { - "type": "custom_tool_call", - "call_id": "call-1", - "name": "fixture_tool", - "input": "{}", - "status": "completed", - }, - { - "type": "custom_tool_call_output", - "call_id": "call-1", - "output": [ - {"type": "input_text", "text": "settled"}, - {"type": "input_text", "text": ""}, - ], - }, - { - "type": "agent_message", - "id": "amsg_00000000-0000-4000-8000-000000000001", - "author": "/root/worker", - "recipient": "/root", - "content": [ - {"type": "input_text", "text": "completed"}, - {"type": "encrypted_content", "encrypted_content": "opaque-response-state"}, - ], - "internal_chat_message_metadata_passthrough": { - "turn_id": "00000000-0000-4000-8000-000000000002", - "create_time": 1.0, - }, - }, - {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, - ] - mutate(items) - payload = ResponsesRequest.model_validate( - { - "model": "gpt-5.6-sol", - "instructions": "sanitized fixture", - "previous_response_id": "resp_stale_fixture", - "prompt_cache_key": "fixture-task", - "input": items, - } - ) - - facts = build_rowless_recovery_capture_facts(payload) - assert facts is None or not facts.self_contained or not facts.account_neutral - - -def test_rowless_capture_rejects_transformable_external_image_url() -> None: - payload = ResponsesRequest.model_validate( - { - "model": "gpt-5.1", - "instructions": "Inspect the supplied image.", - "previous_response_id": "resp_stale", - "prompt_cache_key": "root-task", - "input": [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "inspect"}, - {"type": "input_image", "image_url": "https://example.invalid/dynamic.png"}, - ], - } - ], - } - ) - assert build_rowless_recovery_capture_facts(payload) is None - - -def test_rowless_agent_message_proof_rejects_shape_metadata_and_boundary_drift() -> None: - from tests.unit.test_replay_safety import _rehydrate_sanitized_pending_settlement_shapes - - items, _, _ = _rehydrate_sanitized_pending_settlement_shapes()[1] - agent_index = next( - index for index, item in enumerate(items) if isinstance(item, dict) and item.get("type") == "agent_message" - ) - malformed_variants = [] - - unknown_field = copy.deepcopy(items) - cast(dict[str, object], unknown_field[agent_index])["unexpected"] = "drift" - malformed_variants.append(unknown_field) - - metadata_drift = copy.deepcopy(items) - cast(dict[str, object], metadata_drift[agent_index])["internal_chat_message_metadata_passthrough"] = { - "turn_id": "not-a-uuid" - } - malformed_variants.append(metadata_drift) - - missing_fresh_user_boundary = copy.deepcopy(items[: agent_index + 1]) - malformed_variants.append(missing_fresh_user_boundary) - - equal_agent_paths = copy.deepcopy(items) - equal_agent = cast(dict[str, object], equal_agent_paths[agent_index]) - equal_agent["recipient"] = equal_agent["author"] - malformed_variants.append(equal_agent_paths) - - recipient_path_escape = copy.deepcopy(items) - cast(dict[str, object], recipient_path_escape[agent_index])["recipient"] = "/root/../escape" - malformed_variants.append(recipient_path_escape) - - output_bearing_agent = copy.deepcopy(items) - cast(dict[str, object], output_bearing_agent[agent_index])["content"] = [{"type": "output_text", "text": "drift"}] - malformed_variants.append(output_bearing_agent) - - invalid_agent_id = copy.deepcopy(items) - cast(dict[str, object], invalid_agent_id[agent_index])["id"] = "amsg_not-a-uuid" - malformed_variants.append(invalid_agent_id) - - pending_before_agent = copy.deepcopy(items) - pending_before_agent.insert( - agent_index, - { - "type": "custom_tool_call", - "call_id": "dangling-before-agent", - "name": "fixture_tool", - "input": "{}", - "status": "completed", - }, - ) - malformed_variants.append(pending_before_agent) - - orphan_output = copy.deepcopy(items) - orphan_output.insert( - agent_index, - { - "type": "custom_tool_call_output", - "call_id": "orphan-output", - "output": "fixture-output", - }, - ) - malformed_variants.append(orphan_output) - - dangling_after_agent = copy.deepcopy(items) - dangling_after_agent.insert( - agent_index + 1, - { - "type": "function_call", - "call_id": "dangling-after-agent", - "name": "fixture_tool", - "arguments": "{}", - }, - ) - malformed_variants.append(dangling_after_agent) - - for variant_index, malformed in enumerate(malformed_variants): - payload = ResponsesRequest.model_validate( - { - "model": "gpt-5.1", - "instructions": "sanitized fixture", - "previous_response_id": "resp_stale_fixture", - "prompt_cache_key": "fixture-task", - "input": malformed, - } - ) - facts = build_rowless_recovery_capture_facts(payload) - assert facts is None or not facts.self_contained or not facts.account_neutral, f"variant {variant_index}" - - -@pytest.mark.asyncio -async def test_rowless_task_authority_isolates_children_sharing_root_session_and_prompt( - async_session_factory: Callable[[], AsyncSession], -) -> None: - root_session = "shared-root-session" - child_a = "child-thread-a" - child_b = "child-thread-b" - digest_a = rowless_task_authority_digest( - session_id=root_session, - prompt_cache_key=root_session, - thread_id=child_a, - ) - digest_b = rowless_task_authority_digest( - session_id=root_session, - prompt_cache_key=root_session, - thread_id=child_b, - ) - assert digest_a != digest_b - - async with async_session_factory() as session: - repository = RowlessRecoveryRepository(session) - captured_a = await repository.capture( - api_key_scope="shared-scope", - session_key_kind="session_header", - strong_session_hash=rowless_strong_session_hash("task_authority", digest_a), - stale_anchor_hash="f" * 64, - selected_account_intent="account-a", - task_identity=child_a, - session_identity=root_session, - task_authority_digest=digest_a, - facts=_facts(input_count=85), - ) - captured_b = await repository.capture( - api_key_scope="shared-scope", - session_key_kind="session_header", - strong_session_hash=rowless_strong_session_hash("task_authority", digest_b), - stale_anchor_hash="f" * 64, - selected_account_intent="account-a", - task_identity=child_b, - session_identity=root_session, - task_authority_digest=digest_b, - facts=_facts(input_count=85), - ) - assert captured_a.id != captured_b.id - challenge = await repository.issue_challenge( - authority_id=captured_a.id, - generation=captured_a.generation, - ) - wrong_child_receipt = replace( - _receipt( - task_id=child_b, - strong_session_hash=captured_a.strong_session_hash, - full_ledger_pairs=93, - ), - session_identity=root_session, - task_authority_digest=digest_b, - ) - with pytest.raises(RowlessRecoveryStateError, match="checkpoint_receipt_contract_mismatch"): - await repository.approve( - authority_id=captured_a.id, - generation=captured_a.generation, - challenge=challenge.challenge, - declared_receipt_sha256=wrong_child_receipt.sha256(), - receipt=wrong_child_receipt, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id=None, - ) - - -@pytest.mark.asyncio -async def test_rowless_terminal_persistence_failure_rolls_back_anchor_and_consumption( - async_session_factory: Callable[[], AsyncSession], - monkeypatch, -) -> None: - async with async_session_factory() as session: - rejected_anchor = "resp-terminal-stale" - replacement = HttpBridgeSessionRecord( - session_key_kind="session_header", - session_key_value="task-terminal-rollback", - session_key_hash=canonical_json_sha256("task-terminal-rollback"), - api_key_scope="key-scope", - account_id="account-a", - owner_instance_id="instance-a", - owner_epoch=7, - latest_response_id=rejected_anchor, - recovery_required_anchor_hash=durable_bridge_hash(rejected_anchor), - recovery_required_account_id="account-a", - recovery_required_at=datetime.now(timezone.utc), - ) - session.add(replacement) - await session.commit() - replacement_id = replacement.id - repository, captured, challenge = await _capture_and_challenge( - session, - task_id="task-terminal-rollback", - input_count=85, - stale_anchor_hash=durable_bridge_hash(rejected_anchor), - origin_marker_session_id=replacement_id, - ) - receipt = _receipt( - task_id="task-terminal-rollback", - strong_session_hash=captured.strong_session_hash, - full_ledger_pairs=93, - ) - await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id=None, - ) - await repository.claim_dispatch_preflight( - authority_id=captured.id, - generation=captured.generation, - request_id="request-terminal", - wire_request_fingerprint="9" * 64, - task_authority_digest=captured.captured_task_authority_digest, - ) - await repository.claim_dispatch( - authority_id=captured.id, - generation=captured.generation, - replacement_session_id=replacement_id, - request_id="request-terminal", - wire_request_fingerprint="9" * 64, - model="gpt-5.6", - task_authority_digest=captured.captured_task_authority_digest, - ) - assert await repository.mark_dispatch_send_started( - authority_id=captured.id, - generation=captured.generation, - request_id="request-terminal", - wire_request_fingerprint="9" * 64, - ) - - async def fail_alias(*args, **kwargs): - del args, kwargs - raise RuntimeError("injected terminal persistence failure") - - monkeypatch.setattr(DurableBridgeRepository, "_execute_alias_upsert", fail_alias) - with pytest.raises(RuntimeError, match="injected terminal persistence failure"): - await repository.settle_completed( - authority_id=captured.id, - generation=captured.generation, - replacement_session_id=replacement_id, - owner_instance_id="instance-a", - owner_epoch=7, - request_id="request-terminal", - response_id="resp-terminal", - input_item_count=captured.captured_input_item_count, - input_full_fingerprint=captured.captured_input_fingerprint, - pending_tool_calls={}, - response_transition_manifest=None, - ) - await session.rollback() - stored = await repository.get(captured.id) - assert stored is not None - assert stored.state == HttpBridgeRowlessRecoveryState.UNKNOWN - durable_replacement = await session.get(HttpBridgeSessionRecord, replacement_id) - assert durable_replacement is not None - assert durable_replacement.latest_response_id == rejected_anchor - assert durable_replacement.recovery_required_anchor_hash == durable_bridge_hash(rejected_anchor) - assert durable_replacement.recovery_required_account_id == "account-a" - assert durable_replacement.recovery_required_attempt_fingerprint is not None - attempt = await session.scalar(select(HttpBridgeRecoveryAttemptRecord)) - assert attempt is not None - assert attempt.state.value == "unknown" - assert attempt.response_id is None - - -@pytest.mark.asyncio -async def test_rowless_state_counts_expose_non_captured_image_rollback_floor( - async_session_factory: Callable[[], AsyncSession], -) -> None: - async with async_session_factory() as session: - repository, captured, challenge = await _capture_and_challenge( - session, - task_id="task-rollback-floor", - input_count=85, - ) - assert await repository.authority_state_counts() == { - "captured": 1, - "approved": 0, - "unknown": 0, - "consumed": 0, - } - receipt = _receipt( - task_id="task-rollback-floor", - strong_session_hash=captured.strong_session_hash, - full_ledger_pairs=93, - ) - await repository.approve( - authority_id=captured.id, - generation=captured.generation, - challenge=challenge, - declared_receipt_sha256=receipt.sha256(), - receipt=receipt, - acknowledgement=ROWLESS_SEMANTIC_REBASE_ACKNOWLEDGEMENT, - approved_actor="dashboard-admin", - request_id=None, - ) - counts = await repository.authority_state_counts() - assert counts["captured"] == 0 - assert counts["approved"] == 1 diff --git a/tests/unit/test_select_with_stickiness.py b/tests/unit/test_select_with_stickiness.py index e4d9436a39..fd4f6fd1f7 100644 --- a/tests/unit/test_select_with_stickiness.py +++ b/tests/unit/test_select_with_stickiness.py @@ -9,6 +9,7 @@ import time from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone from typing import cast from unittest.mock import AsyncMock @@ -16,6 +17,10 @@ from app.core.balancer import AccountState, RoutingCost, RoutingCostsByAccount, RoutingStrategy from app.db.models import Account, AccountStatus, StickySessionKind +from app.modules.proxy._load_balancer.sticky_selection import ( + _STICKY_EXISTING_UNSET, + _sticky_refresh_write_skippable, +) from app.modules.proxy.load_balancer import LoadBalancer pytestmark = pytest.mark.unit @@ -77,6 +82,8 @@ async def _invoke_stickiness( relative_availability_power: float = 2.0, relative_availability_top_k: int = 5, routing_costs_by_account_id: RoutingCostsByAccount | None = None, + sticky_refresh_skip_deadline: datetime | None = None, + sticky_existing_account_id: str | None | object = _STICKY_EXISTING_UNSET, ): """Wrapper that calls production LoadBalancer._select_with_stickiness. @@ -107,8 +114,16 @@ async def mock_repo_factory(): relative_availability_top_k=relative_availability_top_k, sticky_repo=sticky_repo, routing_costs_by_account_id=routing_costs_by_account_id, + sticky_refresh_skip_deadline=sticky_refresh_skip_deadline, + sticky_existing_account_id=sticky_existing_account_id, ) - if outcome.mutation is not None: + # Mirror the production persist site (run_sticky_selection_path): a pure + # same-owner freshness rewrite is omitted only after revalidating its + # observed skip deadline at write time. + if outcome.mutation is not None and not _sticky_refresh_write_skippable( + outcome.mutation, + initialize_seed_key=None, + ): await lb._persist_sticky_mutation( sticky_repo=sticky_repo, sticky_key=sticky_key, @@ -118,6 +133,10 @@ async def mock_repo_factory(): return outcome.selection +def _future_deadline(seconds: float = 10.0) -> datetime: + return datetime.now(tz=timezone.utc).replace(tzinfo=None) + timedelta(seconds=seconds) + + # --------------------------------------------------------------------------- # Fix 1+3: sticky session is preserved when pinned account is temporarily down # --------------------------------------------------------------------------- @@ -1085,3 +1104,174 @@ async def test_burn_first_reallocation_only_when_burn_first_is_selectable(): assert result.account.account_id == "a" repo.delete.assert_not_called() repo.upsert.assert_called_once_with("key1", "a", kind=StickySessionKind.PROMPT_CACHE) + + +# --------------------------------------------------------------------------- +# Same-owner refresh skip: hot (key, kind) rows must not be rewritten on every +# request when the lookup already observed a fresh row. +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_refresh_skippable_pinned_retention_skips_upsert(): + """A healthy pinned owner within the refresh-skip window routes to the + pinned account without any sticky write.""" + acc_a = _active("a", used_percent=10.0) + acc_b = _active("b", used_percent=50.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [acc_a, acc_b], + "key1", + repo, + sticky_refresh_skip_deadline=_future_deadline(), + sticky_existing_account_id="a", + ) + + assert result.account is not None + assert result.account.account_id == "a" + repo.upsert.assert_not_called() + repo.delete.assert_not_called() + + +@pytest.mark.asyncio +async def test_refresh_skippable_false_pinned_retention_still_refreshes(): + """Without the freshness observation the pinned retention keeps its + write-through updated_at refresh.""" + acc_a = _active("a", used_percent=10.0) + acc_b = _active("b", used_percent=50.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [acc_a, acc_b], + "key1", + repo, + sticky_refresh_skip_deadline=None, + sticky_existing_account_id="a", + ) + + assert result.account is not None + assert result.account.account_id == "a" + repo.upsert.assert_called_once_with("key1", "a", kind=StickySessionKind.PROMPT_CACHE) + + +@pytest.mark.asyncio +async def test_refresh_skippable_never_suppresses_reallocation_write(): + """Budget-pressure rebind to a different owner must persist immediately + even when the old row was observed fresh.""" + acc_a = _active("a", used_percent=96.0) + acc_b = _active("b", used_percent=50.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [acc_a, acc_b], + "key1", + repo, + sticky_refresh_skip_deadline=_future_deadline(), + sticky_existing_account_id="a", + ) + + assert result.account is not None + assert result.account.account_id == "b" + repo.upsert.assert_called_once_with("key1", "b", kind=StickySessionKind.PROMPT_CACHE) + + +@pytest.mark.asyncio +async def test_refresh_skippable_never_suppresses_departed_owner_rebind(): + """A pinned owner that left the pool is still rebound with an immediate + write even when the old row was observed fresh.""" + acc_b = _active("b", used_percent=50.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [acc_b], + "key1", + repo, + sticky_refresh_skip_deadline=_future_deadline(), + sticky_existing_account_id="a", + ) + + assert result.account is not None + assert result.account.account_id == "b" + repo.upsert.assert_called_once_with("key1", "b", kind=StickySessionKind.PROMPT_CACHE) + + +@pytest.mark.asyncio +async def test_refresh_skippable_grace_period_retention_skips_upsert(): + """The grace-period pinned retention also honors the skip window.""" + now = time.time() + pinned = _rate_limited("a", reset_at=now + 10) + acc_b = _active("b", used_percent=50.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [pinned, acc_b], + "key1", + repo, + sticky_refresh_skip_deadline=_future_deadline(), + sticky_existing_account_id="a", + ) + + assert result.account is not None + assert result.account.account_id == "a" + repo.upsert.assert_not_called() + + +@pytest.mark.asyncio +async def test_refresh_skippable_reset_when_existing_owner_not_prefetched(): + """The freshness observation belongs to the caller-provided lookup; an + internal owner lookup must fall back to write-through refresh.""" + acc_a = _active("a", used_percent=10.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [acc_a], + "key1", + repo, + sticky_refresh_skip_deadline=_future_deadline(), + ) + + assert result.account is not None + assert result.account.account_id == "a" + repo.upsert.assert_called_once_with("key1", "a", kind=StickySessionKind.PROMPT_CACHE) + + +@pytest.mark.asyncio +async def test_refresh_skip_deadline_expired_at_persist_time_still_refreshes(): + """The skip deadline is revalidated at write time: a deadline that lapsed + between lookup and persist must not suppress the refresh, keeping the + mapping's effective expiry within the documented skip-window bound.""" + acc_a = _active("a", used_percent=10.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [acc_a], + "key1", + repo, + sticky_refresh_skip_deadline=_future_deadline(-0.5), + sticky_existing_account_id="a", + ) + + assert result.account is not None + assert result.account.account_id == "a" + repo.upsert.assert_called_once_with("key1", "a", kind=StickySessionKind.PROMPT_CACHE) + + +def test_refresh_write_skippable_guards_seed_and_delete_and_deadline(): + """The persist-time gate never skips deletes, seed-initializing writes, + non-datetime deadlines (auto-vivified test doubles), or lapsed deadlines.""" + from app.modules.proxy._load_balancer.sticky_selection import _StickyMutation + + refresh = _StickyMutation(account_id="a", refresh_skip_deadline=_future_deadline()) + assert _sticky_refresh_write_skippable(refresh, initialize_seed_key=None) is True + # Seed initialization piggybacks on this write and must never be skipped. + assert _sticky_refresh_write_skippable(refresh, initialize_seed_key="seed-key") is False + # Deletes are never skippable. + delete = _StickyMutation(account_id=None, refresh_skip_deadline=_future_deadline()) + assert _sticky_refresh_write_skippable(delete, initialize_seed_key=None) is False + # A lapsed deadline fails revalidation. + expired = _StickyMutation(account_id="a", refresh_skip_deadline=_future_deadline(-0.5)) + assert _sticky_refresh_write_skippable(expired, initialize_seed_key=None) is False + # Mutations without an observed deadline always write through. + plain = _StickyMutation(account_id="a") + assert _sticky_refresh_write_skippable(plain, initialize_seed_key=None) is False diff --git a/tests/unit/test_settings_reference.py b/tests/unit/test_settings_reference.py index d4c129e892..304ca88096 100644 --- a/tests/unit/test_settings_reference.py +++ b/tests/unit/test_settings_reference.py @@ -51,14 +51,25 @@ def _isolated_settings(**overrides: Any) -> Settings: # gate, issue #1535). Not a hardcoded default because the right congestion # threshold depends on pool size and workload mix, and 0-means-off is the P1 # default-off switch; the companion min-guarantee constant stayed hardcoded. -# 117 -> 118: http_responses_session_bridge_anchor_poison_failure_threshold -# (bridge restart anchor poisoning). Not hardcoded because operators need a -# bounded deployment-specific poison threshold while recovery telemetry matures. -# 118 -> 119: database_postgres_schema. Shared-database deployments need one -# explicit schema knob so runtime engines, migration jobs, and schema gates use -# the same non-public search_path; there is no safe fixed default when a single -# PostgreSQL database is reused by multiple applications. -MAX_SETTINGS_FIELDS = 119 +# 117 -> 126: durable HTTP bridge continuity controls (operation ledger, +# ambiguous-continuation recovery, and best-effort transcript spool, #1657). +# These remain operator-selectable because deployments differ in recovery +# safety policy and available persistence/latency budgets; their conservative +# defaults preserve fail-closed behavior and bound background write work. +# 126 -> 127: rate_limit_reset_credits_refresh_enabled (reset-credit polling +# toggle, #1701). Not a hardcoded default because "off" is a deployment +# decision — operators who don't use the reset-credit surface shed the +# per-replica authenticated upstream polling; default true keeps current +# zero-config behavior and the interval setting alone cannot express "off". +# 127 -> 129: telemetry_enabled + telemetry_endpoint (anonymous telemetry, +# #1618). telemetry_enabled has no hardcoded default because tri-state None +# drives the informed-consent dialog; the endpoint stays settable so +# self-hosters can point at their own collector or air-gap it. +# 129 -> 130: timeout_invariant_validation_strict (#1622). This stays +# operator-selectable because startup invariant failures need two supported +# modes: report-only by default for mixed/self-hosted environments, and +# fail-fast when CI or strict operators want config drift to abort startup. +MAX_SETTINGS_FIELDS = 132 def test_generated_settings_reference_matches_code() -> None: @@ -72,6 +83,7 @@ def test_generated_settings_reference_matches_code() -> None: def test_settings_reference_page_is_checked_in_under_docs() -> None: assert OUTPUT_PATH == REPO_ROOT / "docs" / "reference" / "settings.md" assert OUTPUT_PATH.is_file() + assert "openspec/specs/responses-api-compat" in render_settings_reference() def test_settings_surface_ratchet() -> None: diff --git a/tests/unit/test_settings_trace_and_removed.py b/tests/unit/test_settings_trace_and_removed.py index dd197e890d..fcf9c16a92 100644 --- a/tests/unit/test_settings_trace_and_removed.py +++ b/tests/unit/test_settings_trace_and_removed.py @@ -153,7 +153,7 @@ def test_phase_3_removed_settings_are_listed_and_ignored(monkeypatch): settings = Settings() assert not hasattr(settings, "database_pool_recycle_seconds") assert not hasattr(settings, "drain_primary_threshold_pct") - assert settings.database_pool_size == 15 + assert settings.database_pool_size == 25 assert settings.soft_drain_enabled is True found = warn_removed_settings( { diff --git a/tests/unit/test_shared_future_waiters.py b/tests/unit/test_shared_future_waiters.py new file mode 100644 index 0000000000..96abf57dfa --- /dev/null +++ b/tests/unit/test_shared_future_waiters.py @@ -0,0 +1,136 @@ +"""Regression tests for the shared-future waiter helper. + +The helper replaces ``wait_for(shield(shared))`` on futures awaited by many +concurrent waiters (http-bridge inflight/capacity registries, token-refresh +singleflight). The structural invariant under test: no matter how many +waiters attach, time out, or are cancelled, the shared future carries exactly +one done callback and no leaked per-waiter state. Under the old shield +pattern each waiter attached callbacks to the shared future and removed them +with O(n) scans — a mass timeout livelocked the event loop (2026-08-20 +production incident). +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from app.core.utils.shared_future import _WAITERS_ATTR, wait_on_shared_future + +pytestmark = pytest.mark.unit + + +def _callback_count(future: asyncio.Future) -> int | None: + callbacks = getattr(future, "_callbacks", None) + if callbacks is None: + return None + return len(callbacks) + + +async def test_result_propagates_to_all_waiters(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + waiters = [asyncio.create_task(wait_on_shared_future(shared, timeout=5)) for _ in range(10)] + await asyncio.sleep(0) + shared.set_result("session") + assert await asyncio.gather(*waiters) == ["session"] * 10 + + +async def test_exception_propagates_to_all_waiters(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + waiters = [asyncio.create_task(wait_on_shared_future(shared, timeout=5)) for _ in range(4)] + await asyncio.sleep(0) + shared.set_exception(RuntimeError("creation failed")) + results = await asyncio.gather(*waiters, return_exceptions=True) + assert all(isinstance(r, RuntimeError) and str(r) == "creation failed" for r in results) + + +async def test_shared_cancellation_cancels_waiters(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + waiters = [asyncio.create_task(wait_on_shared_future(shared, timeout=5)) for _ in range(4)] + await asyncio.sleep(0) + shared.cancel() + results = await asyncio.gather(*waiters, return_exceptions=True) + assert all(isinstance(r, asyncio.CancelledError) for r in results) + + +async def test_timeout_raises_and_leaves_shared_pending(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + with pytest.raises(TimeoutError): + await wait_on_shared_future(shared, timeout=0.01) + assert not shared.done() + assert not shared.cancelled() + # The owner can still complete the creation after waiters gave up. + shared.set_result("late") + assert await wait_on_shared_future(shared) == "late" + + +async def test_mass_timeout_does_not_accumulate_callbacks_on_shared(): + """The incident shape: many waiters piling onto one pending future and + timing out together must leave the shared future's callback list at its + constant size (one fan-out callback), not one-or-more per waiter.""" + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + for _ in range(3): # repeated retry rounds, as in the admission loop + waiters = [asyncio.create_task(wait_on_shared_future(shared, timeout=0.01)) for _ in range(200)] + results = await asyncio.gather(*waiters, return_exceptions=True) + assert all(isinstance(r, TimeoutError) for r in results) + count = _callback_count(shared) + if count is not None: + assert count == 1 + assert getattr(shared, _WAITERS_ATTR) == set() + assert not shared.done() + shared.cancel() + + +async def test_cancelling_one_waiter_leaves_others_and_shared_intact(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + victim = asyncio.create_task(wait_on_shared_future(shared, timeout=5)) + survivor = asyncio.create_task(wait_on_shared_future(shared, timeout=5)) + await asyncio.sleep(0) + victim.cancel() + with pytest.raises(asyncio.CancelledError): + await victim + assert not shared.done() + shared.set_result("session") + assert await survivor == "session" + + +async def test_done_shared_returns_immediately(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + shared.set_result("cached") + assert await wait_on_shared_future(shared, timeout=0.01) == "cached" + + failed: asyncio.Future[str] = asyncio.get_running_loop().create_future() + failed.set_exception(RuntimeError("boom")) + with pytest.raises(RuntimeError, match="boom"): + await wait_on_shared_future(failed) + + +async def test_late_waiter_after_fan_out_gets_result(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + first = asyncio.create_task(wait_on_shared_future(shared, timeout=5)) + await asyncio.sleep(0) + shared.set_result("session") + assert await first == "session" + # Fan-out already ran and cleared the waiter set; a late waiter must not + # hang on the emptied set. + assert await wait_on_shared_future(shared, timeout=0.01) == "session" + + +async def test_shared_task_keeps_running_when_all_waiters_cancel(): + """Singleflight semantics: waiter cancellation must not abort the work.""" + finished = asyncio.Event() + + async def _work() -> str: + await asyncio.sleep(0.05) + finished.set() + return "refreshed" + + task = asyncio.create_task(_work()) + waiter = asyncio.create_task(wait_on_shared_future(task)) + await asyncio.sleep(0) + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + assert await task == "refreshed" + assert finished.is_set() diff --git a/tests/unit/test_sse.py b/tests/unit/test_sse.py index 69a1e97a53..5151f48025 100644 --- a/tests/unit/test_sse.py +++ b/tests/unit/test_sse.py @@ -6,16 +6,21 @@ from typing import Any, cast import pytest +from hypothesis import given, settings +from hypothesis import strategies as st -from app.core.openai.parsing import parse_sse_event +from app.core.openai.parsing import _LIFECYCLE_EVENT_TYPES, classify_event_type, parse_sse_event from app.core.utils.sse import ( CODEX_KEEPALIVE_FRAME, SSE_KEEPALIVE_FRAME, extract_sse_data, + format_sse_data, format_sse_event, inject_sse_keepalives, parse_sse_data_json, + sse_event_type_from_block, ) +from tests.unit.hypothesis_strategies import json_objects, json_values pytestmark = pytest.mark.unit @@ -26,6 +31,58 @@ def test_format_sse_event_serializes_payload(): assert result == 'event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_1"}}\n\n' +@given(payload=json_objects) +@settings(max_examples=40, deadline=None) +def test_format_sse_event_round_trips_arbitrary_json_objects(payload): + assert parse_sse_data_json(format_sse_event(payload)) == payload + + +@given(payload=json_objects) +@settings(max_examples=40, deadline=None) +def test_format_sse_data_round_trips_arbitrary_json_objects(payload): + assert parse_sse_data_json(format_sse_data(payload)) == payload + + +@given( + boundary=st.sampled_from(["\r", "\n", "\r\n"]), + key=st.text(max_size=40), + value=st.integers(), +) +@settings(max_examples=30, deadline=None) +def test_sse_line_boundaries_are_equivalent_in_multiline_data(boundary, key, value): + encoded_key = json.dumps(key, ensure_ascii=True) + block = f"data: {{{encoded_key}:" + boundary + f"data: {value}}}" + boundary * 2 + + assert parse_sse_data_json(block) == {key: value} + + +@given(text=st.text(max_size=80)) +@settings(max_examples=30, deadline=None) +def test_sse_unicode_line_separators_remain_data(text): + payload = {"value": f"before{text}\u2028middle\u2029after"} + block = "data: " + json.dumps(payload, ensure_ascii=False) + "\n\n" + + assert parse_sse_data_json(block) == payload + + +@given( + boundary=st.sampled_from(["\r", "\n", "\r\n"]), + first=st.text(alphabet=st.characters(blacklist_categories=("C", "Z")), min_size=1, max_size=40), + second=st.text(alphabet=st.characters(blacklist_categories=("C", "Z")), min_size=1, max_size=40), +) +@settings(max_examples=30, deadline=None) +def test_sse_multiline_data_ignores_comments_and_joins_with_newline(boundary, first, second): + block = f": comment{boundary}data: {first}{boundary}event: ignored{boundary}data: {second}{boundary}{boundary}" + + assert extract_sse_data(block) == f"{first}\n{second}" + + +@given(value=st.one_of(st.none(), st.booleans(), st.integers(), st.lists(json_values, max_size=4))) +@settings(max_examples=30, deadline=None) +def test_parse_sse_data_json_rejects_non_object_json(value): + assert parse_sse_data_json("data: " + json.dumps(value) + "\n\n") is None + + async def _agen(items: list[str]) -> AsyncIterator[str]: for item in items: yield item @@ -142,3 +199,67 @@ def test_extract_sse_data_joins_crlf_multiline_data(): block = "data: line1\r\ndata: line2\rdata: line3\n\n" assert extract_sse_data(block) == "line1\nline2\nline3" + + +def test_classify_event_type_prefers_string_type_field(): + assert classify_event_type({"type": "response.output_text.delta", "delta": "x"}) == "response.output_text.delta" + + +def test_classify_event_type_maps_typeless_error_payload_to_error(): + assert classify_event_type({"error": {"message": "boom"}, "status": 400}) == "error" + + +def test_classify_event_type_rejects_non_dict_and_typeless_payloads(): + assert classify_event_type(None) is None + assert classify_event_type([1, 2, 3]) is None + assert classify_event_type({"type": 42}) is None + assert classify_event_type({"delta": "x"}) is None + + +def test_lifecycle_event_types_cover_terminal_and_created_frames(): + assert _LIFECYCLE_EVENT_TYPES == frozenset( + { + "response.created", + "response.completed", + "response.incomplete", + "response.failed", + "error", + } + ) + + +def test_sse_event_type_from_block_extracts_type_from_canonical_block(): + block = 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + assert sse_event_type_from_block(block) == "response.output_text.delta" + + +def test_sse_event_type_from_block_accepts_raw_utf8_payloads(): + block = 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"안녕"}\n\n' + + assert sse_event_type_from_block(block) == "response.output_text.delta" + + +def test_sse_event_type_from_block_rejects_data_only_blocks(): + assert sse_event_type_from_block('data: {"type":"response.output_text.delta","delta":"hi"}\n\n') is None + + +def test_sse_event_type_from_block_rejects_trailing_event_field_ordering(): + # `event:` after `data:` is legal SSE but not the canonical framing this + # proxy relays verbatim; callers must fall back to a full parse. + block = 'data: {"type":"response.output_text.delta","delta":"hi"}\nevent: response.output_text.delta\n\n' + + assert sse_event_type_from_block(block) is None + + +def test_sse_event_type_from_block_rejects_non_lf_framing_and_multiline_data(): + crlf = 'event: response.output_text.delta\r\ndata: {"type":"response.output_text.delta"}\r\n\r\n' + multiline = 'event: response.output_text.delta\ndata: {"type":\ndata: "response.output_text.delta"}\n\n' + + assert sse_event_type_from_block(crlf) is None + assert sse_event_type_from_block(multiline) is None + + +def test_sse_event_type_from_block_rejects_non_object_data_payloads(): + assert sse_event_type_from_block("event: done\ndata: [DONE]\n\n") is None + assert sse_event_type_from_block("event: ping\ndata: \n\n") is None diff --git a/tests/unit/test_sticky_session_cleanup_scheduler.py b/tests/unit/test_sticky_session_cleanup_scheduler.py index 7d04a8ff63..ae904eee52 100644 --- a/tests/unit/test_sticky_session_cleanup_scheduler.py +++ b/tests/unit/test_sticky_session_cleanup_scheduler.py @@ -53,6 +53,7 @@ async def test_cleanup_once_purges_prompt_cache_only(monkeypatch) -> None: lambda: SimpleNamespace( http_responses_session_bridge_idle_ttl_seconds=120.0, http_responses_session_bridge_codex_idle_ttl_seconds=900.0, + http_responses_session_bridge_operation_spool_retention_seconds=604800.0, ), ) @@ -64,8 +65,7 @@ async def test_cleanup_once_purges_prompt_cache_only(monkeypatch) -> None: bridge_repo.purge_closed_before = AsyncMock(return_value=2) bridge_repo.purge_abandoned_before = AsyncMock(return_value=1) bridge_repo.purge_retry_circuits_before = AsyncMock(return_value=3) - rowless_repo = AsyncMock() - rowless_repo.purge_expired_audit_rows = AsyncMock(return_value={"captured": 0}) + bridge_repo.purge_operation_spool = AsyncMock(return_value=0) ring_service = AsyncMock() ring_service.purge_stale_before = AsyncMock(return_value=0) @@ -86,7 +86,6 @@ async def __aexit__(self, *args): patch.object(cleanup_scheduler, "SettingsRepository", return_value=settings_repo), patch.object(cleanup_scheduler, "StickySessionsRepository", return_value=sticky_repo), patch.object(cleanup_scheduler, "DurableBridgeRepository", return_value=bridge_repo), - patch.object(cleanup_scheduler, "RowlessRecoveryRepository", return_value=rowless_repo), patch.object(cleanup_scheduler, "RingMembershipService", return_value=ring_service), patch.object(cleanup_scheduler, "_get_leader_election", lambda: _FakeLeader()), patch.object(cleanup_scheduler.startup_module, "_bridge_durable_schema_ready", True), @@ -98,7 +97,7 @@ async def __aexit__(self, *args): bridge_repo.purge_closed_before.assert_called_once() bridge_repo.purge_abandoned_before.assert_called_once() bridge_repo.purge_retry_circuits_before.assert_called_once() - rowless_repo.purge_expired_audit_rows.assert_called_once() + bridge_repo.purge_operation_spool.assert_called_once() ring_service.purge_stale_before.assert_called_once() sticky_repo.purge_stale_hard_codex_session_mappings.assert_called_once() passed_cutoff = sticky_repo.purge_stale_hard_codex_session_mappings.call_args.args[0] @@ -121,6 +120,7 @@ async def test_cleanup_once_skips_bridge_purge_when_schema_is_not_ready(monkeypa lambda: SimpleNamespace( http_responses_session_bridge_idle_ttl_seconds=120.0, http_responses_session_bridge_codex_idle_ttl_seconds=900.0, + http_responses_session_bridge_operation_spool_retention_seconds=604800.0, ), ) @@ -131,8 +131,7 @@ async def test_cleanup_once_skips_bridge_purge_when_schema_is_not_ready(monkeypa bridge_repo.purge_closed_before = AsyncMock(return_value=0) bridge_repo.purge_abandoned_before = AsyncMock(return_value=0) bridge_repo.purge_retry_circuits_before = AsyncMock(return_value=0) - rowless_repo = AsyncMock() - rowless_repo.purge_expired_audit_rows = AsyncMock(return_value={"captured": 0}) + bridge_repo.purge_operation_spool = AsyncMock(return_value=0) ring_service = AsyncMock() ring_service.purge_stale_before = AsyncMock(return_value=0) @@ -153,7 +152,6 @@ async def __aexit__(self, *args): patch.object(cleanup_scheduler, "SettingsRepository", return_value=settings_repo), patch.object(cleanup_scheduler, "StickySessionsRepository", return_value=sticky_repo), patch.object(cleanup_scheduler, "DurableBridgeRepository", return_value=bridge_repo), - patch.object(cleanup_scheduler, "RowlessRecoveryRepository", return_value=rowless_repo), patch.object(cleanup_scheduler, "RingMembershipService", return_value=ring_service), patch.object(cleanup_scheduler, "_get_leader_election", lambda: _FakeLeader()), patch.object(cleanup_scheduler.startup_module, "_bridge_durable_schema_ready", False), @@ -169,7 +167,6 @@ async def __aexit__(self, *args): bridge_repo.purge_closed_before.assert_not_called() bridge_repo.purge_abandoned_before.assert_not_called() bridge_repo.purge_retry_circuits_before.assert_not_called() - rowless_repo.purge_expired_audit_rows.assert_not_called() ring_service.purge_stale_before.assert_called_once() @@ -188,6 +185,7 @@ async def test_cleanup_once_purges_bridge_when_schema_exists_after_startup_flag_ lambda: SimpleNamespace( http_responses_session_bridge_idle_ttl_seconds=120.0, http_responses_session_bridge_codex_idle_ttl_seconds=900.0, + http_responses_session_bridge_operation_spool_retention_seconds=604800.0, ), ) @@ -198,8 +196,7 @@ async def test_cleanup_once_purges_bridge_when_schema_exists_after_startup_flag_ bridge_repo.purge_closed_before = AsyncMock(return_value=1) bridge_repo.purge_abandoned_before = AsyncMock(return_value=0) bridge_repo.purge_retry_circuits_before = AsyncMock(return_value=0) - rowless_repo = AsyncMock() - rowless_repo.purge_expired_audit_rows = AsyncMock(return_value={"captured": 0}) + bridge_repo.purge_operation_spool = AsyncMock(return_value=0) ring_service = AsyncMock() ring_service.purge_stale_before = AsyncMock(return_value=2) @@ -220,7 +217,6 @@ async def __aexit__(self, *args): patch.object(cleanup_scheduler, "SettingsRepository", return_value=settings_repo), patch.object(cleanup_scheduler, "StickySessionsRepository", return_value=sticky_repo), patch.object(cleanup_scheduler, "DurableBridgeRepository", return_value=bridge_repo), - patch.object(cleanup_scheduler, "RowlessRecoveryRepository", return_value=rowless_repo), patch.object(cleanup_scheduler, "RingMembershipService", return_value=ring_service), patch.object(cleanup_scheduler, "_get_leader_election", lambda: _FakeLeader()), patch.object(cleanup_scheduler.startup_module, "_bridge_durable_schema_ready", False), @@ -232,7 +228,7 @@ async def __aexit__(self, *args): bridge_repo.purge_closed_before.assert_called_once() bridge_repo.purge_abandoned_before.assert_called_once() bridge_repo.purge_retry_circuits_before.assert_called_once() - rowless_repo.purge_expired_audit_rows.assert_called_once() + bridge_repo.purge_operation_spool.assert_called_once() ring_service.purge_stale_before.assert_called_once() @@ -280,6 +276,7 @@ async def test_cleanup_once_gates_abandoned_purge_on_prompt_cache_reuse_ttl(monk lambda: SimpleNamespace( http_responses_session_bridge_idle_ttl_seconds=120.0, http_responses_session_bridge_codex_idle_ttl_seconds=900.0, + http_responses_session_bridge_operation_spool_retention_seconds=604800.0, ), ) @@ -290,8 +287,7 @@ async def test_cleanup_once_gates_abandoned_purge_on_prompt_cache_reuse_ttl(monk bridge_repo.purge_closed_before = AsyncMock(return_value=0) bridge_repo.purge_abandoned_before = AsyncMock(return_value=0) bridge_repo.purge_retry_circuits_before = AsyncMock(return_value=0) - rowless_repo = AsyncMock() - rowless_repo.purge_expired_audit_rows = AsyncMock(return_value={"captured": 0}) + bridge_repo.purge_operation_spool = AsyncMock(return_value=0) ring_service = AsyncMock() ring_service.purge_stale_before = AsyncMock(return_value=0) @@ -312,7 +308,6 @@ async def __aexit__(self, *args): patch.object(cleanup_scheduler, "SettingsRepository", return_value=settings_repo), patch.object(cleanup_scheduler, "StickySessionsRepository", return_value=sticky_repo), patch.object(cleanup_scheduler, "DurableBridgeRepository", return_value=bridge_repo), - patch.object(cleanup_scheduler, "RowlessRecoveryRepository", return_value=rowless_repo), patch.object(cleanup_scheduler, "RingMembershipService", return_value=ring_service), patch.object(cleanup_scheduler, "_get_leader_election", lambda: _FakeLeader()), patch.object(cleanup_scheduler.startup_module, "_bridge_durable_schema_ready", True), @@ -325,3 +320,40 @@ async def __aexit__(self, *args): # must be retained for the full 3600s prompt-cache reuse window. gap_seconds = (closed_cutoff - abandoned_cutoff).total_seconds() assert abs(gap_seconds - 1800.0) < 5.0 + + +@pytest.mark.asyncio +async def test_cleanup_once_retains_operation_purge_when_sticky_cleanup_disabled(monkeypatch) -> None: + settings_repo = AsyncMock() + sticky_repo = AsyncMock() + bridge_repo = AsyncMock() + bridge_repo.purge_operation_spool = AsyncMock(return_value=0) + + class FakeSession: + async def __aenter__(self): + return AsyncMock() + + async def __aexit__(self, *args): + pass + + monkeypatch.setattr( + cleanup_scheduler, + "get_settings", + lambda: SimpleNamespace(http_responses_session_bridge_operation_spool_retention_seconds=604800.0), + ) + scheduler = cleanup_scheduler.StickySessionCleanupScheduler(interval_seconds=60, enabled=False) + + with ( + patch.object(cleanup_scheduler, "get_background_session", FakeSession), + patch.object(cleanup_scheduler, "SettingsRepository", return_value=settings_repo), + patch.object(cleanup_scheduler, "StickySessionsRepository", return_value=sticky_repo), + patch.object(cleanup_scheduler, "DurableBridgeRepository", return_value=bridge_repo), + patch.object(cleanup_scheduler, "_get_leader_election", lambda: _FakeLeader()), + patch.object(cleanup_scheduler.startup_module, "_bridge_durable_schema_ready", True), + ): + await scheduler._cleanup_once() + + settings_repo.get_or_create.assert_not_awaited() + sticky_repo.purge_prompt_cache_before.assert_not_awaited() + bridge_repo.purge_closed_before.assert_not_awaited() + bridge_repo.purge_operation_spool.assert_awaited_once() diff --git a/tests/unit/test_structured_logging.py b/tests/unit/test_structured_logging.py index 9408aa20fb..b0ed706634 100644 --- a/tests/unit/test_structured_logging.py +++ b/tests/unit/test_structured_logging.py @@ -11,6 +11,7 @@ _error_log_field, _redact_log_value, build_log_config, + safe_log_field, ) pytestmark = pytest.mark.unit @@ -40,6 +41,10 @@ def test_error_log_field_quotes_redacted_field_values(): assert field == '"temporary failure status=200 request_id=req-1 api_key=[REDACTED]"' +def test_safe_log_field_is_single_line_and_redacts_secrets(): + assert safe_log_field("user\r\npassword=secret-token") == "user password=[REDACTED]" + + @pytest.mark.parametrize( "value, expected", [ diff --git a/tests/unit/test_sync_codex_ok_labels.py b/tests/unit/test_sync_codex_ok_labels.py deleted file mode 100644 index de6cc200a6..0000000000 --- a/tests/unit/test_sync_codex_ok_labels.py +++ /dev/null @@ -1,2106 +0,0 @@ -from __future__ import annotations - -import importlib.util -import sys -from pathlib import Path -from types import ModuleType -from typing import Any - -import pytest - - -def load_sync_module() -> ModuleType: - script_path = Path(__file__).resolve().parents[2] / ".github" / "scripts" / "sync_codex_ok_labels.py" - spec = importlib.util.spec_from_file_location("sync_codex_ok_labels", script_path) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -def decision(module: ModuleType, **overrides: Any) -> Any: - values = { - "repo": "Soju06/codex-lb", - "number": 714, - "head_sha": "a" * 40, - "has_ok_label": True, - "wants_ok_label": False, - "ok_action": "remove", - "has_needs_work_label": False, - "wants_needs_work_label": False, - "needs_work_action": "keep", - "has_needs_rebase_label": False, - "wants_needs_rebase_label": False, - "needs_rebase_action": "keep", - "legacy_labels": frozenset(), - "reason": "checks are pending", - "review_url": None, - "review_state": "clean", - "checks_state": "pending", - "merge_state": "CLEAN", - "trigger_codex_review": False, - "approve_workflow_run_ids": (), - } - values.update(overrides) - return module.SyncDecision(**values) - - -def codex_review_request(author: str, created_at: str) -> dict[str, Any]: - return { - "__typename": "IssueComment", - "author": {"login": author}, - "bodyText": "@codex review", - "createdAt": created_at, - "url": f"https://github.test/request/{created_at}", - } - - -def codex_issue_comment(body: str, created_at: str) -> dict[str, Any]: - return { - "__typename": "IssueComment", - "author": {"login": "chatgpt-codex-connector"}, - "bodyText": body, - "createdAt": created_at, - "url": f"https://github.test/codex/{created_at}", - } - - -@pytest.mark.parametrize("merge_state", ["CONFLICTING", "DIRTY"]) -def test_needs_rebase_label_target_adds_for_confirmed_conflicts(merge_state: str) -> None: - module = load_sync_module() - - assert module.needs_rebase_label_target(merge_state, has_label=False) is True - - -@pytest.mark.parametrize("merge_state", ["BEHIND", "BLOCKED", "CLEAN", "DRAFT", "HAS_HOOKS", "UNSTABLE"]) -def test_needs_rebase_label_target_removes_for_known_non_conflict_states(merge_state: str) -> None: - module = load_sync_module() - - assert module.needs_rebase_label_target(merge_state, has_label=True) is False - - -@pytest.mark.parametrize("has_label", [False, True]) -def test_needs_rebase_label_target_preserves_unknown_state(has_label: bool) -> None: - module = load_sync_module() - - assert module.needs_rebase_label_target("UNKNOWN", has_label=has_label) is has_label - - -def test_apply_decision_adds_needs_rebase_label(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - calls: list[tuple[str, str, Any | None]] = [] - - def capture_write(path: str, *, method: str = "GET", input_json: Any | None = None) -> None: - calls.append((method, path, input_json)) - - monkeypatch.setattr(module, "gh_api", capture_write) - - warnings = module.apply_decision( - decision( - module, - ok_action="keep", - has_needs_rebase_label=False, - wants_needs_rebase_label=True, - needs_rebase_action="add", - ) - ) - - assert warnings == () - assert calls == [ - ( - "POST", - "/repos/Soju06/codex-lb/issues/714/labels", - {"labels": ["needs rebase"]}, - ) - ] - - -def test_apply_decision_removes_stale_needs_rebase_label(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - calls: list[tuple[str, str, Any | None]] = [] - - def capture_write(path: str, *, method: str = "GET", input_json: Any | None = None) -> None: - calls.append((method, path, input_json)) - - monkeypatch.setattr(module, "gh_api", capture_write) - - warnings = module.apply_decision( - decision( - module, - ok_action="keep", - has_needs_rebase_label=True, - wants_needs_rebase_label=False, - needs_rebase_action="remove", - ) - ) - - assert warnings == () - assert calls == [ - ( - "DELETE", - "/repos/Soju06/codex-lb/issues/714/labels/needs%20rebase", - None, - ) - ] - - -def test_classify_check_state_uses_latest_run_for_duplicate_check_names() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "CI Required", - "status": "completed", - "conclusion": "failure", - "completed_at": "2026-06-11T07:40:59Z", - }, - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "completed_at": "2026-06-11T07:45:35Z", - }, - { - "name": "Type check (ty)", - "status": "completed", - "conclusion": "success", - "completed_at": "2026-06-11T07:41:20Z", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required", "Type check (ty)"}), - ) - == "success" - ) - - -def test_classify_check_state_keeps_latest_pending_duplicate_pending() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "completed_at": "2026-06-11T07:40:59Z", - }, - { - "name": "CI Required", - "status": "in_progress", - "conclusion": None, - "started_at": "2026-06-11T07:45:35Z", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "pending" - ) - - -def test_classify_check_state_ignores_stale_duplicate_that_finishes_late() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "CI Required", - "status": "completed", - "conclusion": "failure", - "started_at": "2026-06-11T07:40:59Z", - "completed_at": "2026-06-11T07:50:00Z", - }, - { - "name": "CI Required", - "status": "in_progress", - "conclusion": None, - "started_at": "2026-06-11T07:45:35Z", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "pending" - ) - - -def test_classify_check_state_ignores_unique_failure_from_superseded_ci_run() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "Tests (pytest, ${{ matrix.slice.name }})", - "status": "completed", - "conclusion": "failure", - "started_at": "2026-07-10T06:00:37Z", - "completed_at": "2026-07-10T06:00:37Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/100/job/1", - "_github_actions_workflow_id": "ci", - }, - { - "name": "CI Required", - "status": "completed", - "conclusion": "failure", - "started_at": "2026-07-10T06:00:38Z", - "completed_at": "2026-07-10T06:00:41Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/100/job/2", - "_github_actions_workflow_id": "ci", - }, - { - "name": "Tests (pytest, unit)", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:01:00Z", - "completed_at": "2026-07-10T06:05:00Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/3", - "_github_actions_workflow_id": "ci", - }, - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:09:01Z", - "completed_at": "2026-07-10T06:09:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - "_github_actions_workflow_id": "ci", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required", "Tests (pytest, unit)"}), - ) - == "success" - ) - - -def test_classify_check_state_keeps_optional_failure_from_authoritative_ci_run() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "optional security scan", - "status": "completed", - "conclusion": "failure", - "started_at": "2026-07-10T06:09:00Z", - "completed_at": "2026-07-10T06:09:04Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/3", - "_github_actions_workflow_id": "ci", - }, - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:09:01Z", - "completed_at": "2026-07-10T06:09:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - "_github_actions_workflow_id": "ci", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "failure" - ) - - -def test_classify_check_state_keeps_newer_same_workflow_run_pending_before_required_job_exists() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:09:01Z", - "completed_at": "2026-07-10T06:09:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - "_github_actions_workflow_id": "ci", - "_github_actions_run_created_at": "2026-07-10T06:00:43Z", - }, - { - "name": "Detect changes", - "status": "in_progress", - "conclusion": None, - "started_at": "2026-07-10T06:50:20Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/300/job/1", - "_github_actions_workflow_id": "ci", - "_github_actions_run_created_at": "2026-07-10T06:50:20Z", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "pending" - ) - - -def test_classify_check_state_keeps_manual_rerun_of_older_run_pending() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:50:20Z", - "completed_at": "2026-07-10T06:59:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - "_github_actions_workflow_id": "ci", - "_github_actions_run_created_at": "2026-07-10T06:50:00Z", - "_github_actions_run_started_at": "2026-07-10T06:50:00Z", - }, - { - "name": "Detect changes", - "status": "in_progress", - "conclusion": None, - "started_at": "2026-07-10T07:10:20Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/100/job/1", - "_github_actions_workflow_id": "ci", - "_github_actions_run_created_at": "2026-07-10T06:00:00Z", - "_github_actions_run_started_at": "2026-07-10T07:10:00Z", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "pending" - ) - - -def test_classify_check_state_keeps_failure_from_manual_rerun_of_older_run() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:50:20Z", - "completed_at": "2026-07-10T06:59:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - "_github_actions_workflow_id": "ci", - "_github_actions_run_created_at": "2026-07-10T06:50:00Z", - "_github_actions_run_started_at": "2026-07-10T06:50:00Z", - }, - { - "name": "CI Required", - "status": "completed", - "conclusion": "failure", - "started_at": "2026-07-10T07:10:20Z", - "completed_at": "2026-07-10T07:15:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/100/job/4", - "_github_actions_workflow_id": "ci", - "_github_actions_run_created_at": "2026-07-10T06:00:00Z", - "_github_actions_run_started_at": "2026-07-10T07:10:00Z", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "failure" - ) - - -def test_classify_check_state_keeps_failure_from_independent_workflow_run() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "independent security scan", - "status": "completed", - "conclusion": "failure", - "started_at": "2026-07-10T06:08:00Z", - "completed_at": "2026-07-10T06:08:30Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/300/job/1", - "_github_actions_workflow_id": "security", - }, - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:09:01Z", - "completed_at": "2026-07-10T06:09:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - "_github_actions_workflow_id": "ci", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "failure" - ) - - -def test_annotate_github_actions_workflow_ids_is_conservative_when_metadata_lookup_fails( - monkeypatch: pytest.MonkeyPatch, -) -> None: - module = load_sync_module() - check_runs = [ - { - "name": "CI Required", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - }, - { - "name": "independent scan", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/300/job/1", - }, - ] - - def workflow_run(path: str) -> dict[str, int | str]: - if path.endswith("/200"): - return { - "workflow_id": 10, - "created_at": "2026-07-10T06:00:43Z", - "run_started_at": "2026-07-10T07:10:00Z", - } - raise module.GhError("metadata unavailable") - - monkeypatch.setattr(module, "gh_api", workflow_run) - - annotated = module.annotate_github_actions_workflow_ids("Soju06/codex-lb", check_runs) - - assert annotated[0]["_github_actions_workflow_id"] == "10" - assert annotated[0]["_github_actions_run_started_at"] == "2026-07-10T07:10:00Z" - assert "_github_actions_workflow_id" not in annotated[1] - - -def test_apply_decision_tolerates_github_app_write_denial(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - def deny_write(*_args: Any, **_kwargs: Any) -> None: - raise module.GhError("gh: Resource not accessible by integration (HTTP 403)") - - monkeypatch.setattr(module, "gh_api", deny_write) - - warnings = module.apply_decision(decision(module), tolerate_permission_errors=True) - - assert len(warnings) == 1 - assert "remove 🤖 codex: ok from Soju06/codex-lb#714" in warnings[0] - assert "Resource not accessible by integration" in warnings[0] - - -def test_apply_decision_still_fails_on_write_denial_without_tolerance(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - def deny_write(*_args: Any, **_kwargs: Any) -> None: - raise module.GhError("gh: Resource not accessible by integration (HTTP 403)") - - monkeypatch.setattr(module, "gh_api", deny_write) - - with pytest.raises(module.GhError): - module.apply_decision(decision(module), tolerate_permission_errors=False) - - -def test_apply_decision_treats_missing_label_delete_as_done(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - calls: list[tuple[str, str]] = [] - - def missing_label(path: str, *, method: str = "GET", **_kwargs: Any) -> None: - calls.append((method, path)) - raise module.GhError("gh: Label does not exist (HTTP 404)") - - monkeypatch.setattr(module, "gh_api", missing_label) - - warnings = module.apply_decision(decision(module), tolerate_permission_errors=False) - - assert warnings == () - assert calls == [ - ( - "DELETE", - "/repos/Soju06/codex-lb/issues/714/labels/%F0%9F%A4%96%20codex%3A%20ok", - ) - ] - - -def test_apply_decision_does_not_swallow_unrelated_delete_404(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - def missing_resource(*_args: Any, **_kwargs: Any) -> None: - raise module.GhError("gh: Not Found (HTTP 404)") - - monkeypatch.setattr(module, "gh_api", missing_resource) - - with pytest.raises(module.GhError): - module.apply_decision(decision(module), tolerate_permission_errors=False) - - -def test_trigger_codex_review_tolerates_github_app_write_denial(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - def deny_write(*_args: Any, **_kwargs: Any) -> None: - raise module.GhError("gh: Resource not accessible by integration (HTTP 403)") - - monkeypatch.setattr(module, "run_gh", deny_write) - request_review = decision(module, trigger_codex_review=True, ok_action="keep") - - warnings = module.trigger_codex_review( - request_review, - body="@codex review", - tolerate_permission_errors=True, - ) - - assert len(warnings) == 1 - assert "request Codex review on Soju06/codex-lb#714" in warnings[0] - - -def test_codex_usage_backoff_blocks_recent_limit_for_same_sender() -> None: - module = load_sync_module() - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - - backoff.observe( - [ - codex_review_request("Komzpa", "2026-07-31T15:00:00Z"), - codex_issue_comment("You've reached your Codex usage limits.", "2026-07-31T15:01:00Z"), - ] - ) - - assert backoff.is_limited() is True - - -def test_codex_usage_backoff_allows_after_newer_normal_reply() -> None: - module = load_sync_module() - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - - backoff.observe( - [ - codex_review_request("Komzpa", "2026-07-31T14:00:00Z"), - codex_issue_comment("You've reached your Codex usage limits.", "2026-07-31T14:01:00Z"), - codex_review_request("Komzpa", "2026-07-31T15:00:00Z"), - codex_issue_comment("Codex Review: Didn't find any major issues.", "2026-07-31T15:02:00Z"), - ] - ) - - assert backoff.is_limited() is False - - -def test_codex_usage_backoff_keeps_accounts_independent() -> None: - module = load_sync_module() - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - - backoff.observe( - [ - codex_review_request("Komzpa", "2026-07-31T14:00:00Z"), - codex_issue_comment("You've reached your Codex usage limits.", "2026-07-31T14:01:00Z"), - codex_review_request("OtherUser", "2026-07-31T15:00:00Z"), - codex_issue_comment("Codex Review: Didn't find any major issues.", "2026-07-31T15:02:00Z"), - ] - ) - - assert backoff.is_limited() is True - - -@pytest.mark.parametrize( - "body", - [ - "You have reached your Codex usage limits for code reviews. " - "You can see your limits in the [Codex usage dashboard](https://chatgpt.com/codex/settings/usage).", - " \nYou have reached your Codex usage limits for code reviews.", - "You've reached your Codex usage limits.", - ], -) -def test_usage_limit_body_matches_real_quota_envelope(body: str) -> None: - module = load_sync_module() - - assert module.is_codex_usage_limit_body(body) is True - - -@pytest.mark.parametrize( - "body", - [ - "**[P1]** The unanchored `usage limit` pattern also matches reviews discussing usage limits.", - "Codex Review: the backoff should latch on a Codex usage limit reply. Didn't find any major issues.", - "This PR adds a usage-limit backoff. You have reached your Codex usage limits is the trigger phrase.", - None, - "", - ], -) -def test_usage_limit_body_ignores_reviews_discussing_usage_limits(body: object) -> None: - module = load_sync_module() - - assert module.is_codex_usage_limit_body(body) is False - - -def codex_review_request_with_reaction( - author: str, - created_at: str, - *, - reaction_user: str, - reaction_content: str, - reaction_created_at: str, -) -> dict[str, Any]: - request = codex_review_request(author, created_at) - request["reactions"] = { - "nodes": [ - { - "content": reaction_content, - "createdAt": reaction_created_at, - "user": {"login": reaction_user}, - } - ] - } - return request - - -def test_codex_usage_backoff_unlatches_on_newer_clean_reaction() -> None: - module = load_sync_module() - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - - backoff.observe( - [ - codex_review_request("Komzpa", "2026-07-31T14:00:00Z"), - codex_issue_comment("You've reached your Codex usage limits.", "2026-07-31T14:01:00Z"), - codex_review_request_with_reaction( - "Komzpa", - "2026-07-31T15:00:00Z", - reaction_user="chatgpt-codex-connector", - reaction_content="THUMBS_UP", - reaction_created_at="2026-07-31T15:05:00Z", - ), - ] - ) - - assert backoff.is_limited() is False - - -def test_codex_usage_backoff_ignores_reactions_from_non_codex_users() -> None: - module = load_sync_module() - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - - backoff.observe( - [ - codex_review_request("Komzpa", "2026-07-31T14:00:00Z"), - codex_issue_comment("You've reached your Codex usage limits.", "2026-07-31T14:01:00Z"), - codex_review_request_with_reaction( - "Komzpa", - "2026-07-31T15:00:00Z", - reaction_user="SomeoneElse", - reaction_content="THUMBS_UP", - reaction_created_at="2026-07-31T15:05:00Z", - ), - ] - ) - - assert backoff.is_limited() is True - - -def test_codex_usage_backoff_ignores_clean_reaction_older_than_limit() -> None: - module = load_sync_module() - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - - backoff.observe( - [ - codex_review_request_with_reaction( - "Komzpa", - "2026-07-31T13:00:00Z", - reaction_user="chatgpt-codex-connector", - reaction_content="THUMBS_UP", - reaction_created_at="2026-07-31T13:05:00Z", - ), - codex_review_request("Komzpa", "2026-07-31T14:00:00Z"), - codex_issue_comment("You've reached your Codex usage limits.", "2026-07-31T14:01:00Z"), - ] - ) - - assert backoff.is_limited() is True - - -def test_resolve_codex_request_sender_prefers_app_slug(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_APP_SLUG", "codex-label-sync") - - def fail_viewer_login() -> str: - raise AssertionError("GET /user must not be called when GH_APP_SLUG is set") - - monkeypatch.setattr(module, "current_viewer_login", fail_viewer_login) - - assert module.resolve_codex_request_sender() == "codex-label-sync[bot]" - - -def test_resolve_codex_request_sender_keeps_explicit_bot_suffix(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_APP_SLUG", "codex-label-sync[bot]") - - assert module.resolve_codex_request_sender() == "codex-label-sync[bot]" - - -def test_resolve_codex_request_sender_falls_back_to_viewer_login(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - - assert module.resolve_codex_request_sender() == "Komzpa" - - -def test_resolve_codex_request_sender_returns_none_when_unresolvable( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - monkeypatch.delenv("GH_APP_SLUG", raising=False) - - def fail_viewer_login() -> str: - raise module.GhError("gh api /user: HTTP 403 (installation token)") - - monkeypatch.setattr(module, "current_viewer_login", fail_viewer_login) - - assert module.resolve_codex_request_sender() is None - assert "cannot resolve @codex review sender" in capsys.readouterr().err - - -def recent_timestamp(module: ModuleType, *, minutes_ago: int) -> str: - moment = module.datetime.now(module.UTC) - module.timedelta(minutes=minutes_ago) - return moment.strftime("%Y-%m-%dT%H:%M:%SZ") - - -def test_main_stops_codex_review_triggers_after_probe_hits_usage_limit( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - def fake_decide_pr(_repo: str, number: int, **kwargs: Any) -> Any: - observer = kwargs.get("timeline_observer") - if observer is not None: - observer( - number, - [ - { - "__typename": "PullRequestCommit", - "commit": {"oid": "a" * 40}, - "committedDate": recent_timestamp(module, minutes_ago=70), - } - ], - ) - return decision(module, number=number, trigger_codex_review=True, ok_action="keep", checks_state="success") - - posted: list[int] = [] - - def fake_trigger(decision: Any, **_kwargs: Any) -> tuple[str, ...]: - posted.append(decision.number) - return () - - def fake_timeline(_repo: str, _number: int) -> tuple[str, list[dict[str, Any]]]: - return ( - "a" * 40, - [ - codex_review_request("Komzpa", recent_timestamp(module, minutes_ago=2)), - codex_issue_comment( - "You've reached your Codex usage limits.", - recent_timestamp(module, minutes_ago=1), - ), - ], - ) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - monkeypatch.setattr(module, "trigger_codex_review", fake_trigger) - monkeypatch.setattr(module, "pr_timeline_evidence", fake_timeline) - - result = module.main( - [ - "--repo", - "Soju06/codex-lb", - "--all-open", - "--apply", - "--codex-review-response-wait-seconds", - "0", - ] - ) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [710] - apply_lines = [line for line in captured.out.splitlines() if line.startswith("apply ")] - assert len(apply_lines) == 2 - assert apply_lines[0].startswith("apply Soju06/codex-lb#710: ") - assert "trigger_codex=True" in apply_lines[0] - assert apply_lines[1].startswith("apply Soju06/codex-lb#714: ") - assert "trigger_codex=False" in apply_lines[1] - assert "recent Codex usage-limit reply" in captured.out - - -def test_main_skips_probe_after_normal_codex_response_observed( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - def fake_decide_pr(_repo: str, number: int, **kwargs: Any) -> Any: - observer = kwargs.get("timeline_observer") - if observer is not None: - observer( - number, - [ - codex_review_request("Komzpa", recent_timestamp(module, minutes_ago=30)), - codex_issue_comment( - "Codex Review: Didn't find any major issues.", - recent_timestamp(module, minutes_ago=29), - ), - ], - ) - return decision(module, number=number, trigger_codex_review=True, ok_action="keep", checks_state="success") - - posted: list[int] = [] - - def fake_trigger(decision: Any, **_kwargs: Any) -> tuple[str, ...]: - posted.append(decision.number) - return () - - probe_calls: list[int] = [] - - def fake_timeline(_repo: str, number: int) -> tuple[str, list[dict[str, Any]]]: - probe_calls.append(number) - return ("a" * 40, []) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - monkeypatch.setattr(module, "trigger_codex_review", fake_trigger) - monkeypatch.setattr(module, "pr_timeline_evidence", fake_timeline) - - result = module.main( - [ - "--repo", - "Soju06/codex-lb", - "--all-open", - "--apply", - "--codex-review-response-wait-seconds", - "0", - ] - ) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [710, 714] - assert probe_calls == [] - assert "apply Soju06/codex-lb#710: " in captured.out - assert "apply Soju06/codex-lb#714: " in captured.out - - -def test_main_continues_label_sync_when_sender_is_unresolvable( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - def fail_viewer_login() -> str: - raise module.GhError("gh api /user: HTTP 403 (installation token)") - - monkeypatch.setattr(module, "current_viewer_login", fail_viewer_login) - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr( - module, - "decide_pr", - lambda _repo, number, **_kwargs: decision( - module, number=number, trigger_codex_review=True, checks_state="success" - ), - ) - - applied: list[int] = [] - monkeypatch.setattr( - module, - "apply_decision", - lambda applied_decision, **_kwargs: (applied.append(applied_decision.number), ())[1], - ) - posted: list[int] = [] - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: (posted.append(request_decision.number), ())[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert applied == [710, 714] - assert posted == [] - assert "cannot determine @codex review sender; skipping review triggers" in captured.err - assert captured.out.count("sender could not be resolved") == 2 - assert captured.out.count("trigger_codex=False") == 2 - - -def test_main_skips_apply_when_head_moved_after_classification( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - calls_by_number: dict[int, int] = {} - - def fake_decide_pr(_repo: str, number: int, **_kwargs: Any) -> Any: - calls_by_number[number] = calls_by_number.get(number, 0) + 1 - # The classification pass sees head a...a; by the time #710 is - # re-classified in the apply loop its head has moved to b...b. - if number == 710 and calls_by_number[number] > 1: - return decision(module, number=number, head_sha="b" * 40) - return decision(module, number=number) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - applied: list[int] = [] - monkeypatch.setattr( - module, - "apply_decision", - lambda applied_decision, **_kwargs: (applied.append(applied_decision.number), ())[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert applied == [714] - assert calls_by_number == {710: 2, 714: 2} - assert "Soju06/codex-lb#710: head moved from" in captured.err - assert "skipping stale decision" in captured.err - assert "apply Soju06/codex-lb#710" not in captured.out - assert "apply Soju06/codex-lb#714" in captured.out - - -def test_main_applies_freshly_reclassified_decision_for_same_head( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [714]) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - calls: list[int] = [] - - def fake_decide_pr(_repo: str, number: int, **_kwargs: Any) -> Any: - calls.append(number) - if len(calls) == 1: - return decision( - module, - number=number, - has_ok_label=False, - wants_ok_label=True, - ok_action="add", - checks_state="success", - ) - # Same head, but by apply time Codex raised a new finding. - return decision( - module, - number=number, - has_ok_label=False, - wants_ok_label=False, - ok_action="keep", - wants_needs_work_label=True, - needs_work_action="add", - review_state="needs_work", - ) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - applied_actions: list[tuple[str, str]] = [] - monkeypatch.setattr( - module, - "apply_decision", - lambda applied_decision, **_kwargs: ( - applied_actions.append((applied_decision.ok_action, applied_decision.needs_work_action)), - (), - )[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert applied_actions == [("keep", "add")] - assert "review=needs_work" in captured.out - - -def test_main_usage_backoff_state_persists_across_repos( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - def fake_decide_pr(repo: str, number: int, **kwargs: Any) -> Any: - observer = kwargs.get("timeline_observer") - if observer is not None: - timeline: list[dict[str, Any]] = [ - { - "__typename": "PullRequestCommit", - "commit": {"oid": "a" * 40}, - "committedDate": recent_timestamp(module, minutes_ago=70), - } - ] - if repo == "Soju06/codex-lb": - timeline.extend( - [ - codex_review_request("Komzpa", recent_timestamp(module, minutes_ago=30)), - codex_issue_comment( - "You have reached your Codex usage limits for code reviews.", - recent_timestamp(module, minutes_ago=29), - ), - ] - ) - observer(number, timeline) - return decision(module, repo=repo, number=number, trigger_codex_review=True, checks_state="success") - - posted: list[str] = [] - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: (posted.append(request_decision.repo), ())[1], - ) - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - result = module.main( - [ - "--repo", - "Soju06/codex-lb", - "--repo", - "Soju06/other-repo", - "--all-open", - "--apply", - "--codex-review-response-wait-seconds", - "0", - ] - ) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [] - assert "apply Soju06/codex-lb#710" in captured.out - assert "apply Soju06/other-repo#710" in captured.out - assert "request Codex review on Soju06/other-repo#710: skipped" in captured.out - assert "recent Codex usage-limit reply" in captured.out - - -def test_main_usage_backoff_counts_evidence_from_non_triggering_repo( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - def fake_decide_pr(repo: str, number: int, **kwargs: Any) -> Any: - observer = kwargs.get("timeline_observer") - if observer is not None and repo == "Soju06/codex-lb": - # The first repo has quota evidence but no trigger of its own. - observer( - number, - [ - codex_review_request("Komzpa", recent_timestamp(module, minutes_ago=30)), - codex_issue_comment( - "You have reached your Codex usage limits for code reviews.", - recent_timestamp(module, minutes_ago=29), - ), - ], - ) - elif observer is not None: - observer(number, []) - trigger = repo == "Soju06/other-repo" - return decision( - module, - repo=repo, - number=number, - ok_action="keep", - has_ok_label=False, - trigger_codex_review=trigger, - checks_state="success", - ) - - posted: list[str] = [] - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: (posted.append(request_decision.repo), ())[1], - ) - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - result = module.main( - [ - "--repo", - "Soju06/codex-lb", - "--repo", - "Soju06/other-repo", - "--all-open", - "--apply", - "--codex-review-response-wait-seconds", - "0", - ] - ) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [] - assert "request Codex review on Soju06/other-repo#710: skipped" in captured.out - assert "recent Codex usage-limit reply" in captured.out - - -def test_main_stops_codex_review_triggers_after_fallback_token_activates( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setenv("GH_APP_SLUG", "codex-label-sync") - monkeypatch.setattr(module, "_fallback_token_active", True) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [714]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "fallback-user") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - monkeypatch.setattr( - module, - "decide_pr", - lambda _repo, number, **_kwargs: decision( - module, - number=number, - ok_action="keep", - has_ok_label=False, - trigger_codex_review=True, - checks_state="success", - ), - ) - - posted: list[int] = [] - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: (posted.append(request_decision.number), ())[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [] - assert "switched to GH_FALLBACK_TOKEN" in captured.out - assert "trigger_codex=False" in captured.out - - -def test_resolve_codex_request_sender_ignores_app_slug_after_fallback(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_APP_SLUG", "codex-label-sync") - monkeypatch.setattr(module, "_fallback_token_active", True) - monkeypatch.setattr(module, "current_viewer_login", lambda: "fallback-user") - - assert module.resolve_codex_request_sender() == "fallback-user" - - -def test_recent_issue_comment_timelines_groups_by_issue(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - comments = [ - { - "body": "@codex review", - "issue_url": "https://api.github.test/repos/Soju06/codex-lb/issues/700", - "created_at": "2026-07-31T14:00:00Z", - "html_url": "https://github.test/pull/700#issuecomment-1", - "user": {"login": "Komzpa"}, - }, - { - "body": "unrelated comment on another issue", - "issue_url": "https://api.github.test/repos/Soju06/codex-lb/issues/701", - "created_at": "2026-07-31T14:00:30Z", - "html_url": "https://github.test/pull/701#issuecomment-2", - "user": {"login": "someone"}, - }, - { - "body": "You have reached your Codex usage limits for code reviews.", - "issue_url": "https://api.github.test/repos/Soju06/codex-lb/issues/700", - "created_at": "2026-07-31T14:01:00Z", - "html_url": "https://github.test/pull/700#issuecomment-3", - "user": {"login": "chatgpt-codex-connector"}, - }, - ] - paths: list[str] = [] - - def fake_paged_api(path: str) -> list[dict[str, Any]]: - paths.append(path) - return comments - - monkeypatch.setattr(module, "paged_api", fake_paged_api) - - timelines = module.recent_issue_comment_timelines( - "Soju06/codex-lb", - since=module.datetime.fromisoformat("2026-07-30T16:00:00+00:00"), - ) - - assert len(paths) == 1 - assert paths[0].startswith("/repos/Soju06/codex-lb/issues/comments?since=2026-07-30T16") - assert len(timelines) == 2 - grouped = {timeline[0]["url"].split("#")[0]: timeline for timeline in timelines} - pr_700 = grouped["https://github.test/pull/700"] - assert [node["bodyText"] for node in pr_700] == [ - "@codex review", - "You have reached your Codex usage limits for code reviews.", - ] - assert all(node["__typename"] == "IssueComment" for node in pr_700) - - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - for timeline in timelines: - backoff.observe(timeline) - assert backoff.is_limited() is True - - -def test_main_gathers_repo_wide_quota_evidence_for_single_pr_run( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - monkeypatch.setattr( - module, - "decide_pr", - lambda _repo, number, **_kwargs: decision( - module, - number=number, - ok_action="keep", - has_ok_label=False, - trigger_codex_review=True, - checks_state="success", - ), - ) - # Quota evidence lives on another (already closed) PR of the repo. - monkeypatch.setattr( - module, - "recent_issue_comment_timelines", - lambda _repo, **_kwargs: [ - [ - codex_review_request("Komzpa", recent_timestamp(module, minutes_ago=30)), - codex_issue_comment( - "You have reached your Codex usage limits for code reviews.", - recent_timestamp(module, minutes_ago=29), - ), - ] - ], - ) - - posted: list[int] = [] - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: (posted.append(request_decision.number), ())[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--pr", "714", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [] - assert "request Codex review on Soju06/codex-lb#714: skipped" in captured.out - assert "recent Codex usage-limit reply" in captured.out - - -def test_main_observes_quota_evidence_from_apply_time_reclassification( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - calls_by_number: dict[int, int] = {} - - def fake_decide_pr(_repo: str, number: int, **kwargs: Any) -> Any: - calls_by_number[number] = calls_by_number.get(number, 0) + 1 - reclassification = calls_by_number[number] > 1 - observer = kwargs.get("timeline_observer") - if observer is not None: - timeline: list[dict[str, Any]] = [ - { - "__typename": "PullRequestCommit", - "commit": {"oid": "a" * 40}, - "committedDate": recent_timestamp(module, minutes_ago=70), - } - ] - if reclassification and number == 710: - # A quota reply arrived between bulk classification and apply. - timeline.extend( - [ - codex_review_request("Komzpa", recent_timestamp(module, minutes_ago=3)), - codex_issue_comment( - "You have reached your Codex usage limits for code reviews.", - recent_timestamp(module, minutes_ago=2), - ), - ] - ) - observer(number, timeline) - trigger = not (reclassification and number == 710) - return decision( - module, - number=number, - ok_action="keep", - has_ok_label=False, - trigger_codex_review=trigger, - checks_state="success", - ) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - posted: list[int] = [] - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: (posted.append(request_decision.number), ())[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [] - assert "request Codex review on Soju06/codex-lb#714: skipped" in captured.out - assert "recent Codex usage-limit reply" in captured.out - - -def test_main_tolerates_apply_time_reclassification_read_errors( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [714]) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - calls: list[int] = [] - - def fake_decide_pr(_repo: str, number: int, **_kwargs: Any) -> Any: - calls.append(number) - if len(calls) > 1: - raise module.GhError("gh: HTTP 502") - return decision(module, number=number) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - applied: list[int] = [] - monkeypatch.setattr( - module, - "apply_decision", - lambda applied_decision, **_kwargs: (applied.append(applied_decision.number), ())[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply", "--tolerate-read-errors"]) - - captured = capsys.readouterr() - assert result == 0 - assert applied == [] - assert "Soju06/codex-lb#714: apply-time reclassification failed" in captured.err - - -def test_main_fails_apply_time_reclassification_read_errors_without_tolerance( - monkeypatch: pytest.MonkeyPatch, -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [714]) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - calls: list[int] = [] - - def fake_decide_pr(_repo: str, number: int, **_kwargs: Any) -> Any: - calls.append(number) - if len(calls) > 1: - raise module.GhError("gh: HTTP 502") - return decision(module, number=number) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - - assert module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) == 1 - - -def test_main_skips_probe_when_trigger_post_was_denied( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [714]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - monkeypatch.setattr( - module, - "decide_pr", - lambda _repo, number, **_kwargs: decision( - module, - number=number, - ok_action="keep", - has_ok_label=False, - trigger_codex_review=True, - checks_state="success", - ), - ) - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: ( - f"request Codex review on {request_decision.repo}#{request_decision.number}: " - "skipped because the GitHub token cannot write this resource", - ), - ) - - probe_calls: list[int] = [] - monkeypatch.setattr( - module, - "pr_timeline_evidence", - lambda _repo, number: (probe_calls.append(number), ("a" * 40, []))[1], - ) - sleeps: list[float] = [] - monkeypatch.setattr(module.time, "sleep", sleeps.append) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert probe_calls == [] - assert sleeps == [] - assert "write_warning=request Codex review on Soju06/codex-lb#714" in captured.out - - -def test_workflow_prefers_privileged_token_and_enables_tolerant_apply() -> None: - workflow = Path(".github/workflows/codex-review-labels.yml").read_text(encoding="utf-8") - - assert "secrets.CODEX_LABEL_SYNC_TOKEN || secrets.RELEASE_PLEASE_TOKEN || github.token" in workflow - app_slug_env = "GH_APP_SLUG: ${{ steps.app-token.outputs.token && steps.app-token.outputs.app-slug || '' }}" - assert workflow.count(app_slug_env) == 2 - assert "pull_request_review_thread:" not in workflow - assert "github.event_name == 'pull_request_review_thread'" not in workflow - assert 'cron: "*/15 * * * *"' in workflow - assert workflow.count("--tolerate-write-permission-errors") == 2 - assert workflow.count("--tolerate-read-errors") == 1 - - -def test_main_tolerates_read_errors_when_requested( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - - def fake_decide_pr(_repo: str, number: int, **_kwargs: Any) -> Any: - if number == 710: - raise module.GhError("gh: HTTP 502") - return decision(module, number=number) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--tolerate-read-errors"]) - - captured = capsys.readouterr() - assert result == 0 - assert "Soju06/codex-lb#710: gh: HTTP 502" in captured.err - assert "dry-run Soju06/codex-lb#714" in captured.out - - -def test_main_fails_tolerant_run_when_every_pr_read_fails( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - monkeypatch.setattr( - module, - "decide_pr", - lambda *_args, **_kwargs: (_ for _ in ()).throw(module.GhError("gh: HTTP 502")), - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--tolerate-read-errors"]) - - captured = capsys.readouterr() - assert result == 1 - assert "all selected PRs failed classification" in captured.err - - -def test_main_fails_read_errors_without_tolerance(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710]) - monkeypatch.setattr( - module, - "decide_pr", - lambda *_args, **_kwargs: (_ for _ in ()).throw(module.GhError("gh: HTTP 502")), - ) - - assert module.main(["--repo", "Soju06/codex-lb", "--all-open"]) == 1 - - -def test_main_fails_apply_errors_even_with_read_error_tolerance( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [714]) - monkeypatch.setattr(module, "decide_pr", lambda *_args, **_kwargs: decision(module)) - - def fail_apply(*_args: Any, **_kwargs: Any) -> tuple[str, ...]: - raise module.GhError("gh: HTTP 500 while writing labels") - - monkeypatch.setattr(module, "apply_decision", fail_apply) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply", "--tolerate-read-errors"]) - - captured = capsys.readouterr() - assert result == 1 - assert "Soju06/codex-lb#714: gh: HTTP 500 while writing labels" in captured.err - - -def test_pull_review_comment_nodes_uses_original_commit_or_head_reference(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - head_sha = "a" * 40 - old_sha = "b" * 40 - comment_data = [ - { - "body": "reanchored current-head inline review", - "commit_id": head_sha, - "original_commit_id": old_sha, - "pull_request_review_id": 1, - "created_at": "2026-06-11T00:00:00Z", - "html_url": "https://github.com/Soju06/codex-lb/pull/714#discussion_r1", - "user": {"login": "openai-codex"}, - }, - { - "body": f"stale but mentions head commit {head_sha[:12]}", - "commit_id": head_sha, - "original_commit_id": old_sha, - "pull_request_review_id": 2, - "created_at": "2026-06-11T00:00:00Z", - "html_url": "https://github.com/Soju06/codex-lb/pull/714#discussion_r2", - "user": {"login": "openai-codex"}, - }, - { - "body": "actual current-head inline review", - "commit_id": old_sha, - "original_commit_id": head_sha, - "pull_request_review_id": 3, - "created_at": "2026-06-11T00:00:00Z", - "html_url": "https://github.com/Soju06/codex-lb/pull/714#discussion_r3", - "user": {"login": "openai-codex"}, - }, - { - "body": "older unrelated comment", - "commit_id": old_sha, - "original_commit_id": old_sha, - "pull_request_review_id": 4, - "created_at": "2026-06-11T00:00:00Z", - "html_url": "https://github.com/Soju06/codex-lb/pull/714#discussion_r4", - "user": {"login": "openai-codex"}, - }, - ] - - monkeypatch.setattr(module, "paged_api", lambda _path: comment_data) - monkeypatch.setattr(module, "unresolved_review_comment_urls", lambda *_args: set()) - - nodes = module.pull_review_comment_nodes("Soju06/codex-lb", 714, head_sha=head_sha) - - assert [node.get("commit", {}).get("oid") for node in nodes] == [head_sha, head_sha, head_sha] - assert [node.get("pullRequestReviewDatabaseId") for node in nodes] == [None, None, 3] - - -def test_head_mentioned_fallback_comment_keeps_timeline_chronology() -> None: - module = load_sync_module() - head_sha = "a" * 40 - review_id = 2 - timeline_nodes = [ - { - "__typename": "PullRequestCommit", - "commit": {"oid": head_sha}, - "committedDate": "2026-06-11T06:30:00Z", - }, - { - "__typename": "PullRequestReview", - "databaseId": review_id, - "author": {"login": "openai-codex"}, - "bodyText": "Reviewed older commit.", - "submittedAt": "2026-06-11T06:32:00Z", - "commit": {"oid": "b" * 40}, - }, - { - "__typename": "IssueComment", - "author": {"login": "openai-codex"}, - "bodyText": "Codex Review: Didn't find any major issues.", - "createdAt": "2026-06-11T06:40:00Z", - }, - ] - comment_nodes = [ - { - "__typename": "PullRequestReviewComment", - "author": {"login": "openai-codex"}, - "bodyText": f"**[P2]** stale finding mentioning {head_sha[:12]}", - "createdAt": "2026-06-11T06:34:00Z", - "commit": {"oid": head_sha}, - "pullRequestReviewDatabaseId": None, - } - ] - - merged = module.merge_review_comment_nodes(timeline_nodes, comment_nodes) - assert [node["__typename"] for node in merged] == [ - "PullRequestCommit", - "PullRequestReview", - "PullRequestReviewComment", - "IssueComment", - ] - - state, node = module.find_current_head_codex_review_state( - merged, - head_sha=head_sha, - allowed_authors={"openai-codex"}, - ) - - assert state == "clean" - assert node is timeline_nodes[-1] - - -def test_unresolved_codex_threads_filter_to_current_head(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - head_sha = "a" * 40 - old_sha = "b" * 40 - - pages = [ - { - "data": { - "repository": { - "pullRequest": { - "reviewThreads": { - "pageInfo": {"hasNextPage": False, "endCursor": None}, - "nodes": [ - { - "isResolved": False, - "isOutdated": False, - "comments": { - "nodes": [ - { - "author": {"login": "openai-codex"}, - "body": "**[P1]** reanchored current-head finding", - "url": "https://example.invalid/reanchored-current", - "commit": {"oid": head_sha}, - "originalCommit": {"oid": old_sha}, - } - ] - }, - }, - { - "isResolved": False, - "isOutdated": False, - "comments": { - "nodes": [ - { - "author": {"login": "openai-codex"}, - "body": "**[P1]** current finding", - "url": "https://example.invalid/current", - "commit": {"oid": head_sha}, - "originalCommit": {"oid": head_sha}, - } - ] - }, - }, - { - "isResolved": False, - "isOutdated": False, - "comments": { - "nodes": [ - { - "author": {"login": "openai-codex"}, - "body": f"**[P2]** stale fallback for {head_sha[:12]}", - "url": "https://example.invalid/fallback", - "commit": {"oid": old_sha}, - "originalCommit": {"oid": old_sha}, - } - ] - }, - }, - { - "isResolved": False, - "isOutdated": False, - "comments": { - "nodes": [ - { - "author": {"login": "openai-codex"}, - "body": "**[P2]** stale old commit finding", - "url": "https://example.invalid/stale", - "commit": {"oid": old_sha}, - "originalCommit": {"oid": old_sha}, - } - ] - }, - }, - { - "isResolved": False, - "isOutdated": False, - "comments": { - "nodes": [ - { - "author": {"login": "openai-codex"}, - "body": "**[P1]** unresolved stale thread without commit metadata", - "url": "https://example.invalid/no-commit-metadata", - "commit": None, - "originalCommit": None, - } - ] - }, - }, - ], - } - } - } - } - } - ] - - monkeypatch.setattr(module, "graphql", lambda *_args, **_kwargs: pages[0]) - - urls = module.unresolved_codex_finding_thread_urls( - "Soju06/codex-lb", - 714, - head_sha=head_sha, - allowed_authors={"openai-codex"}, - ) - - assert urls == ( - "https://example.invalid/reanchored-current", - "https://example.invalid/current", - "https://example.invalid/fallback", - ) - - -def test_resolved_inline_codex_finding_does_not_count_as_review_news( - monkeypatch: pytest.MonkeyPatch, -) -> None: - module = load_sync_module() - - monkeypatch.setattr( - module, - "paged_api", - lambda _path: [ - { - "body": "**P1 Badge** resolved finding", - "commit_id": "a" * 40, - "original_commit_id": "a" * 40, - "pull_request_review_id": 123, - "html_url": "https://github.test/review/resolved", - "created_at": "2026-06-14T00:00:00Z", - "user": {"login": "chatgpt-codex-connector"}, - } - ], - ) - monkeypatch.setattr(module, "unresolved_review_comment_urls", lambda *_args: set()) - - assert module.pull_review_comment_nodes("Soju06/codex-lb", 714, head_sha="a" * 40) == [] - - -def test_unresolved_inline_codex_finding_counts_as_review_news( - monkeypatch: pytest.MonkeyPatch, -) -> None: - module = load_sync_module() - url = "https://github.test/review/unresolved" - - monkeypatch.setattr( - module, - "paged_api", - lambda _path: [ - { - "body": "**P1 Badge** unresolved finding", - "commit_id": "a" * 40, - "original_commit_id": "a" * 40, - "pull_request_review_id": 123, - "html_url": url, - "created_at": "2026-06-14T00:00:00Z", - "user": {"login": "chatgpt-codex-connector"}, - } - ], - ) - monkeypatch.setattr(module, "unresolved_review_comment_urls", lambda *_args: {url}) - - nodes = module.pull_review_comment_nodes("Soju06/codex-lb", 714, head_sha="a" * 40) - - assert len(nodes) == 1 - assert nodes[0]["url"] == url - - -def _rate_limited_proc() -> Any: - class _Proc: - returncode = 1 - stdout = "" - stderr = "gh: API rate limit exceeded for user ID 34199905 (HTTP 403)" - - return _Proc() - - -def _ok_proc(payload: str = "{}") -> Any: - class _Proc: - returncode = 0 - stdout = payload - stderr = "" - - return _Proc() - - -def _transient_gh_proc() -> Any: - class _Proc: - returncode = 1 - stdout = "" - stderr = "gh: HTTP 503" - - return _Proc() - - -def test_run_gh_switches_to_fallback_token_on_rate_limit(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_TOKEN", "primary-token") - monkeypatch.setenv("GH_FALLBACK_TOKEN", "fallback-token") - - calls: list[str] = [] - - def fake_run(command: Any, **kwargs: Any) -> Any: - import os - - calls.append(os.environ["GH_TOKEN"]) - if len(calls) == 1: - return _rate_limited_proc() - return _ok_proc() - - monkeypatch.setattr(module.subprocess, "run", fake_run) - - result = module.run_gh(["api", "/rate-limited-path"]) - - assert result == {} - assert calls == ["primary-token", "fallback-token"] - - -def test_run_gh_retries_transient_read_only_api_failure(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - calls: list[list[str]] = [] - sleeps: list[float] = [] - - def fake_run(command: list[str], **kwargs: Any) -> Any: - del kwargs - calls.append(command) - if len(calls) == 1: - return _transient_gh_proc() - return _ok_proc('{"ok": true}') - - monkeypatch.setattr(module.subprocess, "run", fake_run) - monkeypatch.setattr(module.time, "sleep", sleeps.append) - - assert module.run_gh(["api", "/repos/example/project/issues/1/labels"]) == {"ok": True} - assert len(calls) == 2 - assert sleeps == [2.0] - - -def test_run_gh_does_not_retry_mutating_pr_comment(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - calls: list[list[str]] = [] - - def fake_run(command: list[str], **kwargs: Any) -> Any: - del kwargs - calls.append(command) - return _transient_gh_proc() - - monkeypatch.setattr(module.subprocess, "run", fake_run) - monkeypatch.setattr(module.time, "sleep", lambda _: None) - - with pytest.raises(module.GhError): - module.run_gh(["pr", "comment", "1344", "--body", "@codex review"]) - assert len(calls) == 1 - - -def test_run_gh_activates_fallback_without_retrying_identity_sensitive_command( - monkeypatch: pytest.MonkeyPatch, -) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_TOKEN", "primary-token") - monkeypatch.setenv("GH_FALLBACK_TOKEN", "fallback-token") - - calls: list[list[str]] = [] - - def fake_run(command: list[str], **kwargs: Any) -> Any: - del kwargs - calls.append(command) - return _rate_limited_proc() - - monkeypatch.setattr(module.subprocess, "run", fake_run) - - with pytest.raises(module.GhError, match="without retrying this identity-sensitive command"): - module.run_gh( - ["api", "--method", "POST", "/repos/example/project/issues/1/comments"], - fallback_retry=False, - ) - - assert len(calls) == 1 - # The fallback still activates so the rest of the run stays alive. - assert module._fallback_token_active is True - import os - - assert os.environ["GH_TOKEN"] == "fallback-token" - - -def test_trigger_codex_review_posts_without_fallback_retry(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - captured_kwargs: list[dict[str, Any]] = [] - - def capture_run_gh(_args: list[str], **kwargs: Any) -> None: - captured_kwargs.append(kwargs) - - monkeypatch.setattr(module, "run_gh", capture_run_gh) - - warnings = module.trigger_codex_review( - decision(module, trigger_codex_review=True, ok_action="keep"), - body="@codex review", - ) - - assert warnings == () - assert len(captured_kwargs) == 1 - assert captured_kwargs[0]["fallback_retry"] is False - - -def test_run_gh_fails_without_distinct_fallback_token(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_TOKEN", "primary-token") - monkeypatch.setenv("GH_FALLBACK_TOKEN", "primary-token") - - monkeypatch.setattr(module.subprocess, "run", lambda command, **kwargs: _rate_limited_proc()) - - with pytest.raises(module.GhError): - module.run_gh(["api", "/rate-limited-path"]) - - -def test_run_gh_fails_when_fallback_token_is_also_exhausted(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_TOKEN", "primary-token") - monkeypatch.setenv("GH_FALLBACK_TOKEN", "fallback-token") - - monkeypatch.setattr(module.subprocess, "run", lambda command, **kwargs: _rate_limited_proc()) - - with pytest.raises(module.GhError): - module.run_gh(["api", "/rate-limited-path"]) diff --git a/tests/unit/test_telemetry_api.py b/tests/unit/test_telemetry_api.py new file mode 100644 index 0000000000..f6e3c9f261 --- /dev/null +++ b/tests/unit/test_telemetry_api.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import asyncio +import logging +from unittest.mock import AsyncMock, Mock + +import pytest + +from app.core.config.settings import get_settings + +pytestmark = pytest.mark.unit + + +@pytest.fixture(autouse=True) +def opt_out_sender(monkeypatch): + sender = Mock() + sender.send_opt_out = AsyncMock() + factory = Mock(return_value=sender) + monkeypatch.setattr("app.modules.telemetry.api.TelemetrySender", factory) + return sender + + +@pytest.mark.asyncio +async def test_consent_api_get_preview_and_put_persists_without_restart( + async_client, + monkeypatch, + opt_out_sender, +) -> None: + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + + response = await async_client.get("/api/settings/telemetry") + assert response.status_code == 200 + initial = response.json() + assert initial["state"] == "undecided" + assert initial["source"] == "default" + assert initial["active"] is True + assert set(initial["preview"]) == {"instance_id", "metrics", "timestamp"} + assert initial["preview"]["metrics"]["schema_version"] == 1 + assert initial["preview"]["metrics"]["consent"] == "undecided" + assert initial["preview"]["instance_id"] == initial["preview"]["metrics"]["instance_id"] + + response = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + assert response.status_code == 200 + disabled = response.json() + assert disabled["state"] == "disabled" + assert disabled["source"] == "persisted" + assert disabled["active"] is False + assert disabled["preview"] is None + await asyncio.sleep(0) + opt_out_sender.send_opt_out.assert_awaited_once() + + builder = Mock(side_effect=AssertionError("decided consent must not build a preview")) + monkeypatch.setattr("app.modules.telemetry.api.TelemetrySnapshotBuilder", builder) + response = await async_client.get("/api/settings/telemetry") + assert response.status_code == 200 + assert response.json()["state"] == "disabled" + assert response.json()["preview"] is None + builder.assert_not_called() + + +@pytest.mark.asyncio +async def test_consent_api_builds_decided_preview_only_when_requested( + async_client, + monkeypatch, +) -> None: + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + await async_client.put("/api/settings/telemetry", json={"enabled": False}) + + response = await async_client.get("/api/settings/telemetry?include_preview=true") + + assert response.status_code == 200 + payload = response.json() + assert payload["state"] == "disabled" + assert payload["preview"]["instance_id"] == payload["preview"]["metrics"]["instance_id"] + assert payload["preview"]["metrics"]["consent"] == "enabled" + + +@pytest.mark.asyncio +async def test_consent_api_env_override_wins_and_suppresses_undecided_state(async_client, monkeypatch) -> None: + monkeypatch.setenv("CODEX_LB_TELEMETRY_ENABLED", "true") + get_settings.cache_clear() + + builder = Mock(side_effect=AssertionError("environment override must not build a preview")) + monkeypatch.setattr("app.modules.telemetry.api.TelemetrySnapshotBuilder", builder) + response = await async_client.get("/api/settings/telemetry") + + assert response.status_code == 200 + payload = response.json() + assert payload["state"] == "enabled" + assert payload["source"] == "env" + assert payload["active"] is True + assert payload["preview"] is None + builder.assert_not_called() + + +@pytest.mark.asyncio +async def test_dashboard_active_to_inactive_transitions_each_send_exactly_once( + async_client, + monkeypatch, + opt_out_sender, +) -> None: + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + + first = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + assert first.status_code == 200 + await asyncio.sleep(0) + assert opt_out_sender.send_opt_out.await_count == 1 + + repeated = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + assert repeated.status_code == 200 + await asyncio.sleep(0) + assert opt_out_sender.send_opt_out.await_count == 1 + + enabled = await async_client.put("/api/settings/telemetry", json={"enabled": True}) + assert enabled.status_code == 200 + await asyncio.sleep(0) + assert opt_out_sender.send_opt_out.await_count == 1 + + second = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + assert second.status_code == 200 + await asyncio.sleep(0) + assert opt_out_sender.send_opt_out.await_count == 2 + + call = opt_out_sender.send_opt_out.await_args_list[-1] + assert call.args[0].instance_id + assert call.kwargs["app_version"] + assert call.kwargs["deployment_mode"] in {"docker", "k8s", "pip", "bare"} + assert "/" in call.kwargs["os_arch"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("env_value", "enabled", "expected_active"), + [("true", False, True), ("false", True, False)], +) +async def test_environment_controlled_put_never_sends_opt_out( + async_client, + monkeypatch, + opt_out_sender, + env_value: str, + enabled: bool, + expected_active: bool, +) -> None: + monkeypatch.setenv("CODEX_LB_TELEMETRY_ENABLED", env_value) + get_settings.cache_clear() + + response = await async_client.put("/api/settings/telemetry", json={"enabled": enabled}) + + assert response.status_code == 200 + assert response.json()["source"] == "env" + assert response.json()["active"] is expected_active + await asyncio.sleep(0) + opt_out_sender.send_opt_out.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_opt_out_background_send_does_not_block_settings_response( + async_client, + monkeypatch, + opt_out_sender, +) -> None: + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + started = asyncio.Event() + release = asyncio.Event() + + async def blocked_send(*args, **kwargs) -> None: + del args, kwargs + started.set() + await release.wait() + + opt_out_sender.send_opt_out.side_effect = blocked_send + + response = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + + assert response.status_code == 200 + await asyncio.wait_for(started.wait(), timeout=1) + from app.modules.telemetry import api as telemetry_api + + assert telemetry_api._OPT_OUT_TASKS + release.set() + await asyncio.gather(*tuple(telemetry_api._OPT_OUT_TASKS)) + await asyncio.sleep(0) + assert not telemetry_api._OPT_OUT_TASKS + + +@pytest.mark.asyncio +async def test_opt_out_identity_failure_is_debug_only_and_preserves_disabled_state( + async_client, + monkeypatch, + opt_out_sender, + caplog, +) -> None: + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + + async def fail_identity(_store) -> None: + raise RuntimeError("identity decryption failed") + + monkeypatch.setattr( + "app.modules.telemetry.api.TelemetryConsentStore.get_or_create_identity", + fail_identity, + ) + + with caplog.at_level(logging.DEBUG, logger="app.modules.telemetry.api"): + response = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + + assert response.status_code == 200 + assert response.json()["state"] == "disabled" + persisted = await async_client.get("/api/settings/telemetry") + assert persisted.status_code == 200 + assert persisted.json()["state"] == "disabled" + opt_out_sender.send_opt_out.assert_not_awaited() + assert "Unable to schedule anonymous telemetry opt-out" in caplog.messages + assert all(record.levelno == logging.DEBUG for record in caplog.records) + + +@pytest.mark.asyncio +async def test_unexpected_opt_out_task_failure_is_debug_only_and_does_not_change_response( + async_client, + monkeypatch, + opt_out_sender, + caplog, +) -> None: + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + opt_out_sender.send_opt_out.side_effect = RuntimeError("unexpected sender failure") + + with caplog.at_level(logging.DEBUG, logger="app.modules.telemetry.api"): + response = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + await asyncio.sleep(0) + + assert response.status_code == 200 + assert caplog.records + assert all(record.levelno == logging.DEBUG for record in caplog.records) diff --git a/tests/unit/test_telemetry_consent.py b/tests/unit/test_telemetry_consent.py new file mode 100644 index 0000000000..0130db2da5 --- /dev/null +++ b/tests/unit/test_telemetry_consent.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from unittest.mock import AsyncMock, Mock + +import pytest + +from app.core.config.settings import get_settings +from app.db.models import DashboardSettings +from app.db.session import SessionLocal +from app.modules.telemetry.consent import TelemetryConsentStore, resolve_consent +from app.modules.telemetry.scheduler import TELEMETRY_INTERVAL_SECONDS, TelemetryScheduler + +pytestmark = pytest.mark.unit + + +class _GateLeader: + def __init__(self, *, leader: bool) -> None: + self.leader = leader + self.run_if_leader_calls = 0 + + async def run_if_leader(self, fn: Callable[[], Awaitable[object]]) -> object | None: + self.run_if_leader_calls += 1 + if not self.leader: + return None + return await fn() + + +def test_consent_precedence_and_default_activation() -> None: + assert resolve_consent(False, "enabled").state == "disabled" + assert resolve_consent(False, "enabled").source == "env" + assert resolve_consent(False, "enabled").active is False + + env_enabled = resolve_consent(True, "undecided") + assert env_enabled.state == "enabled" + assert env_enabled.source == "env" + assert env_enabled.active is True + + persisted_disabled = resolve_consent(None, "disabled") + assert persisted_disabled.state == "disabled" + assert persisted_disabled.source == "persisted" + assert persisted_disabled.active is False + + undecided = resolve_consent(None, "undecided") + assert undecided.state == "undecided" + assert undecided.source == "default" + assert undecided.active is True + + +@pytest.mark.asyncio +async def test_random_uuid_v4_identity_is_persisted_and_regenerated_after_deletion(db_setup) -> None: + del db_setup + async with SessionLocal() as session: + store = TelemetryConsentStore(session) + first = await store.get_or_create_identity() + second = await store.get_or_create_identity() + assert first.instance_id == second.instance_id + assert first.public_key_hex == second.public_key_hex + assert first.instance_id.split("-")[2].startswith("4") + + row = await session.get(DashboardSettings, 1) + assert row is not None + row.telemetry_instance_id = None + await session.commit() + session.expire_all() + + replacement = await store.get_or_create_identity() + assert replacement.instance_id != first.instance_id + assert replacement.public_key_hex != first.public_key_hex + + +@pytest.mark.asyncio +async def test_disabled_scheduler_tick_makes_zero_sender_calls(db_setup, monkeypatch) -> None: + del db_setup + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + async with SessionLocal() as session: + store = TelemetryConsentStore(session) + await store.set_decision(False) + + sender = AsyncMock() + scheduler = TelemetryScheduler(sender=sender) + await scheduler._tick() + + sender.send_snapshot.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_enabled_scheduler_snapshot_declares_enabled_consent(db_setup, monkeypatch) -> None: + del db_setup + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + async with SessionLocal() as session: + store = TelemetryConsentStore(session) + await store.set_decision(True) + + sender = AsyncMock() + await TelemetryScheduler(sender=sender)._tick() + + sender.send_snapshot.assert_awaited_once() + assert sender.send_snapshot.await_args.args[0].consent == "enabled" + + +@pytest.mark.asyncio +async def test_non_leader_scheduler_tick_builds_and_transmits_nothing(monkeypatch) -> None: + import app.modules.telemetry.scheduler as scheduler_module + + leader = _GateLeader(leader=False) + builder = Mock(side_effect=AssertionError("non-leader must not construct a snapshot builder")) + monkeypatch.setattr(scheduler_module, "_get_leader_election", lambda: leader) + monkeypatch.setattr(scheduler_module, "TelemetrySnapshotBuilder", builder) + sender = AsyncMock() + + await TelemetryScheduler(sender=sender)._tick(log_undecided_notice=True) + + assert leader.run_if_leader_calls == 1 + builder.assert_not_called() + sender.send_snapshot.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_main_lifespan_constructs_starts_and_stops_telemetry_scheduler(app_instance, monkeypatch) -> None: + import app.main as main_module + + scheduler = Mock() + scheduler.start = AsyncMock() + scheduler.stop = AsyncMock() + factory = Mock(return_value=scheduler) + monkeypatch.setattr(main_module, "build_telemetry_scheduler", factory) + + async with app_instance.router.lifespan_context(app_instance): + factory.assert_called_once_with() + scheduler.start.assert_awaited_once_with() + scheduler.stop.assert_not_awaited() + + scheduler.stop.assert_awaited_once_with() + + +@pytest.mark.asyncio +async def test_scheduler_sends_startup_and_interval_snapshots_with_one_undecided_notice( + db_setup, + monkeypatch, + caplog, +) -> None: + del db_setup + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + assert TELEMETRY_INTERVAL_SECONDS == 24 * 60 * 60 + + sender = AsyncMock() + scheduler = TelemetryScheduler(sender=sender, interval_seconds=0.01) + with caplog.at_level(logging.INFO, logger="app.modules.telemetry.scheduler"): + await scheduler.start() + for _ in range(50): + if sender.send_snapshot.await_count >= 2: + break + await asyncio.sleep(0.01) + await scheduler.stop() + + assert sender.send_snapshot.await_count >= 2 + assert all(call.args[0].consent == "undecided" for call in sender.send_snapshot.await_args_list) + notices = [ + record.getMessage() for record in caplog.records if "Anonymous telemetry is active" in record.getMessage() + ] + assert len(notices) == 1 + assert "https://soju06.github.io/codex-lb/telemetry/" in notices[0] + assert "CODEX_LB_TELEMETRY_ENABLED=false" in notices[0] + assert scheduler._task is None diff --git a/tests/unit/test_telemetry_migration.py b/tests/unit/test_telemetry_migration.py new file mode 100644 index 0000000000..b41ed5fb4d --- /dev/null +++ b/tests/unit/test_telemetry_migration.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import pytest +from alembic import command +from anyio import to_thread +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +from app.db.migrate import _build_alembic_config, inspect_migration_state, run_upgrade + +pytestmark = pytest.mark.unit + + +@pytest.mark.asyncio +async def test_telemetry_migration_upgrade_defaults_and_downgrade(tmp_path) -> None: + db_url = f"sqlite+aiosqlite:///{tmp_path / 'telemetry.sqlite'}" + parent = "20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads" + revision = "20260806_000000_add_anonymous_telemetry" + telemetry_columns = { + "telemetry_consent", + "telemetry_instance_id", + "telemetry_private_key_encrypted", + } + + async def columns_and_rows(engine): + async with engine.connect() as connection: + columns = {row[1] for row in await connection.execute(text("PRAGMA table_info('dashboard_settings')"))} + rows = [] + if telemetry_columns <= columns: + rows = ( + await connection.execute( + text( + "SELECT telemetry_consent, telemetry_instance_id, " + "telemetry_private_key_encrypted FROM dashboard_settings" + ) + ) + ).all() + return columns, rows + + await to_thread.run_sync(lambda: run_upgrade(db_url, parent, bootstrap_legacy=False)) + engine = create_async_engine(db_url) + try: + columns, _ = await columns_and_rows(engine) + assert not telemetry_columns & columns + + await to_thread.run_sync(lambda: run_upgrade(db_url, revision, bootstrap_legacy=False)) + columns, rows = await columns_and_rows(engine) + assert telemetry_columns <= columns + assert rows + assert all(row == ("undecided", None, None) for row in rows) + + await to_thread.run_sync(lambda: command.downgrade(_build_alembic_config(db_url), parent)) + columns, _ = await columns_and_rows(engine) + assert not telemetry_columns & columns + + result = await to_thread.run_sync(lambda: run_upgrade(db_url, "head", bootstrap_legacy=False)) + assert result.current_revision == inspect_migration_state(db_url).head_revision + columns, _ = await columns_and_rows(engine) + assert telemetry_columns <= columns + finally: + await engine.dispose() diff --git a/tests/unit/test_telemetry_sender.py b/tests/unit/test_telemetry_sender.py new file mode 100644 index 0000000000..abe4ff3b99 --- /dev/null +++ b/tests/unit/test_telemetry_sender.py @@ -0,0 +1,369 @@ +from __future__ import annotations + +import json +import logging +from datetime import datetime +from unittest.mock import AsyncMock, Mock + +import pytest +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from app.modules.telemetry.consent import TelemetryIdentity +from app.modules.telemetry.schemas import TelemetrySnapshot, build_snapshot_envelope +from app.modules.telemetry.sender import TelemetrySender + +pytestmark = pytest.mark.unit + + +class _FakeResponse: + status = 200 + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, traceback) -> None: + return None + + async def read(self) -> bytes: + return b"" + + +class _FakeClientSession: + def __init__(self) -> None: + self.requests: list[tuple[str, bytes, dict[str, str]]] = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, traceback) -> None: + return None + + def post(self, url: str, *, data: bytes, headers: dict[str, str]) -> _FakeResponse: + self.requests.append((url, data, headers)) + return _FakeResponse() + + +def _snapshot() -> TelemetrySnapshot: + return TelemetrySnapshot.model_validate( + { + "consent": "enabled", + "instance_id": "00000000-0000-4000-8000-000000000004", + "version": "1.0.0", + "python": "3.13", + "os": "linux", + "arch": "x86_64", + "uptime_hours": 1, + "deploy": { + "method": "bare", + "db_backend": "sqlite", + "db_size_bucket": "<100MB", + "replicas": 1, + "reverse_proxy": False, + }, + "accounts": { + "pool_bucket": "0", + "plan_mix": {"plus": "0", "pro": "0", "team": "0", "free": "0"}, + "workspace_accounts": False, + "routing_policy": "capacity_weighted", + "limit_warmup_enabled": False, + "egress_proxy_used": False, + }, + "usage_7d": { + "requests": 0, + "success_rate": 0, + "tokens_input": 0, + "tokens_output": 0, + "tokens_cached_ratio": 0, + "cost_usd_bucket": "<10", + "request_kinds": {"responses": 0, "chat": 0, "images": 0, "unknown": 0}, + "transport_mix": {"ws": 0, "http_bridge": 0}, + "service_tier_mix": {"default": 0, "flex": 0, "priority": 0}, + "clients": {}, + "clients_other_ratio": 0, + "models": [], + "latency_ms_p50": 0, + "ttft_ms_p50": 0, + "ttft_ms_p95": 0, + "rate_limit_429_ratio": 0, + "top_upstream_errors": [], + }, + "features": { + "api_firewall": False, + "quota_planner": False, + "sticky_sessions": False, + "conversation_archive": False, + "automations": False, + "fleet": True, + "model_sources_count": 0, + "api_keys_bucket": "0", + "prometheus": False, + "otel": False, + "dashboard_auth": True, + "reset_credits": False, + "image_api_used": False, + }, + } + ) + + +@pytest.mark.asyncio +async def test_sender_failure_isolated_retries_once_and_logs_debug_only(caplog) -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + + async def context_provider(): + return True, identity + + sender = TelemetrySender("http://127.0.0.1:1", context_provider=context_provider) + sender._transmit_once = AsyncMock(side_effect=OSError("endpoint unreachable")) + + with caplog.at_level(logging.DEBUG, logger="app.modules.telemetry.sender"): + await sender.send_snapshot(snapshot) + + assert sender._transmit_once.await_count == 2 + assert caplog.records + assert all(record.levelno == logging.DEBUG for record in caplog.records) + + +@pytest.mark.asyncio +async def test_sender_disabled_guard_does_not_construct_http_client(monkeypatch) -> None: + async def context_provider(): + return False, None + + client_session = Mock() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", client_session) + + await TelemetrySender(context_provider=context_provider).send_snapshot(_snapshot()) + + client_session.assert_not_called() + + +@pytest.mark.asyncio +async def test_sender_aborts_snapshot_when_consent_becomes_inactive_before_post(monkeypatch) -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + context_provider = AsyncMock(side_effect=[(True, identity), (False, None)]) + session = _FakeClientSession() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", Mock(return_value=session)) + + await TelemetrySender( + "https://telemetry.example", + context_provider=context_provider, + ).send_snapshot(snapshot) + + assert [request[0] for request in session.requests] == [ + "https://telemetry.example/v1/register", + "https://telemetry.example/v1/activate", + ] + assert context_provider.await_count == 2 + + +@pytest.mark.asyncio +async def test_sender_posts_snapshot_once_when_consent_stays_active(monkeypatch) -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + context_provider = AsyncMock(return_value=(True, identity)) + session = _FakeClientSession() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", Mock(return_value=session)) + + await TelemetrySender( + "https://telemetry.example", + context_provider=context_provider, + ).send_snapshot(snapshot) + + assert [request[0] for request in session.requests] == [ + "https://telemetry.example/v1/register", + "https://telemetry.example/v1/activate", + "https://telemetry.example/v1/snapshot", + ] + assert context_provider.await_count == 2 + + +@pytest.mark.asyncio +async def test_sender_aborts_snapshot_when_identity_changes_before_post(monkeypatch) -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + replacement_identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + context_provider = AsyncMock(side_effect=[(True, identity), (True, replacement_identity)]) + session = _FakeClientSession() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", Mock(return_value=session)) + + await TelemetrySender( + "https://telemetry.example", + context_provider=context_provider, + ).send_snapshot(snapshot) + + assert [request[0] for request in session.requests] == [ + "https://telemetry.example/v1/register", + "https://telemetry.example/v1/activate", + ] + assert context_provider.await_count == 2 + + +@pytest.mark.asyncio +async def test_sender_aborts_snapshot_when_consent_recheck_fails(monkeypatch, caplog) -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + context_provider = AsyncMock(side_effect=[(True, identity), OSError("database unavailable")]) + session = _FakeClientSession() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", Mock(return_value=session)) + + with caplog.at_level(logging.DEBUG, logger="app.modules.telemetry.sender"): + await TelemetrySender( + "https://telemetry.example", + context_provider=context_provider, + ).send_snapshot(snapshot) + + assert [request[0] for request in session.requests] == [ + "https://telemetry.example/v1/register", + "https://telemetry.example/v1/activate", + ] + assert context_provider.await_count == 2 + assert [record.message for record in caplog.records] == ["Anonymous telemetry consent re-check failed"] + + +@pytest.mark.asyncio +async def test_opt_out_with_inactive_consent_registers_activates_and_posts_exact_signed_canonical_body( + monkeypatch, +) -> None: + identity = TelemetryIdentity("00000000-0000-4000-8000-000000000004", Ed25519PrivateKey.generate()) + session = _FakeClientSession() + client_session = Mock(return_value=session) + context_provider = AsyncMock(return_value=(False, None)) + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", client_session) + monkeypatch.setattr("app.modules.telemetry.sender.utcnow", lambda: datetime(2026, 8, 20, 12, 0, 0)) + + await TelemetrySender( + "https://telemetry.example", + context_provider=context_provider, + ).send_opt_out( + identity, + app_version="1.24.0", + deployment_mode="docker", + os_arch="linux/x86_64", + ) + + assert [request[0] for request in session.requests] == [ + "https://telemetry.example/v1/register", + "https://telemetry.example/v1/activate", + "https://telemetry.example/v1/optout", + ] + expected_body = ( + b'{"app_version":"1.24.0","event":"optout",' + b'"instance_id":"00000000-0000-4000-8000-000000000004",' + b'"occurred_at":"2026-08-20T12:00:00Z"}' + ) + _, body, headers = session.requests[-1] + assert body == expected_body + assert headers["X-Instance-ID"] == identity.instance_id + identity.private_key.public_key().verify(bytes.fromhex(headers["X-Signature"]), body) + context_provider.assert_not_awaited() + client_session.assert_called_once() + assert client_session.call_args.kwargs["timeout"].total == 5.0 + assert client_session.call_args.kwargs["trust_env"] is False + + +@pytest.mark.asyncio +async def test_opt_out_retries_once_then_succeeds(monkeypatch) -> None: + identity = TelemetryIdentity("00000000-0000-4000-8000-000000000004", Ed25519PrivateKey.generate()) + session = _FakeClientSession() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", Mock(return_value=session)) + sender = TelemetrySender() + sender._transmit_opt_out_once = AsyncMock(side_effect=[OSError("transient"), None]) + + await sender.send_opt_out( + identity, + app_version="1.24.0", + deployment_mode="bare", + os_arch="linux/x86_64", + ) + + assert sender._transmit_opt_out_once.await_count == 2 + + +@pytest.mark.asyncio +async def test_opt_out_failure_is_swallowed_and_logged_at_debug(monkeypatch, caplog) -> None: + identity = TelemetryIdentity("00000000-0000-4000-8000-000000000004", Ed25519PrivateKey.generate()) + session = _FakeClientSession() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", Mock(return_value=session)) + sender = TelemetrySender() + sender._transmit_opt_out_once = AsyncMock(side_effect=OSError("collector unavailable")) + + with caplog.at_level(logging.DEBUG, logger="app.modules.telemetry.sender"): + await sender.send_opt_out( + identity, + app_version="1.24.0", + deployment_mode="bare", + os_arch="linux/x86_64", + ) + + assert sender._transmit_opt_out_once.await_count == 2 + assert caplog.records + assert all(record.levelno == logging.DEBUG for record in caplog.records) + + +@pytest.mark.asyncio +async def test_sender_uses_canonical_shm_paths_and_valid_ed25519_signature() -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + sender = TelemetrySender(context_provider=AsyncMock(return_value=(True, identity))) + sender._post = AsyncMock() + sender._post_signed = AsyncMock() + session = Mock() + + await sender._transmit_once(session, snapshot, identity) + + register_call = sender._post.await_args + assert register_call is not None + assert register_call.args[1] == "/v1/register" + assert [call.args[1] for call in sender._post_signed.await_args_list] == ["/v1/activate", "/v1/snapshot"] + assert all("/api/v1/" not in call.args[1] for call in sender._post_signed.await_args_list) + + registration = json.loads(register_call.args[2]) + activation = json.loads(sender._post_signed.await_args_list[0].args[2]) + envelope = json.loads(sender._post_signed.await_args_list[1].args[2]) + assert set(registration) == { + "app_name", + "app_version", + "deployment_mode", + "environment", + "instance_id", + "os_arch", + "public_key", + } + assert set(activation) == {"action"} + assert set(envelope) == {"instance_id", "metrics", "timestamp"} + + signing_sender = TelemetrySender() + signing_sender._post = AsyncMock() + body = b'{"action":"activate"}' + await signing_sender._post_signed(session, "/v1/activate", body, identity, accepted={200}) + signed_call = signing_sender._post.await_args + assert signed_call is not None + headers = signed_call.kwargs["headers"] + identity.private_key.public_key().verify(bytes.fromhex(headers["X-Signature"]), body) + assert headers["X-Instance-ID"] == identity.instance_id + + +def _key_structure(value): + if isinstance(value, dict): + return {key: _key_structure(child) for key, child in value.items()} + if isinstance(value, list): + return [_key_structure(value[0])] if value else [] + return None + + +@pytest.mark.asyncio +async def test_preview_and_sender_snapshot_envelopes_have_identical_key_structure() -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + preview = build_snapshot_envelope(snapshot) + sender = TelemetrySender(context_provider=AsyncMock(return_value=(True, identity))) + sender._post = AsyncMock() + sender._post_signed = AsyncMock() + + await sender._transmit_once(Mock(), snapshot, identity) + + sender_body = json.loads(sender._post_signed.await_args_list[-1].args[2]) + preview_body = json.loads(preview.model_dump_json()) + assert _key_structure(sender_body) == _key_structure(preview_body) diff --git a/tests/unit/test_telemetry_snapshot.py b/tests/unit/test_telemetry_snapshot.py new file mode 100644 index 0000000000..7d51559234 --- /dev/null +++ b/tests/unit/test_telemetry_snapshot.py @@ -0,0 +1,537 @@ +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from pathlib import Path +from typing import get_args + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.core.balancer.logic import RoutingStrategy +from app.core.crypto import TokenEncryptor +from app.core.utils.time import utcnow +from app.db.models import Account, AccountStatus, ApiKey, Base, ModelSource, RequestLog +from app.modules.telemetry.clients import ( + CANONICAL_CLIENT_FAMILIES, + CLIENT_FAMILY_BY_RAW_GROUP, + ClientCount, + client_family, + client_shares, +) +from app.modules.telemetry.schemas import ( + TelemetryActivation, + TelemetryOptOut, + TelemetryRegistration, + build_snapshot_envelope, +) +from app.modules.telemetry.snapshot import ( + _ROUTING_POLICIES, + TelemetrySnapshotBuilder, + _canonical_routing_policy, + cost_bucket, + count_bucket, + db_size_bucket, + output_tokens_bucket, +) + +pytestmark = pytest.mark.unit + + +@pytest.fixture +async def async_session() -> AsyncIterator[AsyncSession]: + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + factory = async_sessionmaker(engine, expire_on_commit=False) + async with factory() as session: + yield session + await engine.dispose() + + +def _request_log( + request_id: str, + *, + model: str, + useragent_group: str, + reasoning_effort: str | None = None, + output_tokens: int = 100, + account_id: str | None = None, + status: str = "success", + **values, +) -> RequestLog: + return RequestLog( + account_id=account_id, + request_id=request_id, + requested_at=utcnow(), + model=model, + status=status, + useragent_group=useragent_group, + reasoning_effort=reasoning_effort, + input_tokens=200, + output_tokens=output_tokens, + cached_input_tokens=50, + cost_usd=1.0, + latency_ms=1_000, + latency_first_token_ms=400, + transport="http", + **values, + ) + + +@pytest.mark.asyncio +async def test_snapshot_serialized_field_set_matches_documented_schema(async_session: AsyncSession) -> None: + async_session.add(_request_log("schema", model="gpt-5.4", useragent_group="codex_exec")) + await async_session.commit() + + snapshot = await TelemetrySnapshotBuilder(async_session).build( + "00000000-0000-4000-8000-000000000001", + consent="undecided", + ) + payload = snapshot.model_dump() + + assert payload["consent"] == "undecided" + assert set(payload) == { + "schema_version", + "consent", + "instance_id", + "version", + "python", + "os", + "arch", + "uptime_hours", + "deploy", + "accounts", + "usage_7d", + "features", + } + assert set(payload["deploy"]) == {"method", "db_backend", "db_size_bucket", "replicas", "reverse_proxy"} + assert set(payload["accounts"]) == { + "pool_bucket", + "plan_mix", + "workspace_accounts", + "routing_policy", + "limit_warmup_enabled", + "egress_proxy_used", + } + assert set(payload["accounts"]["plan_mix"]) == {"plus", "pro", "team", "free"} + assert set(payload["usage_7d"]) == { + "requests", + "success_rate", + "tokens_input", + "tokens_output", + "tokens_cached_ratio", + "cost_usd_bucket", + "request_kinds", + "transport_mix", + "service_tier_mix", + "clients", + "clients_other_ratio", + "models", + "latency_ms_p50", + "ttft_ms_p50", + "ttft_ms_p95", + "rate_limit_429_ratio", + "top_upstream_errors", + } + assert set(payload["usage_7d"]["request_kinds"]) == {"responses", "chat", "images", "unknown"} + assert set(payload["usage_7d"]["transport_mix"]) == {"ws", "http_bridge"} + assert set(payload["usage_7d"]["service_tier_mix"]) == {"default", "flex", "priority"} + assert set(payload["usage_7d"]["models"][0]) == { + "name", + "share", + "reasoning", + "avg_output_tokens_bucket", + } + assert set(payload["features"]) == { + "api_firewall", + "quota_planner", + "sticky_sessions", + "conversation_archive", + "automations", + "fleet", + "model_sources_count", + "api_keys_bucket", + "prometheus", + "otel", + "dashboard_auth", + "reset_credits", + "image_api_used", + } + + registration = TelemetryRegistration( + app_version=snapshot.version, + deployment_mode=snapshot.deploy.method, + instance_id=snapshot.instance_id, + os_arch=f"{snapshot.os}/{snapshot.arch}", + public_key="00", + ).model_dump(mode="json") + activation = TelemetryActivation().model_dump(mode="json") + opt_out = TelemetryOptOut( + app_version=snapshot.version, + instance_id=snapshot.instance_id, + occurred_at="2026-08-20T12:00:00Z", + ).model_dump(mode="json") + envelope = build_snapshot_envelope(snapshot).model_dump(mode="json") + assert set(registration) == { + "app_name", + "app_version", + "deployment_mode", + "environment", + "instance_id", + "os_arch", + "public_key", + } + assert set(activation) == {"action"} + assert set(opt_out) == {"app_version", "event", "instance_id", "occurred_at"} + assert set(envelope) == {"instance_id", "metrics", "timestamp"} + + +def test_client_mapping_table_and_unknown_family_are_allowlisted() -> None: + for raw_group, expected_family in CLIENT_FAMILY_BY_RAW_GROUP.items(): + assert client_family(raw_group) == expected_family + assert client_family("senpi") == "other" + + shares, other_ratio = client_shares( + [ + ClientCount("codex_exec", 2), + ClientCount("codex-tui", 3), + ClientCount("senpi", 1), + ] + ) + assert shares == {"codex-cli": 0.833333, "other": 0.166667} + assert other_ratio == 0.166667 + assert "senpi" not in str(shares) + + +def test_client_share_emission_rejects_noncanonical_mapping(monkeypatch) -> None: + monkeypatch.setitem(CLIENT_FAMILY_BY_RAW_GROUP, "unexpected", "private-client") + assert "private-client" not in CANONICAL_CLIENT_FAMILIES + + with pytest.raises(ValueError, match="non-canonical telemetry client family"): + client_shares([ClientCount("unexpected", 1)]) + + +def test_routing_policy_allowlist_is_derived_from_balancer_declaration() -> None: + assert _ROUTING_POLICIES == frozenset(get_args(RoutingStrategy)) + for strategy in get_args(RoutingStrategy): + assert _canonical_routing_policy(strategy) == strategy + + +@pytest.mark.asyncio +async def test_model_catalog_filter_merges_custom_models_and_scopes_reasoning( + async_session: AsyncSession, +) -> None: + async_session.add_all( + [ + _request_log("official-high", model="gpt-5.4", useragent_group="OpenAI", reasoning_effort="high"), + _request_log("official-low", model="gpt-5.4", useragent_group="OpenAI", reasoning_effort="low"), + _request_log( + "private-high", + model="corp-internal-gpt", + useragent_group="senpi", + reasoning_effort="high", + output_tokens=2_000, + ), + _request_log( + "private-custom-effort", + model="another-private-model", + useragent_group="senpi", + reasoning_effort="secret-effort", + output_tokens=2_000, + ), + ] + ) + await async_session.commit() + + payload = ( + await TelemetrySnapshotBuilder(async_session).build( + "00000000-0000-4000-8000-000000000002", + consent="enabled", + ) + ).model_dump() + assert payload["consent"] == "enabled" + models = {model["name"]: model for model in payload["usage_7d"]["models"]} + + assert set(models) == {"gpt-5.4", "other"} + assert models["gpt-5.4"]["reasoning"] == {"high": 0.5, "low": 0.5} + assert models["other"]["reasoning"] == {"high": 0.5, "other": 0.5} + assert models["other"]["share"] == 0.5 + assert models["other"]["avg_output_tokens_bucket"] == "1k-4k" + assert "reasoning" not in payload["usage_7d"] + serialized = str(payload) + assert "corp-internal-gpt" not in serialized + assert "another-private-model" not in serialized + assert "secret-effort" not in serialized + + +@pytest.mark.asyncio +async def test_request_kind_mix_fails_honest_without_persisted_route_family(async_session: AsyncSession) -> None: + async_session.add_all( + [ + _request_log("subscription", model="gpt-5.4", useragent_group="codex_exec"), + _request_log( + "source-backed", + model="gpt-5.4", + useragent_group="OpenAI", + source="model_source", + ), + _request_log( + "image-shaped", + model="gpt-image-1", + useragent_group="OpenAI", + source="model_source", + ), + ] + ) + await async_session.commit() + + payload = await TelemetrySnapshotBuilder(async_session).build( + "00000000-0000-4000-8000-000000000005", + consent="undecided", + ) + + assert payload.usage_7d.request_kinds.model_dump() == { + "responses": 0.0, + "chat": 0.0, + "images": 0.0, + "unknown": 1.0, + } + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (0, "0"), + (1, "1"), + (2, "2-5"), + (5, "2-5"), + (6, "6-20"), + (20, "6-20"), + (21, "21-100"), + (100, "21-100"), + (101, "100+"), + ], +) +def test_count_bucket_edges(value: int, expected: str) -> None: + assert count_bucket(value) == expected + + +def test_sensitive_aggregate_bucket_edges() -> None: + mib = 1024**2 + gib = 1024**3 + assert [ + db_size_bucket(value) for value in (None, 0, 100 * mib - 1, 100 * mib, gib, 5 * gib, 10 * gib, 50 * gib) + ] == [ + "unknown", + "<100MB", + "<100MB", + "100MB-1GB", + "1-5GB", + "5-10GB", + "10-50GB", + "50GB+", + ] + assert [cost_bucket(value) for value in (0, 9.99, 10, 99.99, 100, 999.99, 1_000, 10_000, 50_000)] == [ + "<10", + "<10", + "10-100", + "10-100", + "100-1k", + "100-1k", + "1k-10k", + "10k-50k", + "50k+", + ] + assert [output_tokens_bucket(value) for value in (0, 249, 250, 999, 1_000, 3_999, 4_000, 15_999, 16_000)] == [ + "<250", + "<250", + "250-1k", + "250-1k", + "1k-4k", + "1k-4k", + "4k-16k", + "4k-16k", + "16k+", + ] + + +@pytest.mark.asyncio +async def test_unmeasurable_database_size_is_unknown_and_logs_original_exception( + async_session: AsyncSession, + monkeypatch, + caplog, +) -> None: + error = OSError("stat denied") + target = Path("/tmp/telemetry-unmeasurable-db.sqlite3") + real_stat = Path.stat + + def fail_stat(path: Path, *, follow_symlinks: bool = True): + if path == target: + raise error + return real_stat(path, follow_symlinks=follow_symlinks) + + monkeypatch.setattr("app.modules.telemetry.snapshot.sqlite_db_path_from_url", lambda _url: str(target)) + monkeypatch.setattr(Path, "stat", fail_stat) + builder = TelemetrySnapshotBuilder(async_session) + + with caplog.at_level(logging.DEBUG, logger="app.modules.telemetry.snapshot"): + size = await builder._database_size_bytes() + + assert size is None + assert db_size_bucket(size) == "unknown" + assert caplog.records[-1].exc_info is not None + assert caplog.records[-1].exc_info[1] is error + + +@pytest.mark.asyncio +async def test_privacy_quick_check_identifying_values_never_serialize(async_session: AsyncSession) -> None: + encryptor = TokenEncryptor() + account = Account( + id="account-private-id", + email="alice@corp.com", + workspace_id="W1", + plan_type="team", + access_token_encrypted=encryptor.encrypt("access-private"), + refresh_token_encrypted=encryptor.encrypt("refresh-private"), + id_token_encrypted=encryptor.encrypt("id-private"), + last_refresh=utcnow(), + status=AccountStatus.ACTIVE, + deactivation_reason=None, + ) + async_session.add(account) + async_session.add( + ApiKey( + id="api-key-private-id", + name="private-key-name", + key_hash="super-secret-api-key-hash", + key_prefix="sk-private", + is_active=True, + ) + ) + async_session.add( + ModelSource( + id="private-source-id", + name="private-source-name", + base_url="https://private.example.test", + api_key_encrypted=encryptor.encrypt("source-api-key"), + is_enabled=True, + ) + ) + async_session.add( + _request_log( + "privacy", + account_id=account.id, + model="corp-internal-gpt", + useragent_group="senpi", + useragent="senpi/1.0 alice@corp.com", + client_ip="192.0.2.9", + # Error status so the private code exercises the top-errors + # sanitizer; cancelled/success rows are excluded from that metric. + status="error", + error_message="free text alice W1 super-secret-api-key-hash", + upstream_error_code="private-upstream-message", + ) + ) + await async_session.commit() + + serialized = ( + await TelemetrySnapshotBuilder(async_session).build( + "00000000-0000-4000-8000-000000000003", + consent="undecided", + ) + ).model_dump_json() + + for private_value in ( + "alice", + "corp.com", + "W1", + "corp-internal-gpt", + "senpi", + "192.0.2.9", + "super-secret-api-key-hash", + "private-source-name", + "private-source-id", + "private-upstream-message", + ): + assert private_value not in serialized + assert '"pool_bucket":"1"' in serialized + assert '"workspace_accounts":true' in serialized + assert '"name":"other"' in serialized + assert '"clients":{"other":1.0}' in serialized + assert '"top_upstream_errors":["other"]' in serialized + + +@pytest.mark.asyncio +async def test_success_rate_excludes_cancelled_terminals(async_session: AsyncSession) -> None: + async_session.add(_request_log("ok", model="gpt-5.4", useragent_group="codex_exec")) + async_session.add( + _request_log( + "cancel-1", + model="gpt-5.4", + useragent_group="codex_exec", + status="cancelled", + upstream_error_code="client_disconnected", + ) + ) + async_session.add( + _request_log( + "cancel-2", + model="gpt-5.4", + useragent_group="codex_exec", + status="cancelled", + upstream_error_code="client_disconnected", + ) + ) + async_session.add( + _request_log( + "err", + model="gpt-5.4", + useragent_group="codex_exec", + status="error", + upstream_error_code="server_error", + ) + ) + await async_session.commit() + + snapshot = await TelemetrySnapshotBuilder(async_session).build( + "00000000-0000-4000-8000-000000000004", + consent="undecided", + ) + + # 1 success out of 4 requests: cancellations are neither successes nor + # errors, so they must not inflate the numerator. + assert snapshot.usage_7d.success_rate == 0.25 + + +@pytest.mark.asyncio +async def test_top_upstream_errors_exclude_cancelled_terminals(async_session: AsyncSession) -> None: + for index in range(3): + async_session.add( + _request_log( + f"cancel-{index}", + model="gpt-5.4", + useragent_group="codex_exec", + status="cancelled", + upstream_error_code="client_disconnected", + ) + ) + async_session.add( + _request_log( + "err", + model="gpt-5.4", + useragent_group="codex_exec", + status="error", + upstream_error_code="server_error", + ) + ) + await async_session.commit() + + snapshot = await TelemetrySnapshotBuilder(async_session).build( + "00000000-0000-4000-8000-000000000005", + consent="undecided", + ) + + # High-volume disconnects (status='cancelled' with a retained + # client_disconnected code) must not displace genuine upstream failures. + assert snapshot.usage_7d.top_upstream_errors == ["server_error"] diff --git a/tests/unit/test_timeout_invariants.py b/tests/unit/test_timeout_invariants.py new file mode 100644 index 0000000000..49ca18b80c --- /dev/null +++ b/tests/unit/test_timeout_invariants.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import logging +from types import SimpleNamespace + +import pytest + +from app.core.config.settings import Settings, get_settings +from app.core.timeout_invariants import ( + TIMEOUT_INVARIANT_RULES, + TimeoutInvariantError, + find_timeout_invariant_violations, + main, + validate_runtime_timeout_invariants, + validate_timeout_invariants, +) +from app.modules.proxy import durable_bridge_repository +from app.modules.proxy._service.http_bridge import retry_circuit + +pytestmark = pytest.mark.unit + + +def test_default_settings_satisfy_timeout_invariants() -> None: + settings = Settings() + assert len(TIMEOUT_INVARIANT_RULES) == 8 + assert find_timeout_invariant_violations(settings) == [] + + +def _timeout_settings(**overrides: float | bool) -> SimpleNamespace: + settings = Settings() + values = { + name: getattr(settings, name) + for name in ( + "upstream_connect_timeout_seconds", + "proxy_request_budget_seconds", + "http_responses_stream_request_budget_seconds", + "compact_request_budget_seconds", + "stream_idle_timeout_seconds", + "sse_keepalive_interval_seconds", + "usage_fetch_timeout_seconds", + "usage_refresh_interval_seconds", + "rate_limit_reset_credits_refresh_interval_seconds", + "http_responses_session_bridge_request_budget_seconds", + "http_responses_session_bridge_idle_ttl_seconds", + "http_responses_session_bridge_codex_idle_ttl_seconds", + "http_responses_session_bridge_stuck_gate_retire_after_seconds", + "http_responses_session_bridge_clean_close_retry_jitter_max_seconds", + "proxy_admission_wait_timeout_seconds", + "proxy_account_lease_ttl_seconds", + "proxy_refresh_failure_cooldown_seconds", + "model_registry_enabled", + "model_registry_snapshot_max_age_seconds", + "timeout_invariant_validation_strict", + ) + } + values.update(overrides) + return SimpleNamespace(**values) + + +@pytest.mark.parametrize( + ("rule_id", "overrides"), + [ + ("admission-wait-within-proxy-budget", {"proxy_request_budget_seconds": 9.0}), + ("admission-wait-within-stream-budget", {"http_responses_stream_request_budget_seconds": 9.0}), + ("admission-wait-within-compact-budget", {"compact_request_budget_seconds": 9.0}), + ( + "bridge-stuck-gate-retire-within-bridge-budget", + {"http_responses_session_bridge_request_budget_seconds": 600.0}, + ), + ("account-lease-ttl-covers-proxy-budget", {"proxy_account_lease_ttl_seconds": 599.0}), + ("account-lease-ttl-covers-compact-budget", {"proxy_account_lease_ttl_seconds": 179.0}), + ( + "model-registry-snapshot-outlives-refresh-interval", + {"model_registry_enabled": True, "model_registry_snapshot_max_age_seconds": 300.0}, + ), + ], +) +def test_each_settings_backed_rule_names_violation(rule_id: str, overrides: dict[str, float]) -> None: + settings = _timeout_settings(**overrides) + + violations = find_timeout_invariant_violations(settings) + + assert any(violation.rule.id == rule_id for violation in violations) + formatted = "\n".join(violation.format() for violation in violations) + assert rule_id in formatted + + +def test_disabled_model_registry_skips_snapshot_cadence_rule() -> None: + settings = _timeout_settings( + model_registry_enabled=False, + model_registry_snapshot_max_age_seconds=1.0, + ) + + violations = find_timeout_invariant_violations(settings) + + assert all(violation.rule.id != "model-registry-snapshot-outlives-refresh-interval" for violation in violations) + + +def test_durable_bridge_retry_circuit_rule_names_violation(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + durable_bridge_repository, + "DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS", + retry_circuit._HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS + + retry_circuit._HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS + - 1.0, + ) + + violations = find_timeout_invariant_violations(Settings()) + + rule_id = "durable-bridge-retry-circuit-ttl-covers-backoff-and-half-open" + assert any(violation.rule.id == rule_id for violation in violations) + assert rule_id in "\n".join(violation.format() for violation in violations) + + +def test_non_strict_startup_validation_logs_critical(caplog: pytest.LogCaptureFixture) -> None: + settings = Settings(proxy_request_budget_seconds=5.0) + + with caplog.at_level(logging.CRITICAL, logger="app.core.timeout_invariants"): + violations = validate_runtime_timeout_invariants(settings) + + assert violations + assert "timeout invariant violation: admission-wait-within-proxy-budget" in caplog.text + + +def test_strict_mode_raises() -> None: + settings = Settings( + proxy_request_budget_seconds=5.0, + timeout_invariant_validation_strict=True, + ) + + with pytest.raises(TimeoutInvariantError) as exc_info: + validate_runtime_timeout_invariants(settings) + + assert "admission-wait-within-proxy-budget" in str(exc_info.value) + + +def test_explicit_strict_validation_raises() -> None: + settings = Settings(proxy_request_budget_seconds=5.0) + + with pytest.raises(TimeoutInvariantError, match="admission-wait-within-proxy-budget"): + validate_timeout_invariants(settings, strict=True, log=False) + + +def test_cli_entrypoint_accepts_defaults(capsys: pytest.CaptureFixture[str]) -> None: + assert main([]) == 0 + captured = capsys.readouterr() + assert "timeout invariant rules satisfied" in captured.out + + +def test_cli_strict_flag_exits_one_and_reports_rule( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + get_settings.cache_clear() + monkeypatch.setenv("CODEX_LB_PROXY_REQUEST_BUDGET_SECONDS", "5") + try: + assert main(["--strict"]) == 1 + finally: + get_settings.cache_clear() + + captured = capsys.readouterr() + assert "admission-wait-within-proxy-budget" in captured.err + + +def test_cli_without_strict_exits_zero_and_reports_violation( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + capsys: pytest.CaptureFixture[str], +) -> None: + get_settings.cache_clear() + monkeypatch.setenv("CODEX_LB_PROXY_REQUEST_BUDGET_SECONDS", "5") + try: + with caplog.at_level(logging.CRITICAL, logger="app.core.timeout_invariants"): + assert main([]) == 0 + finally: + get_settings.cache_clear() + + captured = capsys.readouterr() + assert "admission-wait-within-proxy-budget" in captured.err + assert "timeout invariant violation: admission-wait-within-proxy-budget" in caplog.text diff --git a/tests/unit/test_usage_refresh_scheduler_recovery.py b/tests/unit/test_usage_refresh_scheduler_recovery.py index f4ebb89d95..b43ea35e19 100644 --- a/tests/unit/test_usage_refresh_scheduler_recovery.py +++ b/tests/unit/test_usage_refresh_scheduler_recovery.py @@ -65,6 +65,111 @@ def _epoch_to_naive_utc(epoch: float) -> datetime: return datetime.fromtimestamp(epoch, timezone.utc).replace(tzinfo=None) +def _reset_evidence( + before: UsageHistory, + after: UsageHistory, + *, + baseline: UsageHistory | None = None, +): + return refresh_scheduler_module._MonthlyResetEvidence( + baseline=baseline or before, + before=before, + after=after, + ) + + +def test_historical_reset_recovery_scans_adjacent_sliding_samples() -> None: + now = 1_700_000_000 + legacy_reset_at = now + 7 * 24 * 60 * 60 + transition_recorded_at = now - 120 + history = [ + _make_usage( + "acc_free_history", + window="monthly", + used_percent=used_percent, + reset_at=reset_at, + recorded_at=_epoch_to_naive_utc(recorded_at), + window_minutes=43_200, + ) + for used_percent, reset_at, recorded_at in ( + (100.0, legacy_reset_at, now - 300), + (100.0, legacy_reset_at + 60, now - 240), + (100.0, legacy_reset_at + 120, now - 180), + (0.0, transition_recorded_at + 43_200 * 60, transition_recorded_at), + (0.0, now - 60 + 43_200 * 60, now - 60), + ) + ] + + evidence = refresh_scheduler_module._latest_confirmed_reset_transition_after_baseline( + history, + expected_reset_at=legacy_reset_at, + reset_at_tolerance_seconds=5, + ) + + assert evidence is not None + assert evidence.baseline is history[0] + assert (evidence.before, evidence.after) == (history[2], history[3]) + + +def test_historical_reset_recovery_fails_closed_without_matching_baseline() -> None: + now = 1_700_000_000 + history = [ + _make_usage( + "acc_free_no_baseline", + window="monthly", + used_percent=100.0, + reset_at=now + 60, + recorded_at=_epoch_to_naive_utc(now - 60), + window_minutes=43_200, + ), + _make_usage( + "acc_free_no_baseline", + window="monthly", + used_percent=0.0, + reset_at=now + 43_200 * 60, + recorded_at=_epoch_to_naive_utc(now), + window_minutes=43_200, + ), + ] + + evidence = refresh_scheduler_module._latest_confirmed_reset_transition_after_baseline( + history, + expected_reset_at=now + 7 * 24 * 60 * 60, + reset_at_tolerance_seconds=5, + ) + + assert evidence is None + + +def test_historical_reset_recovery_never_skips_an_exhausted_successor() -> None: + now = 1_700_000_000 + legacy_reset_at = now + 7 * 24 * 60 * 60 + next_reset_at = now + 30 * 24 * 60 * 60 + history = [ + _make_usage( + "acc_free_exhausted_successor", + window="monthly", + used_percent=used_percent, + reset_at=reset_at, + recorded_at=_epoch_to_naive_utc(now + offset), + window_minutes=43_200, + ) + for offset, used_percent, reset_at in ( + (0, 100.0, legacy_reset_at), + (60, 100.0, next_reset_at), + (120, 0.0, next_reset_at), + ) + ] + + evidence = refresh_scheduler_module._latest_confirmed_reset_transition_after_baseline( + history, + expected_reset_at=legacy_reset_at, + reset_at_tolerance_seconds=5, + ) + + assert evidence is None + + class StubAccountsRepository: def __init__(self, accounts: list[Account]) -> None: self._accounts = {account.id: account for account in accounts} @@ -248,6 +353,386 @@ async def test_reconcile_recoverable_account_statuses_keeps_rate_limited_until_r assert accounts_repo.status_updates == [] +@pytest.mark.asyncio +async def test_reconcile_recovers_free_after_confirmed_monthly_reset_before_legacy_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + now = 1_700_000_000.0 + blocked_at = int(now - 3600) + legacy_reset_at = int(now + 7 * 24 * 3600) + previous_monthly_reset = legacy_reset_at + next_monthly_reset = int(now - 60 + 30 * 24 * 3600) + monkeypatch.setattr("app.modules.proxy.load_balancer.time.time", lambda: now) + monkeypatch.setattr("app.core.usage.quota.time.time", lambda: now) + monkeypatch.setattr(refresh_scheduler_module.time, "time", lambda: now) + + account = _make_account( + "acc_free_confirmed_reset", + status=AccountStatus.RATE_LIMITED, + plan_type="free", + reset_at=legacy_reset_at, + blocked_at=blocked_at, + ) + before = _make_usage( + account.id, + window="monthly", + used_percent=100.0, + reset_at=previous_monthly_reset, + recorded_at=_epoch_to_naive_utc(now - 120), + window_minutes=43200, + ) + after = _make_usage( + account.id, + window="monthly", + used_percent=0.0, + reset_at=next_monthly_reset, + recorded_at=_epoch_to_naive_utc(now - 60), + window_minutes=43200, + ) + + recovered = await refresh_scheduler_module.reconcile_recoverable_account_statuses( + accounts_repo=StubAccountsRepository([account]), + usage_repo=StubUsageRepository( + primary={ + account.id: _make_usage( + account.id, + window="primary", + used_percent=100.0, + reset_at=legacy_reset_at, + recorded_at=_epoch_to_naive_utc(now - 1), + window_minutes=300, + ) + }, + monthly={account.id: after}, + ), + accounts=[account], + monthly_reset_evidence={account.id: _reset_evidence(before, after)}, + ) + + assert recovered == 1 + assert (account.status, account.reset_at, account.blocked_at) == (AccountStatus.ACTIVE, None, None) + + +@pytest.mark.asyncio +async def test_confirmed_monthly_reset_recovery_loses_cas_to_newer_marker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + now = 1_700_000_000.0 + blocked_at = int(now - 3600) + legacy_reset_at = int(now + 7 * 24 * 3600) + monkeypatch.setattr("app.modules.proxy.load_balancer.time.time", lambda: now) + monkeypatch.setattr("app.core.usage.quota.time.time", lambda: now) + monkeypatch.setattr(refresh_scheduler_module.time, "time", lambda: now) + + account = _make_account( + "acc_free_confirmed_reset_cas", + status=AccountStatus.RATE_LIMITED, + plan_type="free", + reset_at=legacy_reset_at, + blocked_at=blocked_at, + ) + before = _make_usage( + account.id, + window="monthly", + used_percent=100.0, + reset_at=legacy_reset_at, + recorded_at=_epoch_to_naive_utc(now - 120), + window_minutes=43200, + ) + after = _make_usage( + account.id, + window="monthly", + used_percent=0.0, + reset_at=int(now - 60 + 30 * 24 * 3600), + recorded_at=_epoch_to_naive_utc(now - 60), + window_minutes=43200, + ) + repo = MutatingAccountsRepository([account]) + + recovered = await refresh_scheduler_module.reconcile_recoverable_account_statuses( + accounts_repo=repo, + usage_repo=StubUsageRepository(monthly={account.id: after}), + accounts=[account], + monthly_reset_evidence={account.id: _reset_evidence(before, after)}, + ) + + assert recovered == 0 + assert repo.status_updates == [] + assert account.status == AccountStatus.RATE_LIMITED + assert account.reset_at == 42 + + +@pytest.mark.asyncio +async def test_confirmed_monthly_reset_recovery_honors_post_429_floor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + now = 1_700_000_000.0 + blocked_at = int(now - 10) + legacy_reset_at = int(now + 7 * 24 * 3600) + monkeypatch.setattr("app.modules.proxy.load_balancer.time.time", lambda: now) + monkeypatch.setattr("app.core.usage.quota.time.time", lambda: now) + monkeypatch.setattr(refresh_scheduler_module.time, "time", lambda: now) + + account = _make_account( + "acc_free_confirmed_reset_floor", + status=AccountStatus.RATE_LIMITED, + plan_type="free", + reset_at=legacy_reset_at, + blocked_at=blocked_at, + ) + before = _make_usage( + account.id, + window="monthly", + used_percent=100.0, + reset_at=legacy_reset_at, + recorded_at=_epoch_to_naive_utc(now - 9), + window_minutes=43200, + ) + after = _make_usage( + account.id, + window="monthly", + used_percent=0.0, + reset_at=int(now - 1 + 30 * 24 * 3600), + recorded_at=_epoch_to_naive_utc(now - 1), + window_minutes=43200, + ) + repo = StubAccountsRepository([account]) + + recovered = await refresh_scheduler_module.reconcile_recoverable_account_statuses( + accounts_repo=repo, + usage_repo=StubUsageRepository(monthly={account.id: after}), + accounts=[account], + monthly_reset_evidence={account.id: _reset_evidence(before, after)}, + ) + + assert recovered == 0 + assert account.status == AccountStatus.RATE_LIMITED + assert repo.status_updates == [] + + +@pytest.mark.asyncio +async def test_reconcile_keeps_free_blocked_without_confirmed_monthly_reset( + monkeypatch: pytest.MonkeyPatch, +) -> None: + now = 1_700_000_000.0 + blocked_at = int(now - 3600) + legacy_reset_at = int(now + 7 * 24 * 3600) + monthly_reset_at = legacy_reset_at + monkeypatch.setattr("app.modules.proxy.load_balancer.time.time", lambda: now) + monkeypatch.setattr("app.core.usage.quota.time.time", lambda: now) + monkeypatch.setattr(refresh_scheduler_module.time, "time", lambda: now) + + account = _make_account( + "acc_free_no_reset", + status=AccountStatus.RATE_LIMITED, + plan_type="free", + reset_at=legacy_reset_at, + blocked_at=blocked_at, + ) + before = _make_usage( + account.id, + window="monthly", + used_percent=0.0, + reset_at=monthly_reset_at, + recorded_at=_epoch_to_naive_utc(now - 120), + window_minutes=43200, + ) + after = _make_usage( + account.id, + window="monthly", + used_percent=0.0, + reset_at=monthly_reset_at + 60, + recorded_at=_epoch_to_naive_utc(now - 60), + window_minutes=43200, + ) + repo = StubAccountsRepository([account]) + + recovered = await refresh_scheduler_module.reconcile_recoverable_account_statuses( + accounts_repo=repo, + usage_repo=StubUsageRepository(monthly={account.id: after}), + accounts=[account], + monthly_reset_evidence={account.id: _reset_evidence(before, after)}, + ) + + assert recovered == 0 + assert (account.status, account.reset_at, account.blocked_at) == ( + AccountStatus.RATE_LIMITED, + legacy_reset_at, + blocked_at, + ) + assert repo.status_updates == [] + + +@pytest.mark.asyncio +async def test_reconcile_keeps_free_blocked_when_current_monthly_quota_is_exhausted( + monkeypatch: pytest.MonkeyPatch, +) -> None: + now = 1_700_000_000.0 + blocked_at = int(now - 3600) + legacy_reset_at = int(now + 7 * 24 * 3600) + monkeypatch.setattr("app.modules.proxy.load_balancer.time.time", lambda: now) + monkeypatch.setattr("app.core.usage.quota.time.time", lambda: now) + monkeypatch.setattr(refresh_scheduler_module.time, "time", lambda: now) + + account = _make_account( + "acc_free_current_exhausted", + status=AccountStatus.RATE_LIMITED, + plan_type="free", + reset_at=legacy_reset_at, + blocked_at=blocked_at, + ) + before = _make_usage( + account.id, + window="monthly", + used_percent=100.0, + reset_at=legacy_reset_at, + recorded_at=_epoch_to_naive_utc(now - 120), + window_minutes=43200, + ) + reset_sample = _make_usage( + account.id, + window="monthly", + used_percent=0.0, + reset_at=int(now - 60 + 30 * 24 * 3600), + recorded_at=_epoch_to_naive_utc(now - 60), + window_minutes=43200, + ) + current = _make_usage( + account.id, + window="monthly", + used_percent=100.0, + reset_at=reset_sample.reset_at or 0, + recorded_at=_epoch_to_naive_utc(now - 1), + window_minutes=43200, + ) + repo = StubAccountsRepository([account]) + + recovered = await refresh_scheduler_module.reconcile_recoverable_account_statuses( + accounts_repo=repo, + usage_repo=StubUsageRepository(monthly={account.id: current}), + accounts=[account], + monthly_reset_evidence={account.id: _reset_evidence(before, reset_sample)}, + ) + + assert recovered == 0 + assert account.status == AccountStatus.RATE_LIMITED + assert repo.status_updates == [] + + +@pytest.mark.asyncio +async def test_reconcile_keeps_free_blocked_when_matching_baseline_predates_block( + monkeypatch: pytest.MonkeyPatch, +) -> None: + now = 1_700_000_000.0 + blocked_at = int(now - 90) + legacy_reset_at = int(now + 7 * 24 * 3600) + monkeypatch.setattr("app.modules.proxy.load_balancer.time.time", lambda: now) + monkeypatch.setattr("app.core.usage.quota.time.time", lambda: now) + monkeypatch.setattr(refresh_scheduler_module.time, "time", lambda: now) + + account = _make_account( + "acc_free_pre_block_baseline", + status=AccountStatus.RATE_LIMITED, + plan_type="free", + reset_at=legacy_reset_at, + blocked_at=blocked_at, + ) + baseline = _make_usage( + account.id, + window="monthly", + used_percent=100.0, + reset_at=legacy_reset_at, + recorded_at=_epoch_to_naive_utc(blocked_at - 1), + window_minutes=43_200, + ) + after = _make_usage( + account.id, + window="monthly", + used_percent=0.0, + reset_at=int(now - 60 + 43_200 * 60), + recorded_at=_epoch_to_naive_utc(now - 60), + window_minutes=43_200, + ) + repo = StubAccountsRepository([account]) + + recovered = await refresh_scheduler_module.reconcile_recoverable_account_statuses( + accounts_repo=repo, + usage_repo=StubUsageRepository(monthly={account.id: after}), + accounts=[account], + monthly_reset_evidence={account.id: _reset_evidence(baseline, after)}, + ) + + assert recovered == 0 + assert (account.status, account.reset_at, account.blocked_at) == ( + AccountStatus.RATE_LIMITED, + legacy_reset_at, + blocked_at, + ) + assert repo.status_updates == [] + + +@pytest.mark.asyncio +async def test_reconcile_does_not_apply_monthly_reset_override_to_plus( + monkeypatch: pytest.MonkeyPatch, +) -> None: + now = 1_700_000_000.0 + blocked_at = int(now - 3600) + legacy_reset_at = int(now + 7 * 24 * 3600) + monkeypatch.setattr("app.modules.proxy.load_balancer.time.time", lambda: now) + monkeypatch.setattr("app.core.usage.quota.time.time", lambda: now) + monkeypatch.setattr(refresh_scheduler_module.time, "time", lambda: now) + + account = _make_account( + "acc_plus_monthly_reset", + status=AccountStatus.RATE_LIMITED, + plan_type="plus", + reset_at=legacy_reset_at, + blocked_at=blocked_at, + ) + before = _make_usage( + account.id, + window="monthly", + used_percent=100.0, + reset_at=legacy_reset_at, + recorded_at=_epoch_to_naive_utc(now - 120), + window_minutes=43200, + ) + after = _make_usage( + account.id, + window="monthly", + used_percent=0.0, + reset_at=int(now - 60 + 30 * 24 * 3600), + recorded_at=_epoch_to_naive_utc(now - 60), + window_minutes=43200, + ) + + recovered = await refresh_scheduler_module.reconcile_recoverable_account_statuses( + accounts_repo=StubAccountsRepository([account]), + usage_repo=StubUsageRepository( + primary={ + account.id: _make_usage( + account.id, + window="primary", + used_percent=100.0, + reset_at=legacy_reset_at, + recorded_at=_epoch_to_naive_utc(now - 1), + window_minutes=300, + ) + }, + monthly={account.id: after}, + ), + accounts=[account], + monthly_reset_evidence={account.id: _reset_evidence(before, after)}, + ) + + assert recovered == 0 + assert (account.status, account.reset_at, account.blocked_at) == ( + AccountStatus.RATE_LIMITED, + legacy_reset_at, + blocked_at, + ) + + @pytest.mark.asyncio async def test_reconcile_recoverable_account_statuses_restores_rate_limited_after_reset_elapses( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_usage_snapshot_repository.py b/tests/unit/test_usage_snapshot_repository.py index 117ab1142d..10b696fa32 100644 --- a/tests/unit/test_usage_snapshot_repository.py +++ b/tests/unit/test_usage_snapshot_repository.py @@ -3,13 +3,16 @@ from collections.abc import AsyncIterator, Collection from contextlib import asynccontextmanager from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock import pytest from sqlalchemy import event, func, select +from sqlalchemy.dialects import postgresql from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from app.db.models import Account, AccountStatus, Base, UsageHistory from app.modules.usage import background_repository as background_repository_module +from app.modules.usage import repository as usage_repository_module from app.modules.usage.background_repository import BackgroundUsageRepository from app.modules.usage.repository import UsageRepository, UsageWindowWrite @@ -26,6 +29,43 @@ async def session_factory() -> AsyncIterator[async_sessionmaker[AsyncSession]]: await engine.dispose() +@pytest.mark.asyncio +async def test_postgresql_settlement_lookups_compile_for_no_key_update( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = MagicMock() + session.get_bind.return_value.dialect.name = "postgresql" + observed_result = MagicMock() + observed_result.one_or_none.return_value = MagicMock( + id="acc_current", + chatgpt_account_id="workspace-current", + ) + locked_result = MagicMock() + locked_result.one_or_none.return_value = MagicMock( + id="acc_current", + chatgpt_account_id="workspace-current", + ) + session.execute = AsyncMock(side_effect=[observed_result, locked_result]) + session.add_all = MagicMock() + session.commit = AsyncMock() + session.rollback = AsyncMock() + monkeypatch.setattr(usage_repository_module, "lock_postgresql_account_identities", AsyncMock()) + monkeypatch.setattr(usage_repository_module, "relax_commit_durability", AsyncMock()) + + resolved = await UsageRepository(session).settle_live_account_snapshot( + account_id="acc_stale", + chatgpt_account_id="workspace-current", + windows=[UsageWindowWrite(window="primary", used_percent=25.0)], + should_skip=lambda _account_id: False, + ) + + assert resolved == "acc_current" + observed_stmt = session.execute.await_args_list[0].args[0] + locked_stmt = session.execute.await_args_list[1].args[0] + assert "FOR NO KEY UPDATE" not in str(observed_stmt.compile(dialect=postgresql.dialect())) + assert "FOR NO KEY UPDATE" in str(locked_stmt.compile(dialect=postgresql.dialect())) + + def _account(account_id: str) -> Account: return Account( id=account_id, diff --git a/tests/unit/test_usage_updater.py b/tests/unit/test_usage_updater.py index bf624e747e..069b68e96a 100644 --- a/tests/unit/test_usage_updater.py +++ b/tests/unit/test_usage_updater.py @@ -17,6 +17,7 @@ from app.core.usage import refresh_scheduler as refresh_scheduler_module from app.core.usage.models import UsagePayload from app.core.usage.refresh_scheduler import _select_long_window_entries +from app.core.utils.shared_future import _WAITERS_ATTR, wait_on_shared_future from app.core.utils.time import utcnow from app.db.models import Account, AccountStatus, UsageHistory from app.modules.usage import updater as usage_updater_module @@ -77,6 +78,162 @@ async def factory(): assert usage_updater_module._USAGE_REFRESH_SINGLEFLIGHT._inflight == {} +@pytest.mark.asyncio +async def test_usage_refresh_singleflight_concurrent_waiters_share_result() -> None: + singleflight = usage_updater_module._UsageRefreshSingleflight() + started = asyncio.Event() + release = asyncio.Event() + result = usage_updater_module.AccountRefreshResult(usage_written=True) + factory_calls = 0 + + async def factory() -> usage_updater_module.AccountRefreshResult: + nonlocal factory_calls + factory_calls += 1 + started.set() + await release.wait() + return result + + waiters = [asyncio.create_task(singleflight.run("acc_shared_result", factory)) for _ in range(50)] + await asyncio.wait_for(started.wait(), timeout=1) + release.set() + + results = await asyncio.gather(*waiters) + + assert factory_calls == 1 + assert all(item is result for item in results) + + +@pytest.mark.asyncio +async def test_usage_refresh_singleflight_waiter_cancellation_leaves_factory_running() -> None: + singleflight = usage_updater_module._UsageRefreshSingleflight() + started = asyncio.Event() + release = asyncio.Event() + factory_cancelled = asyncio.Event() + result = usage_updater_module.AccountRefreshResult(usage_written=True) + + async def factory() -> usage_updater_module.AccountRefreshResult: + started.set() + try: + await release.wait() + except asyncio.CancelledError: + factory_cancelled.set() + raise + return result + + waiters = [asyncio.create_task(singleflight.run("acc_cancel_waiters", factory)) for _ in range(20)] + await asyncio.wait_for(started.wait(), timeout=1) + await asyncio.sleep(0) + inflight = singleflight._inflight["acc_cancel_waiters"] + + for waiter in waiters[:-1]: + waiter.cancel() + cancelled = await asyncio.gather(*waiters[:-1], return_exceptions=True) + + assert all(isinstance(item, asyncio.CancelledError) for item in cancelled) + assert not inflight.done() + assert not factory_cancelled.is_set() + + release.set() + assert await waiters[-1] is result + assert not factory_cancelled.is_set() + + +@pytest.mark.asyncio +async def test_usage_refresh_singleflight_cancelled_waiters_keep_callback_fanout_bounded() -> None: + singleflight = usage_updater_module._UsageRefreshSingleflight() + started = asyncio.Event() + release = asyncio.Event() + result = usage_updater_module.AccountRefreshResult(usage_written=False) + + async def factory() -> usage_updater_module.AccountRefreshResult: + started.set() + await release.wait() + return result + + waiters = [asyncio.create_task(singleflight.run("acc_callback_fanout", factory)) for _ in range(100)] + await asyncio.wait_for(started.wait(), timeout=1) + inflight = singleflight._inflight["acc_callback_fanout"] + for _ in range(10): + if len(getattr(inflight, _WAITERS_ATTR, set())) == len(waiters): + break + await asyncio.sleep(0) + + callbacks = getattr(inflight, "_callbacks", None) + assert callbacks is not None and len(callbacks) == 2, ( + "usage-refresh waiters must share one fan-out callback in addition to " + f"singleflight cleanup; found {None if callbacks is None else len(callbacks)} callbacks" + ) + assert len(getattr(inflight, _WAITERS_ATTR)) == len(waiters) + + for waiter in waiters: + waiter.cancel() + cancelled = await asyncio.gather(*waiters, return_exceptions=True) + await asyncio.sleep(0) + + assert all(isinstance(item, asyncio.CancelledError) for item in cancelled) + assert not inflight.done() + callbacks = getattr(inflight, "_callbacks", None) + assert callbacks is not None and len(callbacks) == 2 + assert getattr(inflight, _WAITERS_ATTR) == set() + + release.set() + assert await inflight is result + + +@pytest.mark.asyncio +async def test_usage_refresh_singleflight_non_joiner_waits_then_starts_successor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + singleflight = usage_updater_module._UsageRefreshSingleflight() + first_started = asyncio.Event() + release_first = asyncio.Event() + successor_started = asyncio.Event() + release_successor = asyncio.Event() + first_result = usage_updater_module.AccountRefreshResult(usage_written=False) + successor_result = usage_updater_module.AccountRefreshResult(usage_written=True) + shared_waits: list[asyncio.Future[usage_updater_module.AccountRefreshResult]] = [] + + async def recording_wait( + shared: asyncio.Future[usage_updater_module.AccountRefreshResult], + *, + timeout: float | None = None, + ) -> usage_updater_module.AccountRefreshResult: + shared_waits.append(shared) + return await wait_on_shared_future(shared, timeout=timeout) + + async def first_factory() -> usage_updater_module.AccountRefreshResult: + first_started.set() + await release_first.wait() + return first_result + + async def successor_factory() -> usage_updater_module.AccountRefreshResult: + successor_started.set() + await release_successor.wait() + return successor_result + + monkeypatch.setattr(usage_updater_module, "wait_on_shared_future", recording_wait) + first_waiter = asyncio.create_task(singleflight.run("acc_non_joiner", first_factory)) + await asyncio.wait_for(first_started.wait(), timeout=1) + first_task = singleflight._inflight["acc_non_joiner"] + non_joiner = asyncio.create_task( + singleflight.run("acc_non_joiner", successor_factory, join_existing=False), + ) + await asyncio.sleep(0) + + assert not successor_started.is_set() + assert shared_waits.count(first_task) == 2 + + release_first.set() + assert await first_waiter is first_result + await asyncio.wait_for(successor_started.wait(), timeout=1) + successor_task = singleflight._inflight["acc_non_joiner"] + assert successor_task is not first_task + + release_successor.set() + assert await non_joiner is successor_result + assert successor_task in shared_waits + + @pytest.mark.asyncio async def test_refresh_accounts_owned_singleflight_session_outlives_caller_cancellation( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_websocket_terminal_cancellation.py b/tests/unit/test_websocket_terminal_cancellation.py index 5d2665c35b..b33d5b6fd5 100644 --- a/tests/unit/test_websocket_terminal_cancellation.py +++ b/tests/unit/test_websocket_terminal_cancellation.py @@ -152,6 +152,7 @@ async def test_transport_end_replay_requires_send_boundary_only_for_direct_webso @pytest.mark.asyncio async def test_cancelled_websocket_scope_cleanup_is_deadline_bounded_and_remains_drain_owned( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ) -> None: @asynccontextmanager async def repo_factory() -> AsyncIterator[SimpleNamespace]: @@ -163,27 +164,94 @@ async def repo_factory() -> AsyncIterator[SimpleNamespace]: sticky_threads_enabled=False, openai_cache_affinity_max_age_seconds=0, prohibit_fast_mode=False, + proxy_downstream_websocket_idle_timeout_seconds=30.0, + proxy_request_budget_seconds=30.0, + stream_idle_timeout_seconds=30.0, + sse_keepalive_interval_seconds=0.0, ) class _SettingsCache: async def get(self) -> SimpleNamespace: return settings - receive_started = asyncio.Event() + request_text = json.dumps( + { + "type": "response.create", + "model": "gpt-5.6-sol", + "input": "pending cleanup", + }, + separators=(",", ":"), + ) + request_state = _request_state("request_pending_cleanup") + request_state.request_text = request_text + request_sent = asyncio.Event() cleanup_started = asyncio.Event() cleanup_cancelled = asyncio.Event() release_cleanup = asyncio.Event() + cleanup_request_ids: list[str] = [] class _BlockingDownstreamWebSocket: + def __init__(self) -> None: + self._received = False + async def receive(self) -> dict[str, object]: - receive_started.set() + if not self._received: + self._received = True + return {"type": "websocket.receive", "text": request_text} await asyncio.Event().wait() raise AssertionError("unreachable") + async def send_text(self, _text: str) -> None: + return None + + async def send_bytes(self, _data: bytes) -> None: + return None + async def close(self, code: int = 1000, reason: str | None = None) -> None: del code, reason - async def block_cleanup(*_args: object, **_kwargs: object) -> None: + class _PendingUpstream: + async def send_text(self, _text: str) -> None: + request_sent.set() + + async def send_bytes(self, _data: bytes) -> None: + raise AssertionError("binary send is not expected") + + async def close(self) -> None: + return None + + upstream = _PendingUpstream() + + async def prepare_request(*_args: object, **_kwargs: object) -> proxy_service._PreparedWebSocketRequest: + return proxy_service._PreparedWebSocketRequest( + text_data=request_text, + request_state=request_state, + affinity_policy=proxy_service._AffinityPolicy(), + ) + + async def acquire_admission( + state: proxy_service._WebSocketRequestState, + *, + response_create_gate: asyncio.Semaphore, + ) -> None: + state.response_create_gate = response_create_gate + await response_create_gate.acquire() + state.response_create_gate_acquired = True + state.awaiting_response_created = True + + async def connect_upstream(*_args: object, **_kwargs: object) -> tuple[Account, UpstreamWebSocket]: + account = cast(Account, SimpleNamespace(id="account_pending_cleanup", codex_installation_id=None)) + return account, cast(UpstreamWebSocket, upstream) + + async def relay_until_cancelled(*_args: object, **_kwargs: object) -> None: + await asyncio.Event().wait() + + async def block_cleanup( + *_args: object, + pending_requests: deque[proxy_service._WebSocketRequestState], + **_kwargs: object, + ) -> None: + cleanup_request_ids.extend(state.request_id for state in pending_requests) cleanup_started.set() try: await release_cleanup.wait() @@ -192,13 +260,17 @@ async def block_cleanup(*_args: object, **_kwargs: object) -> None: raise monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache()) - monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: SimpleNamespace(proxy_downstream_websocket_idle_timeout_seconds=30.0), - ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) monkeypatch.setattr(proxy_service, "_routing_strategy", lambda _settings: "usage_weighted") + monkeypatch.setattr(proxy_service, "_enforce_response_create_size_limit", lambda _request_state: None) + monkeypatch.setattr(websocket_mixin, "effective_account_concurrency_caps", lambda _settings: object()) monkeypatch.setattr(service, "_websocket_continuity_state_for_request", lambda *_args, **_kwargs: None) + monkeypatch.setattr(service, "_prepare_websocket_response_create_request", prepare_request) + monkeypatch.setattr(service, "_start_request_state_api_key_reservation_heartbeat", lambda *_args, **_kwargs: None) + monkeypatch.setattr(service, "_acquire_request_state_response_create_admission", acquire_admission) + monkeypatch.setattr(service, "_connect_proxy_websocket", connect_upstream) + monkeypatch.setattr(service, "_relay_upstream_websocket_messages", relay_until_cancelled) + monkeypatch.setattr(service, "_acquire_account_response_create_lease_or_overload", AsyncMock(return_value=object())) monkeypatch.setattr(service, "_fail_pending_websocket_requests", block_cleanup) monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) @@ -211,8 +283,9 @@ async def block_cleanup(*_args: object, **_kwargs: object) -> None: api_key=None, ) ) - await asyncio.wait_for(receive_started.wait(), timeout=1) + await asyncio.wait_for(request_sent.wait(), timeout=1) + caplog.set_level(logging.WARNING) shutdown_state.commit_shutdown(timeout_seconds=0.1) started_at = asyncio.get_running_loop().time() scope_task.cancel() @@ -225,12 +298,17 @@ async def block_cleanup(*_args: object, **_kwargs: object) -> None: elapsed = asyncio.get_running_loop().time() - started_at assert cleanup_cancelled.is_set() is False + assert cleanup_request_ids == [request_state.request_id] assert 0.05 <= elapsed < 0.3 assert any( task.get_name() == "proxy-websocket-finalization-scope-cleanup" for task in service._background_cleanup_tasks if not task.done() ) + assert any( + "Websocket scope cleanup exceeded its cleanup budget" in message and "cleanup_phase=pending_requests" in message + for message in caplog.messages + ) persistence_drain = asyncio.create_task(service.drain_persistence_tasks(timeout_seconds=1)) await asyncio.sleep(0) @@ -240,6 +318,85 @@ async def block_cleanup(*_args: object, **_kwargs: object) -> None: assert service._background_cleanup_tasks == set() +@pytest.mark.asyncio +async def test_normal_websocket_scope_cleanup_uses_separate_scope_budget( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + @asynccontextmanager + async def repo_factory() -> AsyncIterator[SimpleNamespace]: + yield SimpleNamespace(request_logs=_RequestLogsRecorder(), api_keys=object()) + + service = proxy_service.ProxyService(cast(proxy_service.ProxyRepoFactory, repo_factory)) + settings = SimpleNamespace( + prefer_earlier_reset_accounts=False, + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=0, + prohibit_fast_mode=False, + ) + + class _SettingsCache: + async def get(self) -> SimpleNamespace: + return settings + + receive_started = asyncio.Event() + cleanup_started = asyncio.Event() + release_cleanup = asyncio.Event() + + class _BlockingDownstreamWebSocket: + async def receive(self) -> dict[str, object]: + receive_started.set() + await asyncio.Event().wait() + raise AssertionError("unreachable") + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + del code, reason + + async def block_cleanup(*_args: object, **_kwargs: object) -> None: + cleanup_started.set() + await release_cleanup.wait() + + monkeypatch.setattr(proxy_service, "_TASK_CANCEL_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(websocket_mixin, "_WEBSOCKET_SCOPE_CLEANUP_TIMEOUT_SECONDS", 0.08) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache()) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace(proxy_downstream_websocket_idle_timeout_seconds=30.0), + ) + monkeypatch.setattr(proxy_service, "_routing_strategy", lambda _settings: "usage_weighted") + monkeypatch.setattr(service, "_websocket_continuity_state_for_request", lambda *_args, **_kwargs: None) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", block_cleanup) + monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) + caplog.set_level(logging.WARNING) + + scope_task = asyncio.create_task( + service.proxy_responses_websocket( + cast(WebSocket, _BlockingDownstreamWebSocket()), + {}, + codex_session_affinity=False, + openai_cache_affinity=False, + api_key=None, + ) + ) + await asyncio.wait_for(receive_started.wait(), timeout=1) + + scope_task.cancel() + await asyncio.wait_for(cleanup_started.wait(), timeout=1) + await asyncio.sleep(0.03) + assert scope_task.done() is False + release_cleanup.set() + + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(scope_task, timeout=0.5) + await asyncio.sleep(0) + + assert not any( + message.startswith("Websocket scope cleanup exceeded its cleanup budget") for message in caplog.messages + ) + assert service._background_cleanup_tasks == set() + + @pytest.mark.asyncio @pytest.mark.parametrize( "failing_child", @@ -945,12 +1102,14 @@ async def _observed_archive_attribution( *, pending_requests: deque[proxy_service._WebSocketRequestState], pending_lock: anyio.Lock, + parsed_frame: object | None = None, ) -> str | None: archive_attribution_started.set() return await original_archive_attribution( message, pending_requests=pending_requests, pending_lock=pending_lock, + parsed_frame=cast("websocket_mixin._ParsedUpstreamWebSocketFrame | None", parsed_frame), ) async def _blocking_release_gate( @@ -1240,6 +1399,79 @@ async def owned_child() -> None: await asyncio.wait_for(child, timeout=1) +@pytest.mark.asyncio +async def test_stuck_upstream_close_is_cancelled_after_scope_cleanup_timeout() -> None: + @asynccontextmanager + async def repo_factory() -> AsyncIterator[SimpleNamespace]: + yield SimpleNamespace(request_logs=_RequestLogsRecorder(), api_keys=object()) + + service = proxy_service.ProxyService(cast(proxy_service.ProxyRepoFactory, repo_factory)) + close_started = asyncio.Event() + release_close = asyncio.Event() + close_cancelled = False + + async def close() -> None: + nonlocal close_cancelled + close_started.set() + try: + await release_close.wait() + except asyncio.CancelledError: + close_cancelled = True + raise + + upstream = cast(UpstreamWebSocket, SimpleNamespace(close=close)) + + cleanup = asyncio.create_task( + websocket_mixin._close_websocket_upstream_for_cleanup( + service, + upstream, + timeout_seconds=1.0, + ) + ) + await asyncio.wait_for(close_started.wait(), timeout=1) + await asyncio.wait_for(cleanup, timeout=1) + + assert close_cancelled is True + assert service._background_cleanup_tasks == set() + release_close.set() + + +@pytest.mark.asyncio +async def test_upstream_close_is_cancelled_when_cleanup_budget_is_exhausted() -> None: + @asynccontextmanager + async def repo_factory() -> AsyncIterator[SimpleNamespace]: + yield SimpleNamespace(request_logs=_RequestLogsRecorder(), api_keys=object()) + + service = proxy_service.ProxyService(cast(proxy_service.ProxyRepoFactory, repo_factory)) + close_started = asyncio.Event() + close_cancelled = False + + async def close() -> None: + nonlocal close_cancelled + close_started.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + close_cancelled = True + raise + + upstream = cast(UpstreamWebSocket, SimpleNamespace(close=close)) + + await websocket_mixin._close_websocket_upstream_for_cleanup( + service, + upstream, + timeout_seconds=0.0, + ) + + await asyncio.wait_for(close_started.wait(), timeout=1) + for _ in range(20): + if close_cancelled and not service._background_cleanup_tasks: + break + await asyncio.sleep(0) + assert close_cancelled is True + assert service._background_cleanup_tasks == set() + + @pytest.mark.asyncio @pytest.mark.parametrize("message_kind", ["text", "transport_end"]) async def test_reader_cancellation_remains_cancelled_when_owned_child_fails( diff --git a/uv.lock b/uv.lock index 902c566c5f..e6e2f58026 100644 --- a/uv.lock +++ b/uv.lock @@ -110,15 +110,15 @@ wheels = [ [[package]] name = "aiohttp-socks" -version = "0.11.0" +version = "0.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, { name = "python-socks" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1f/cc/e5bbd54f76bd56291522251e47267b645dac76327b2657ade9545e30522c/aiohttp_socks-0.11.0.tar.gz", hash = "sha256:0afe51638527c79077e4bd6e57052c87c4824233d6e20bb061c53766421b10f0", size = 11196, upload-time = "2025-12-09T13:35:52.564Z" } +sdist = { url = "https://files.pythonhosted.org/packages/18/1d/a306e0111222180e60f17131a3f5d9bc694dd999a8115959a7dd76c2238e/aiohttp_socks-0.12.0.tar.gz", hash = "sha256:3caf9f5a4164611122d412bc11b2f9114fd29c85e1ba27bb38060d3c236bdc8d", size = 12061, upload-time = "2026-08-12T04:43:15.791Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/7d/4b633d709b8901d59444d2e512b93e72fe62d2b492a040097c3f7ba017bb/aiohttp_socks-0.11.0-py3-none-any.whl", hash = "sha256:9aacce57c931b8fbf8f6d333cf3cafe4c35b971b35430309e167a35a8aab9ec1", size = 10556, upload-time = "2025-12-09T13:35:50.18Z" }, + { url = "https://files.pythonhosted.org/packages/86/64/ca6289632020523ea1841f01363f83ada10eeb0f21dd931fc1118dd85668/aiohttp_socks-0.12.0-py3-none-any.whl", hash = "sha256:ba6f95ec775c761d87f8578ab48f137d0457c676da104984202bf75e747d5ee6", size = 10693, upload-time = "2026-08-12T04:43:14.524Z" }, ] [[package]] @@ -144,16 +144,16 @@ wheels = [ [[package]] name = "alembic" -version = "1.19.0" +version = "1.19.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mako" }, { name = "sqlalchemy" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fb/01/a48dab7827ac4421272399f7ed9a2ec17edd12c8bcde4417bd7b6821b71a/alembic-1.19.0.tar.gz", hash = "sha256:6487c612fc719dcfa22b17d2dd5b2b458929641e6aa2f0b65b135727f5e6d501", size = 2069906, upload-time = "2026-08-04T18:57:04.599Z" } +sdist = { url = "https://files.pythonhosted.org/packages/16/2b/e4153978368de59918115c9e01d3ebf58a558a7285efa7e960c383c4b59a/alembic-1.19.1.tar.gz", hash = "sha256:e0fca0518118c78acc493e31bcb5402f190057aaf6df8b5b95ce94c4789cf648", size = 2070816, upload-time = "2026-08-08T16:32:01.565Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/34/79/59ab6f1fe72eee229de91aa393225389a85df12a0fbbbfa264efcf6d7872/alembic-1.19.0-py3-none-any.whl", hash = "sha256:cf839d3849116aab3cc047e09c6968b9bd6b2fde61b6bb7c1e97352fe5503580", size = 265738, upload-time = "2026-08-04T18:57:06.219Z" }, + { url = "https://files.pythonhosted.org/packages/20/89/e62cc37b69ad357cc8ecd6e7367f5245f523d3cbb338a66197212bdf6749/alembic-1.19.1-py3-none-any.whl", hash = "sha256:b39018cb3d9413a19cbd54cf3c02ad33998641f0538eb77413a488a21c3e14be", size = 265946, upload-time = "2026-08-08T16:32:03.153Z" }, ] [[package]] @@ -534,6 +534,7 @@ tracing = [ [package.dev-dependencies] dev = [ { name = "httpx" }, + { name = "hypothesis" }, { name = "openai" }, { name = "pre-commit" }, { name = "pytest" }, @@ -590,6 +591,7 @@ provides-extras = ["metrics", "tracing"] [package.metadata.requires-dev] dev = [ { name = "httpx", specifier = ">=0.28.1" }, + { name = "hypothesis", specifier = ">=6.165.3" }, { name = "openai", specifier = ">=2.16.0" }, { name = "pre-commit", specifier = ">=4.5.1" }, { name = "pytest", specifier = ">=9.0.2" }, @@ -598,7 +600,7 @@ dev = [ { name = "pytest-timeout", specifier = ">=2.4.0" }, { name = "pytest-xdist", specifier = ">=3.8.0" }, { name = "ruff", specifier = ">=0.14.13" }, - { name = "ty", specifier = "==0.0.69" }, + { name = "ty", specifier = "==0.0.73" }, ] docs = [{ name = "mkdocs-material", specifier = ">=9.6" }] @@ -739,15 +741,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, ] -[[package]] -name = "distro" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, -] - [[package]] name = "dnspython" version = "2.8.0" @@ -1011,59 +1004,59 @@ wheels = [ [[package]] name = "greenlet" -version = "3.5.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/74/b13368064b09053253555d3f2839cc2684d22d5aed0d2ccffbf7a6736558/greenlet-3.5.4.tar.gz", hash = "sha256:0232ae1de90a8e07867bb127d7a6ba2301e859145489f25cda8a6096dabe1d20", size = 206538, upload-time = "2026-07-22T12:47:14.468Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/9a/e51225dcd58713f16ccbdcc501a8da21098ea14515b7870f1f94459e5ff5/greenlet-3.5.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:24e61b88cb7e1b1d794b32a10cc346ac779681d6d74ff137a3e0a444d2bf1f02", size = 294831, upload-time = "2026-07-22T11:38:53.389Z" }, - { url = "https://files.pythonhosted.org/packages/9f/ea/de50a50fadf979713ab18b46f22ad5ff5f2dcfc637a3ebdecf669801e1a5/greenlet-3.5.4-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:870d730fec833f5a06906a32596cc099b9161594642a92a520b7a88911c95356", size = 614619, upload-time = "2026-07-22T12:26:42.282Z" }, - { url = "https://files.pythonhosted.org/packages/db/c7/2aae27fea41205b8650294c301f042a2a4bb6155eea48c995b890a92f2c1/greenlet-3.5.4-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ec5ff0d1878df6af3bf9b638a5a92a7d5693291de77c91bff10fa48519c604ef", size = 627021, upload-time = "2026-07-22T12:29:03.445Z" }, - { url = "https://files.pythonhosted.org/packages/1b/80/fb4d4788bbc8e54761f1fc88533af9523a6e86299fa113d6e8a8503ed9fc/greenlet-3.5.4-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:07bd44616608d873d06735b63ef1a88191d6ca57c8d291d6559c71bc14c0893c", size = 632845, upload-time = "2026-07-22T12:43:45.19Z" }, - { url = "https://files.pythonhosted.org/packages/eb/56/79fd826f9ccaae0b84e1b4ef68dabba5e105bb044ffcd448a0b782fcba9a/greenlet-3.5.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d84d993f6e575c950d91a23c1345d18fe1a4310d447bf630849d7809196b52f0", size = 624002, upload-time = "2026-07-22T11:51:11.391Z" }, - { url = "https://files.pythonhosted.org/packages/42/e3/6086fa578ebb72772722cdc4bcd628459814b42e0c2db1e3cbd6552b3271/greenlet-3.5.4-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:3529a8a933582ad19e224792cac7372489526576b75b4c124e8e4f29948f4861", size = 435053, upload-time = "2026-07-22T12:39:52.715Z" }, - { url = "https://files.pythonhosted.org/packages/0a/1a/27319f97e731298513dcba1a2e91b63e9d8811d9de22130f960b129b1bf1/greenlet-3.5.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:58023945f421093de5e6fa108c0985a8659d43f49e0216da25099369a121bcbd", size = 1581533, upload-time = "2026-07-22T12:25:05.322Z" }, - { url = "https://files.pythonhosted.org/packages/b1/6d/24240bf562e9786dd2799ee0a4a4dadb4ded22510f41b20245099159ac8c/greenlet-3.5.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bae2728e1897aa8df8cb1af38cd48b3a743aefe29372de7b8b7a9f532501e69f", size = 1645781, upload-time = "2026-07-22T11:51:14.805Z" }, - { url = "https://files.pythonhosted.org/packages/c1/5a/442ab1a9ef7ca6bf7210e5397a95972206a91a31033a03c8900866a10039/greenlet-3.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:ca5726c0b08ca35ae873557266a78b2c3f3b2b7d7401aa5ff886c2045dd0111c", size = 247133, upload-time = "2026-07-22T11:39:20.661Z" }, - { url = "https://files.pythonhosted.org/packages/3e/e6/9160210222386b1a378ff94db846b9508ca24a121cf684991561fdb69280/greenlet-3.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:7c1303791d603080cac6fc3b34df51c3b75b723739c282c8029e48a0d241672f", size = 245500, upload-time = "2026-07-22T11:40:22.185Z" }, - { url = "https://files.pythonhosted.org/packages/a5/a7/6ab1d4f9cd548d15ab90da29947f2076100130bb179b0bde59f795a459e3/greenlet-3.5.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:7e8afa5eac028f8140ceafe5ceec66e6aa127ddcb21452d2a564dcd2900b5f22", size = 295410, upload-time = "2026-07-22T11:40:35.747Z" }, - { url = "https://files.pythonhosted.org/packages/cd/7a/422f63b4715cbc0b24385305407adf38b48f6bb68b3e6b04090e994d0f5a/greenlet-3.5.4-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73b37afe369021423ea53dd3123e04bffa7e93ac64429b9f50835b2e4fcae7cf", size = 661286, upload-time = "2026-07-22T12:26:43.8Z" }, - { url = "https://files.pythonhosted.org/packages/d0/31/5a1cac663bf5582190c5a714ef81364f03cde232227f39748f8ae4c11da5/greenlet-3.5.4-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ef964f56dfcb6f9bbef2a190d9126795eac408716aeae47b5e7c73c32aafca9", size = 673517, upload-time = "2026-07-22T12:29:04.815Z" }, - { url = "https://files.pythonhosted.org/packages/9c/bf/250c2921c7b585dde12f5239e313ca2dcbc464d161ecca36e4e6ef21762d/greenlet-3.5.4-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cef589bc65fae02d10bca2ac341191c5b33acc2967892ebf4fcbd10eabb7a74c", size = 677968, upload-time = "2026-07-22T12:43:46.788Z" }, - { url = "https://files.pythonhosted.org/packages/15/4a/2a82a1e3f8aaca020853ac8d12211280ca2b231aa08ea39f636f1060c319/greenlet-3.5.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c53ff01a5c53a40f2c16820ebc56d7c61a77f5fbe009dadd96292d5682f80f8", size = 670917, upload-time = "2026-07-22T11:51:13.589Z" }, - { url = "https://files.pythonhosted.org/packages/18/40/10bfcf6513558d82f7b95dd728001c63bd388259fe27d3e30ae01f103430/greenlet-3.5.4-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:dfc41ae893d9ceaf22c824f2153a88b30651b20e8758c2cd9ac143f23640563c", size = 480643, upload-time = "2026-07-22T12:39:54.149Z" }, - { url = "https://files.pythonhosted.org/packages/68/b0/e379a152b17bfdfa95795af4049e37c0fd1b4d81f020d426db104ed07c77/greenlet-3.5.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecca4d80d55a01ad6b23b33262662956149fbb7b2c6be2910f1705921958cbf3", size = 1628478, upload-time = "2026-07-22T12:25:06.678Z" }, - { url = "https://files.pythonhosted.org/packages/5e/43/bffdfa64f7317f954c5c1230b5dd5922676ce198689a68c1ac1ed4b1b1a5/greenlet-3.5.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ffbc533e0eaf8e80d8471411646ab88fe58f641d508c0b02b24494479f4d9ec", size = 1692021, upload-time = "2026-07-22T11:51:17.008Z" }, - { url = "https://files.pythonhosted.org/packages/d0/11/f799f9637e2c6e9b0b716015e339040598b058cf7654dfc0d67468b177ed/greenlet-3.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:305f69e6c4523d7f6979ed001cff4e5853c063e5da04880296603aa0227e544c", size = 248031, upload-time = "2026-07-22T11:40:11.007Z" }, - { url = "https://files.pythonhosted.org/packages/05/75/625bcdd74d5e6b2dca1ecba3c3ac77bcf8a026c21a649a46cef23e421f97/greenlet-3.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:f260930bbbbcf9caee661211235a5111c86dfe5832fdf6ae4570da1e0995320f", size = 246892, upload-time = "2026-07-22T11:40:27.357Z" }, - { url = "https://files.pythonhosted.org/packages/ec/69/35c62ed49c320cb4d98e14698ccca5467d3bfe683984172be9cb564d9ce3/greenlet-3.5.4-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:41ddab54e4b238f4a6c323f39b4e59e176affd5a94d461a9fb7583dac74240a3", size = 305571, upload-time = "2026-07-22T11:40:31.659Z" }, - { url = "https://files.pythonhosted.org/packages/5b/6c/64d60216b3640dcb0b62d913dd9e0d80030c09115bb2e4ba70c95d10ca45/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b3dabe3e2809013052c68bdf0b7fa5f5f2859c43a80803131ad61af9cabd7867", size = 672568, upload-time = "2026-07-22T12:26:45.298Z" }, - { url = "https://files.pythonhosted.org/packages/5c/de/ba3ab0a96292e53039530333b0d2ae18d9e508f3a325cd7bf15f8172944c/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:27d3f00718634d4520a3a150154ac5da36f257869d41321953375b90bfbbc72c", size = 680076, upload-time = "2026-07-22T12:29:06.125Z" }, - { url = "https://files.pythonhosted.org/packages/ae/db/24a10af12bf8e639cec46c38b9ce1a282543ba42ff4fb0b31a970f1ab603/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:39169a11d87a6a263afda3e9a27d1df16d0f919d40a4837cc73986c9884c0dd8", size = 681690, upload-time = "2026-07-22T12:43:48.109Z" }, - { url = "https://files.pythonhosted.org/packages/a5/be/aeada79083c6f1c15f45d77a332f9c441af263ee298e3eb17522cd337d22/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbd60b5763c6543c1827e48faaf14ea9bfbad245f52b1a4d76a2a2d8884c6c66", size = 676733, upload-time = "2026-07-22T11:51:16.027Z" }, - { url = "https://files.pythonhosted.org/packages/f4/60/44a2eca7b9fd71ae0fae7ff184da1cd3169d176652b97aa1cffcbb0ef961/greenlet-3.5.4-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:bd3d1145f603b2db19feb9078c2e6855eb7c67e15580c010ed815cee519b86fd", size = 510263, upload-time = "2026-07-22T12:39:55.678Z" }, - { url = "https://files.pythonhosted.org/packages/e9/10/2392fc3a98948652ef5fd1e7275c04f861dd13f74b78a2b4309f4ee4d090/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f00f910f0e7b35416c63b23ad78b769aeccfc1775f712b43c4ee525624a2eef7", size = 1637327, upload-time = "2026-07-22T12:25:07.879Z" }, - { url = "https://files.pythonhosted.org/packages/55/c6/e7237a3dfa1f205ed0d9ea1e46d70bd2811b32d516266399fb59d28ab90a/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:91c26423753b92caf41ab3f98fd547d7374d4d9fc2d85be041886c1579d9255e", size = 1697493, upload-time = "2026-07-22T11:51:19.214Z" }, - { url = "https://files.pythonhosted.org/packages/55/e3/4ba8154ba2a3d43729e499f72471b4b5c993f3826d3e24da81d5f06d6572/greenlet-3.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:ee032b91fd8ec29ec6c4cea2b8c561b178435134bd0752c7334b94e9c736c132", size = 251637, upload-time = "2026-07-22T11:40:37.44Z" }, - { url = "https://files.pythonhosted.org/packages/90/03/e3f96dfc100261a29545ddc8270cafe58f9195b6651466910e820910de77/greenlet-3.5.4-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:178111881dd7a6c946471fda85485ec796e1043c2b939f694b096e2ecf986809", size = 296076, upload-time = "2026-07-22T11:39:38.364Z" }, - { url = "https://files.pythonhosted.org/packages/a4/3d/da52d208e5c977bce8667e784729e584e38b5785f4c1ec0f4c836e9a1c42/greenlet-3.5.4-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d92df08dd65fede97fc37aad36c2e9dcda3b31c467f8e0c2c096456cb818e927", size = 666870, upload-time = "2026-07-22T12:26:46.691Z" }, - { url = "https://files.pythonhosted.org/packages/2d/8a/7e6dee25cb8a8cf9b362c8e597cc269593378bd916f16c736c059a52e85a/greenlet-3.5.4-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:99e8f8c4ebc4fd80aa26c1280ae9ad43a0976e786349703a181cf0bae60413e5", size = 677678, upload-time = "2026-07-22T12:29:07.508Z" }, - { url = "https://files.pythonhosted.org/packages/51/a7/dafc7415d430b0a43a16396eb49ecb3b62fd720877fb259cc4dcfaf5f31e/greenlet-3.5.4-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1f17e362d78e37559e0506c5a7d066bdd45073c36a0127a543e8a0df27242ff3", size = 681428, upload-time = "2026-07-22T12:43:49.623Z" }, - { url = "https://files.pythonhosted.org/packages/6c/21/5a38699fa45de749e3857d93b8f07e4c20489e77c2d35d915a2e1c456606/greenlet-3.5.4-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:394de08dad5ffcb1f50c2159d93e398d9d2da3ed437645eaa54771fa720db9f0", size = 676067, upload-time = "2026-07-22T11:51:18.163Z" }, - { url = "https://files.pythonhosted.org/packages/2e/d9/6298f3432de301d4718766cf934bd73c418c73f81fbb77247319364b0d96/greenlet-3.5.4-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:cd320d998cbaa032932830448e39abf3c6a12901295e386e8114db926e10cffb", size = 487446, upload-time = "2026-07-22T12:39:57.044Z" }, - { url = "https://files.pythonhosted.org/packages/5e/ba/863116ab8ff1ca7a729e327800268939d182db47aa433db70e216e7d9194/greenlet-3.5.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:c883d61f2282d72c767a14936641b3efcbde9d82f1080712aaea0b1d3126cb88", size = 1633489, upload-time = "2026-07-22T12:25:09.605Z" }, - { url = "https://files.pythonhosted.org/packages/fa/06/7466ced82818d6132462d7f26b3f83c66ea15d2b193a6c0088d558ed7d95/greenlet-3.5.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2a924f15d17957e252a810acefcb5942f5ca712298e8b6fcaed9a307d357522c", size = 1696584, upload-time = "2026-07-22T11:51:21.304Z" }, - { url = "https://files.pythonhosted.org/packages/f9/4d/55b638489260065de9ffce606c8b5d04507bef705de4b212a0c3d6a1a0df/greenlet-3.5.4-cp315-cp315-win_amd64.whl", hash = "sha256:ed17e5f3420360d5b459de8462efb52060399a5326a613d4cde31cef63ef95da", size = 248297, upload-time = "2026-07-22T11:42:04.055Z" }, - { url = "https://files.pythonhosted.org/packages/bb/08/9dd4ae635da93d41dc268bc34bd62a9d711ed8b8825c5d22ac910c7d6e6d/greenlet-3.5.4-cp315-cp315-win_arm64.whl", hash = "sha256:f908898d6fa484ce4b6f447ce70ea99b52c503fee419e53cf74d60a16bc9e667", size = 247423, upload-time = "2026-07-22T11:44:00.764Z" }, - { url = "https://files.pythonhosted.org/packages/19/66/7c87ed9cdbf1d49c2c6cd1c7b9dd4d16c33b24235ca03972293a1876b30c/greenlet-3.5.4-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:1833637f17d5e7472548a48575c394fe39f1b1890d676d162d86593610f44d8c", size = 306487, upload-time = "2026-07-22T11:41:25.118Z" }, - { url = "https://files.pythonhosted.org/packages/5b/05/0a4201e7c0054866eefc05da234f236dd4c950d0fbf9ca0517141f01b269/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:12cda9122e03341f1cb6b8207a19d7a9d375e52f1b4e9243918375f40fd7b4b9", size = 676479, upload-time = "2026-07-22T12:26:48.129Z" }, - { url = "https://files.pythonhosted.org/packages/3d/c0/4b6b8c5a3aec70f0649cd89662d120fdd6421e2bdc8e3b15c3ab5ec568d8/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d83ae0e32d14957ab7170785a20f582635c8474deab1bfbb552b17e769a6ce25", size = 684321, upload-time = "2026-07-22T12:29:08.925Z" }, - { url = "https://files.pythonhosted.org/packages/88/15/0b167aeea95285b0e654ddce651922f666c089363c2ec528ca8b9a9ba74f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:123aa379c962ed5fe90a880327e0c3066124ac64ec99e12a238be9fd8eb3db3d", size = 685995, upload-time = "2026-07-22T12:43:50.993Z" }, - { url = "https://files.pythonhosted.org/packages/24/c9/b49c31c9a972eee91e260445770e922244a7efc542697f51a012ec046d0f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f1467de1bb767f75db0aa34c195e3a496d8d1278c796e70c24ce205d3e99cde", size = 681293, upload-time = "2026-07-22T11:51:20.43Z" }, - { url = "https://files.pythonhosted.org/packages/de/90/c023ec337f32ff505be7db759c80d98f0532bb94d0c6fa13645efe9bee2e/greenlet-3.5.4-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:adf2244d7f69409925a8f22ed22cc5f93cdfe5c9dc87ff3476be2c2aaae61a05", size = 516928, upload-time = "2026-07-22T12:39:58.359Z" }, - { url = "https://files.pythonhosted.org/packages/95/6f/7f2d4653770500eee667866016d42d7a68e3d3462f80df6b8e3fcd48a0eb/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:0fa53040b78b578120eecdc0265e3f1051487cc425d11a2b7c761daadf4feaa8", size = 1642474, upload-time = "2026-07-22T12:25:10.819Z" }, - { url = "https://files.pythonhosted.org/packages/5a/d8/8cba31036a4caae448087ba5d150660ab03b4a0f54d9150f6495a3be7262/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:60e0bc961d367df506660e9ac0177a76bc6d81305300704b0977d1634f76efe2", size = 1701012, upload-time = "2026-07-22T11:51:23.17Z" }, - { url = "https://files.pythonhosted.org/packages/e3/cd/3f77a4cce3bae631b08eb52f53a82a976669600337e21dfdba811cb50267/greenlet-3.5.4-cp315-cp315t-win_amd64.whl", hash = "sha256:f680e549edb3eaf21eea4e7fe101e15ec180c74b7879ab46adc080f22d4015d2", size = 251977, upload-time = "2026-07-22T11:41:38.125Z" }, - { url = "https://files.pythonhosted.org/packages/93/e8/65e8707d00fe2a49bf12f609a9b2b39ba6dd23c2810eacad877c4fc94bfe/greenlet-3.5.4-cp315-cp315t-win_arm64.whl", hash = "sha256:08fc36de8442d5c3e95b044550dbea9bf144d31ec0cc58e36fb241cb6ef6a994", size = 250538, upload-time = "2026-07-22T11:40:17.985Z" }, +version = "3.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/d8/7cc97c142388aef03f622e001c572c4f84e9252a439549d483f555771970/greenlet-3.5.5.tar.gz", hash = "sha256:adb4bae02e91a8e863e48b177e4014bdcac8a6b5e047ea1df687a61534b85e6c", size = 207585, upload-time = "2026-08-10T15:09:36.136Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/3d/8cef5f724ec0d4add2af8961d504535ec60c3cca9e464f6d03bdba29d85b/greenlet-3.5.5-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b79fd2a5bc099b5e744f34c4c9a58954a5f4cb7529fb4b6e8446057d61b6edaa", size = 294730, upload-time = "2026-08-10T13:27:51.206Z" }, + { url = "https://files.pythonhosted.org/packages/88/4b/8e7aa3f514273aecff30a16ab1bac09ff54cfc7e6860fdd8058c37ff2499/greenlet-3.5.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:634cf15a233a949136879dd388e25d3296e16f3f1e217d2456797b8579ebc6ed", size = 614536, upload-time = "2026-08-10T14:14:36.589Z" }, + { url = "https://files.pythonhosted.org/packages/85/48/4e95e9dd5a8a397dc6a6345dd7f1935113d0fca4f85e89d3976da9cd988d/greenlet-3.5.5-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:499adea519f748407fc6806d20eedabac2884fd73b9f38d81236e190ba20dfef", size = 626924, upload-time = "2026-08-10T14:27:27.048Z" }, + { url = "https://files.pythonhosted.org/packages/0e/84/eaa476d6bf3816828d0d70e80dcc36bf30a058233bd889e707e693f6e860/greenlet-3.5.5-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7278591501941bb2456af102bb9cd59aab48c6cfd6e2dd68fa1290bb0c49a42", size = 632726, upload-time = "2026-08-10T14:30:09.874Z" }, + { url = "https://files.pythonhosted.org/packages/89/5d/398a1c71fa7a277deeb376c999979de6786f08fc2d5747a0b9d6e11738dd/greenlet-3.5.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2eabb980975cba5b93a95f6f69287d05fc05ac955bfd6a320a7c083eeb52c0b0", size = 623906, upload-time = "2026-08-10T13:40:50.501Z" }, + { url = "https://files.pythonhosted.org/packages/d0/f2/0cc2849ede68579291e9c59b3ab6ec1958f98681cca5b14d8fc75bf674a4/greenlet-3.5.5-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:4dfc7c4470354e7b09184d1a3a985761053a2fd694ddb5b5c80242afc2c8c90b", size = 434966, upload-time = "2026-08-10T14:30:03.729Z" }, + { url = "https://files.pythonhosted.org/packages/04/1b/745450fc5ea9e0cb17d840d248f284db3363de736d362c7d2d883e3eadba/greenlet-3.5.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:03115c2e0a371999bf8ae616aa8d653f96641d4705c457aebaa187276e9f7537", size = 1581430, upload-time = "2026-08-10T14:15:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/d4/29/d51b296e3191bb15d3d81ec375af1909e4466c0f395d744ed475801798a9/greenlet-3.5.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4441153ffba21b90d3ca89fe3d31f5c093ae6c0bf0cfdfc98f54cde22f95b62e", size = 1645684, upload-time = "2026-08-10T13:40:32.133Z" }, + { url = "https://files.pythonhosted.org/packages/12/63/369f1a1625e64e9e31df3963c6044056e3fdfa3fa3fdba3c54ffefa6e987/greenlet-3.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:95c5b1f4b3a193f8a0c2de4bfdcb48d119f7f1063941f1de1f2168051b3e52dd", size = 324075, upload-time = "2026-08-10T13:26:58.974Z" }, + { url = "https://files.pythonhosted.org/packages/45/78/649cb5c09d4d81f6dd1444e75474a7206784743283a21d24171562ac4899/greenlet-3.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:1af90aa4bc129883b340cdd6957a3bc74f60528a4993bbd1f53aaebe1d9981cc", size = 308260, upload-time = "2026-08-10T13:27:50.795Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8c/080e881fa2be95ff1ddbd6994b2bab3b1a78df3b3fcab39306011764fcc7/greenlet-3.5.5-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d4a389a852e392a6366058651a20fa5ba40d979865aa81bea2ccbdc44805070d", size = 295309, upload-time = "2026-08-10T13:26:03.032Z" }, + { url = "https://files.pythonhosted.org/packages/25/cc/0ac614e6586c0e42d4cc281a5819150f4f43685744a4c5ff77139286409d/greenlet-3.5.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70b157cd319873e8b544ddc2de158f55bbd0a9b0218c8ce9332039801518e328", size = 661185, upload-time = "2026-08-10T14:14:37.867Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b9/6808725354be8ad305dfe5172377664fc9642d4fc043be246b3314cf4482/greenlet-3.5.5-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8bdfd1424abcf26832961e766570cae79efdb9599d709088c9cb6ef82b194926", size = 673419, upload-time = "2026-08-10T14:27:28.652Z" }, + { url = "https://files.pythonhosted.org/packages/eb/52/f005d579acde46c3d1cc3cab1c9f3d5708c8a3006a4120e8cf5da801afe9/greenlet-3.5.5-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d98ef6f92e67c6dbf299dbfd8facc1b0d2d9cedf91e325e73b3d0373fe4309d8", size = 677863, upload-time = "2026-08-10T14:30:11.663Z" }, + { url = "https://files.pythonhosted.org/packages/42/2e/40c509967da7f254680826a2fa0dd22138ec79946c70b97542d74cde8b43/greenlet-3.5.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:182de51c6b572a705f2fafaab2e783bcf7d2760940229dfe73086cbae037af3e", size = 670822, upload-time = "2026-08-10T13:40:51.833Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8a/a75f8a2bdcef3c358a3147cdc9db3aa83755f0a038f766ab0bedb66f512c/greenlet-3.5.5-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:159df1942d88e8f784cbb38d6f18bdb365cd11319cfbb3e89623de2b97892d53", size = 480554, upload-time = "2026-08-10T14:30:05.171Z" }, + { url = "https://files.pythonhosted.org/packages/2d/22/c3c2eee4a8fe191d6d1d183086c56133d646024e3d70bfd414829f64560b/greenlet-3.5.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8fec3f165dfe332e490c3247c0f6c23b0bfc45f06496ad7f00ddb00e3d35e4dc", size = 1628469, upload-time = "2026-08-10T14:15:08.11Z" }, + { url = "https://files.pythonhosted.org/packages/f7/87/25babd09b94cb1f03e71db815fde463f0262e40cfbd953d58a8d77311351/greenlet-3.5.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c6ce25fee6cabc8bf22cb8b52e642cbb821be5b9aec8094d07ff03378141b8e9", size = 1691952, upload-time = "2026-08-10T13:40:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3d/5cc9701117ea4dc0eb7bf1f4f9b7888a6e2e5277ddfae095805ace50f2b6/greenlet-3.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:7dffc5c859fe6059974df1e37d7923d654a83e2ae18fdd616994270e001115e1", size = 327458, upload-time = "2026-08-10T13:27:02.868Z" }, + { url = "https://files.pythonhosted.org/packages/a7/6b/594fa2de7fae7629168a404a4305d7d7e31a5742c50a801b1839543cb93d/greenlet-3.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:5e2afcfc4d4305dd715809b03da5cbe437c8984f61d8917751eb5fe4aefa3e07", size = 311146, upload-time = "2026-08-10T13:27:25.046Z" }, + { url = "https://files.pythonhosted.org/packages/24/e0/50cd600b469e5734c72709b6b1838b6bc63f307b573c772c3132d6ecfe92/greenlet-3.5.5-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:0e5a7de979d764aea1f5b6e95cf92b5b37741b9823702041f34b126e7f690277", size = 305471, upload-time = "2026-08-10T13:26:20.568Z" }, + { url = "https://files.pythonhosted.org/packages/75/a3/77acd66dfc6387b5219b2080806c0cabb73c10eb1bb44b413c40a62015ba/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fef01bd457f11fc158b130ca0027a3c365693280e8e231b65bdaf57999f39f5b", size = 672470, upload-time = "2026-08-10T14:14:39.058Z" }, + { url = "https://files.pythonhosted.org/packages/b9/71/0d178142dca3ec19f46fb2212ae73d30ad53b9d548dc64804086033a7089/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5173a72310725a74afc82c164f0e52cb8ad0de62f2bb623f24f6c0cc07d80272", size = 679973, upload-time = "2026-08-10T14:27:30.072Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ac/0d7887aa4bbfc9eba075cc428244dfc96f623478454d5ec81180d0d6bd5a/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5e9ec2e7c98e895fcea0c5cc57b2606cf86ece6d0a56578f3eb225e2af4f0387", size = 681587, upload-time = "2026-08-10T14:30:13.519Z" }, + { url = "https://files.pythonhosted.org/packages/6e/31/46eb8567302eaf787abf88d09df014e14ae3baf460af1b8b0efdbd3efcd5/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44f08341873200ba8a60a8bc14ace3d91f1754f7fa7bc66157714a8cd420a476", size = 676634, upload-time = "2026-08-10T13:40:53.004Z" }, + { url = "https://files.pythonhosted.org/packages/4f/18/8d58ba1c429b0383e3219a3d0e0bba241d0444d8ed05b73349953c7d7c7b/greenlet-3.5.5-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:102817506f6090b5176c746a82603341a549b40e5c3d5b72a4c672228a918c41", size = 510175, upload-time = "2026-08-10T14:30:07.047Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e9/b88bbf5b29970cb84172dc2c32aa3e5e579ceb94c808e81c826454138850/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d246c0db9a2513cd45f019ba178ea4d4d4705bd210ee465e2c15d76a1ab13874", size = 1637320, upload-time = "2026-08-10T14:15:09.317Z" }, + { url = "https://files.pythonhosted.org/packages/6d/8c/7631ed29cc6f0392f11830076e172ce4885e70b0bc2c1bce1731176d4b4e/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:72507285b5caa1d17904a3f7c322ca780823a54170a0e04ec3f37bcc60d4db71", size = 1697412, upload-time = "2026-08-10T13:40:34.924Z" }, + { url = "https://files.pythonhosted.org/packages/da/0f/f7dd935f9c4cb1be49098770587f54d8a78518e55c89bce86c4fb4109057/greenlet-3.5.5-cp314-cp314t-win_amd64.whl", hash = "sha256:7805655781fb8f28a55d05fe57ed61f5f10f1892fb587673e3bb5264f28041f0", size = 331514, upload-time = "2026-08-10T13:29:20.611Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e5/681b01f8fbc1b55232822f99e8f8afeb78a55a7c76a7bf9dbdc7ccb03a6d/greenlet-3.5.5-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:c0db80fcd5b8aece93f66c64f78a786bbb6b96c5fe63ef5a5a4581ecf8bab206", size = 295975, upload-time = "2026-08-10T13:28:45.985Z" }, + { url = "https://files.pythonhosted.org/packages/11/f2/69b488cd9e7267bf4b0fe8cdebf25d8d6df680d21bdf41150d23e23d6652/greenlet-3.5.5-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b241c32f912ada659808d68e308c568baf577eebf757d15471472de0c18cfad", size = 666823, upload-time = "2026-08-10T14:14:40.222Z" }, + { url = "https://files.pythonhosted.org/packages/84/d4/d5bc2fdebbdda0c94555925ba79948b8395d75a7f6a36cc85dce5bab9f11/greenlet-3.5.5-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ef6a08349401d8eaf3cb12688ac8557de95788556b8631ef17555a4a173022c0", size = 677613, upload-time = "2026-08-10T14:27:31.543Z" }, + { url = "https://files.pythonhosted.org/packages/65/53/4e13642efc4d7ad6554ecb2242a5be42666b2e1a067323e88dfc0124a04b/greenlet-3.5.5-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:37faa97daccb6d9f4c2141ce3118d023c3c5506864a7d8bdf726f665018c1f76", size = 681436, upload-time = "2026-08-10T14:30:14.839Z" }, + { url = "https://files.pythonhosted.org/packages/bd/93/542d8a3a90f3b35c6ad8bf7e56a03010287f2cafa289a5b7985b5207db39/greenlet-3.5.5-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2e3d061b8e13aec2f0441689b3c71b244a20e5d274a52cb0f7e31bd1d139552", size = 675930, upload-time = "2026-08-10T13:40:54.205Z" }, + { url = "https://files.pythonhosted.org/packages/cd/32/188447c9a468d6977d2989397226b0c6b65ab6f4cf943f931643328512fc/greenlet-3.5.5-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:b18007dc2473a7942fd157366b55f01da6fed7ce85318591005b419e0a439474", size = 487404, upload-time = "2026-08-10T14:30:08.903Z" }, + { url = "https://files.pythonhosted.org/packages/52/b5/89c9f2e8460d71101037d47a1feed11928615a5edd42370be290e0657eeb/greenlet-3.5.5-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9ab5f5b93655e77fe0d6c2dfd22b5eac751bb1f876d8ec21761b7c1fb9266007", size = 1633878, upload-time = "2026-08-10T14:15:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/b8/60/297de93f3b02ac78a5e04d32bb8bbe3080f4a73d8ed95016561463b70618/greenlet-3.5.5-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f0e5a21bd4452a88cf032fc43c4a5b307ab1380eacb63b5988f9c0317885e773", size = 1696597, upload-time = "2026-08-10T13:40:36.252Z" }, + { url = "https://files.pythonhosted.org/packages/18/25/54c6eaff4f337fb670215e89eb2d00d9499487b658e709d4b477be4a342e/greenlet-3.5.5-cp315-cp315-win_amd64.whl", hash = "sha256:469dbb0a78625642f4a626cfd0c6e8bccc0385b5e49189b6308bbe849ec88a8e", size = 327700, upload-time = "2026-08-10T13:28:06.752Z" }, + { url = "https://files.pythonhosted.org/packages/67/67/857e88a36301caa0e029870132c2478bd55d896630321432afab03a3115f/greenlet-3.5.5-cp315-cp315-win_arm64.whl", hash = "sha256:2d57406c3efd32d7a81e17a674314e8bd00792cdab49ea3228a49aa1bfb2e769", size = 311750, upload-time = "2026-08-10T13:34:08.815Z" }, + { url = "https://files.pythonhosted.org/packages/10/e2/3144c0a116067ac1e30457b0139a94d60d1d36a86e015de68e9ac87cb3bc/greenlet-3.5.5-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:68184dfcf50ccaa8e864770fe0633a7e27250ea9329f8192ef47ee9ecfd78e1c", size = 306387, upload-time = "2026-08-10T13:27:00.897Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a1/cb4223a7e9b9f43b8807e8eb212358bfe2dfaa174a9ea2889eb1714dcba2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ec0dc0e59dc9c61af5c47348365ccbbd7addfafe0a93b00336ff3da2907bdc6", size = 676472, upload-time = "2026-08-10T14:14:41.417Z" }, + { url = "https://files.pythonhosted.org/packages/9e/cd/a154b4498e5d8f12ada291cfb3b8d596eadde2177f5bf09a9be699d2a446/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e604f58e35833fc46ef20302bcb314dddbfd3fcf33a4f936216d51dd678d63ae", size = 684238, upload-time = "2026-08-10T14:27:32.946Z" }, + { url = "https://files.pythonhosted.org/packages/ce/f4/e450a68a152f819491d8c7df6a8254e761d87e6a78759268961f8c5bd4dd/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2888a3a38bc5ee5bb6c438372197152e815837e4fab7ed7a1f86ef18ffd58ad1", size = 686022, upload-time = "2026-08-10T14:30:15.96Z" }, + { url = "https://files.pythonhosted.org/packages/bf/bb/b0031d260c2968a3c87deebc51d80c64e499377f993aafe06ee3b7488cc2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:40239b5384f96da3963585cc6d7eaa9b56f8ae67e8d92cc82dd9e202fc847de3", size = 681246, upload-time = "2026-08-10T13:40:55.402Z" }, + { url = "https://files.pythonhosted.org/packages/18/23/17e63d6bf3b9c9b9dbea981b7f643a71f79603bdfb4f1c3a9cf353e22aed/greenlet-3.5.5-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:1e8d9391fe77f15649589a907cef972dbbd6352ef7ff7dc0492f658c0c26495f", size = 516951, upload-time = "2026-08-10T14:30:10.907Z" }, + { url = "https://files.pythonhosted.org/packages/9a/07/da554b71ab88e649da146e1065d86a48a5c5d92e50ab74ef41b504aa7f56/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:a1eaccf5c3a1d3e46dead602c72e6836731e8e245c9de6a27764567b6b62d4c0", size = 1642735, upload-time = "2026-08-10T14:15:11.92Z" }, + { url = "https://files.pythonhosted.org/packages/78/76/26a3782a051677668af9d92beaa47cd87ba9dd5072f762961144a03dd4c6/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:19e4e026fe20691f333b8eb1a3bc9625eceba8c3f9d62ec5a6f8581afbc6b5a5", size = 1700925, upload-time = "2026-08-10T13:40:37.656Z" }, + { url = "https://files.pythonhosted.org/packages/28/d9/fe7baf4190c2ae71f267efb9de21b3172bb35bc0ed1ef53dd6027d658e33/greenlet-3.5.5-cp315-cp315t-win_amd64.whl", hash = "sha256:712aee154f648bde84634654bb38bb78c69ac640c37a45c9effed800735049d8", size = 331829, upload-time = "2026-08-10T13:26:48.851Z" }, + { url = "https://files.pythonhosted.org/packages/df/af/419a4e383bd600858a9b67e9b280a60fdc383ee3f2fe5b6c0c1ef04e74d1/greenlet-3.5.5-cp315-cp315t-win_arm64.whl", hash = "sha256:7f049911ee81a16a03c33d5450d8d5867d27f596ca5fb201b86f4524e874468b", size = 315093, upload-time = "2026-08-10T13:29:34.949Z" }, ] [[package]] @@ -1119,6 +1112,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/83/a896fc59940fc5a6e2aff3a4be1d92fa890112936803b331cae75a993c34/httpcore2-2.10.0.tar.gz", hash = "sha256:13c0cc3d1919d4f28457f60cd2c2abe04113a8af184ccf1142811beba936f9dc", size = 67427, upload-time = "2026-08-09T09:11:32.123Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/4f/d149104195a35e2853a2fc203a8e3477747e58c80e17dda686dace174383/httpcore2-2.10.0-py3-none-any.whl", hash = "sha256:7df06cfb34070cae4f7c89be69dc1095eca138e9704ceffb98d25c1912ab6f01", size = 83000, upload-time = "2026-08-09T09:11:29.555Z" }, +] + [[package]] name = "httptools" version = "0.8.0" @@ -1163,6 +1169,96 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "sys_platform == 'emscripten'" }, + { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/3d/f9a8c07a3884f3e5b26205e8436a18b3af61c5d53192c3bea235574dbbec/httpx2-2.10.0.tar.gz", hash = "sha256:8741d7329fe2c7885fc9ceb61c8217acfb87a85f75723714b89ebf7ad7196338", size = 98749, upload-time = "2026-08-09T09:11:33.24Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/6d/a637d52449d98a6892d9a4dc0262587afdb6a66f201871842dce5a97b1c1/httpx2-2.10.0-py3-none-any.whl", hash = "sha256:5e3194a432701e1cc6f69a8b1b2fa199ef907013fede8d9a09a2c5b7b8141a18", size = 94355, upload-time = "2026-08-09T09:11:30.882Z" }, +] + +[[package]] +name = "httpx2-jsfetch" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, +] + +[[package]] +name = "hypothesis" +version = "6.165.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/e2/0fad246d2b6330e1f78479bfc566b5c22be82aee8a865cde9a08f648487d/hypothesis-6.165.10.tar.gz", hash = "sha256:68b45e09834cd80523cb1eb274463073c7a9af4e4ef7cff34d9615f355572d32", size = 503703, upload-time = "2026-08-16T22:56:15.404Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/c1/9a9538e6d185baf5cc7f15bc3b76e08efbb3de4b3c782f234356449c0dd7/hypothesis-6.165.10-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f839d29d0cc12048cf073d88ca4fdf94d420bc2b8afd69641ff6d496422ccd4f", size = 783243, upload-time = "2026-08-16T22:55:44.058Z" }, + { url = "https://files.pythonhosted.org/packages/a1/30/b70d9d79e871a75cbdeccd9067f20ecdb9eb2a1dfa03c630be3ad13b8b30/hypothesis-6.165.10-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e10858f57ed0e74baa04393845f469fe8ad502c16ece4499bef7700c575611bd", size = 778815, upload-time = "2026-08-16T22:55:46.948Z" }, + { url = "https://files.pythonhosted.org/packages/db/52/6f0a9b7aab24b0635e2238f3fbddea5b54b17879ac813df42a3cc3384c5c/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76a7be86d986223b9f1bdb7e7cbcdb048649901fdb956c598ef73bdab1786cd5", size = 1108009, upload-time = "2026-08-16T22:54:53.082Z" }, + { url = "https://files.pythonhosted.org/packages/f6/06/8d0d4e11ff02350d09ec9f9e90af354158e59e16a8907ba5199a4ff2d7e8/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:717aea574e0e5edba2868aa66b1caae335d8f1ad3fb29f01dd6502953fa823a1", size = 1136596, upload-time = "2026-08-16T22:54:54.443Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/01a1e440f2e38dc1ccf5d597af5b8a0bee5f21b674c99c123b5554de9690/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4334058033e0214475f019e15492a50f3854fe8728cf51fe25c6191a2c3f8e52", size = 1135234, upload-time = "2026-08-16T22:55:08.911Z" }, + { url = "https://files.pythonhosted.org/packages/7d/18/8a26c24d3d9db20265f39df341ab265858c094e209571e3179cf237935f4/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2abb50cf1cf77d721de0a24c3f99d9c4ffdeb2cbd1e12aebb5a7a93e2b6b6d1f", size = 1157528, upload-time = "2026-08-16T22:56:02.159Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8e/ce3c829b1937402d7944420ca26a05a0c8563e894dcff03d34ffa279d306/hypothesis-6.165.10-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:3de69aa8b924b400291a3cc42aaf78e6ab65c905a3e7e1a5dc39d95ef1b428cb", size = 1112870, upload-time = "2026-08-16T22:54:55.919Z" }, + { url = "https://files.pythonhosted.org/packages/f2/1b/4c4926d6c9a2b5d7cc090cc1e91219d6796102aa2a2c4b8f961c939e60b5/hypothesis-6.165.10-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5841331c504e02d7c334591681cb8587cdd59dee7e149db6d3db8e3f9e9f02eb", size = 1149683, upload-time = "2026-08-16T22:55:30.567Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f9/df24eb28412f82465e2b7707f0ff1ec274d580bce389d4d9156617dc7bba/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2d0e0f8263d34dd8fa3b39eaa9a50bba56a8470b3dd9ebf6672d10840abe063e", size = 1283402, upload-time = "2026-08-16T22:54:18.054Z" }, + { url = "https://files.pythonhosted.org/packages/4d/07/c2b2a761300cf60b90ccebba4328175331e67d34f4fbd39429a7ddcdce49/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:0c4e6869817c3cfdf5a2b4d348497b95159bdecb3365be732c9b8570e36a4eef", size = 1409948, upload-time = "2026-08-16T22:54:22.343Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ec/1c2bf1acdd0e273d81f833f85caf0ae5423db68a783554992fca36e6c541/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9f07ae36c3b093e13687a894e79fe69e98a94c0b67fef656c575247682218143", size = 1265023, upload-time = "2026-08-16T22:54:41.402Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a8/7f984908b7391160c7801b84e51ca8e4ba88c89e8d8811aa1aa7c03de73c/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:aff1f584c9538e8979cd180b1d70bf99bc16be19d4666414f49e5942b21a4f2c", size = 1282698, upload-time = "2026-08-16T22:56:06.998Z" }, + { url = "https://files.pythonhosted.org/packages/48/78/3a5d91c2d0250521736c42dfa2402b75049bc5fe2fb716c10bc84bb91ed1/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1f2c4db25fb8ec1a16a8dba580666337b8ffb1887c4cf1750cc954313897cef7", size = 1324816, upload-time = "2026-08-16T22:54:46.675Z" }, + { url = "https://files.pythonhosted.org/packages/6f/99/27450763853a034bca1574d3e0a315164b33ff49c3862df6872dda45e25e/hypothesis-6.165.10-cp310-abi3-win32.whl", hash = "sha256:b33dc30170a7402e03c180f2c5ef69dc077152f35b91621e9cebcde9c7d71746", size = 669039, upload-time = "2026-08-16T22:55:11.962Z" }, + { url = "https://files.pythonhosted.org/packages/2c/fc/ff2988b72b5705ad9ca500444bf3f43e3c2f41edfa034bbfeb23b215791a/hypothesis-6.165.10-cp310-abi3-win_amd64.whl", hash = "sha256:e9f924aa610c0618445e1e8738c822c3190ce2a2699a0cb48ec3a351a96761f2", size = 675213, upload-time = "2026-08-16T22:55:01.697Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8b/821810d36f78d9d9421cd2c5d9d36983b45bb3575c3086276cc5c76f9f73/hypothesis-6.165.10-cp310-abi3-win_arm64.whl", hash = "sha256:1d305448e9bd8e2f4f3cea0eafd809efdaab4e998a0019bc615650c8463e42f1", size = 673537, upload-time = "2026-08-16T22:54:47.898Z" }, + { url = "https://files.pythonhosted.org/packages/b1/fb/c82c5bd92864ffcf319772fedc8c9bf2dbe4ca14baa0fee6e49e67b5ba1c/hypothesis-6.165.10-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9d77c3be7b429875036ad0f0597c6e5cc6bb17894a4da005e3807de64d2673ad", size = 784726, upload-time = "2026-08-16T22:54:32.371Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b9/3d7acd08506da85557e65147b7f3fca8c47684e33be90bee0acb523920db/hypothesis-6.165.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:490c56b830772b0eca3b4b2cecb3741a1ed26b1d7206a279e1525dbf0aa95ee4", size = 776375, upload-time = "2026-08-16T22:55:13.303Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/922e8b3f9a706dd89d440b9545d2c6231c65e74da1c1fee3ff36c251b9c4/hypothesis-6.165.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed68e27b8a61e57a3ccdc7c5a14499e00b54dfe223087204d5d40b3b5ef58b6d", size = 1106763, upload-time = "2026-08-16T22:55:06.129Z" }, + { url = "https://files.pythonhosted.org/packages/01/39/f5b9a5d390d4edd1ad472334493ac442963ebeb4daaa74ff4bdac6ef292f/hypothesis-6.165.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6caadcd1afb62630ff5c5ff353626eaa616553a5971295ad6dc2b19ca8a39620", size = 1156778, upload-time = "2026-08-16T22:54:33.824Z" }, + { url = "https://files.pythonhosted.org/packages/b5/5f/5fbe1be4326337fd6acefe2d18ed44007ee1dc1f98fe5b3c0eb22942364d/hypothesis-6.165.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9145fe43ebb22e66672967c3fab411793b226ed776e4fe282271bca6ad3c0bb", size = 1280756, upload-time = "2026-08-16T22:55:54.834Z" }, + { url = "https://files.pythonhosted.org/packages/25/c0/cf6f9e1ef632a1a75694eed0db3a02e6fc75c367a363e94acee52f043c64/hypothesis-6.165.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:79900a9920a0b1d3a626c03a90ac6bf7042e78d46906a565b86a0dbe926f1d96", size = 1323889, upload-time = "2026-08-16T22:55:56.567Z" }, + { url = "https://files.pythonhosted.org/packages/cc/cc/662b94880f260b0a88de1fdcf60fc9984f6e2a796da549542adc10a7bc83/hypothesis-6.165.10-cp313-cp313-win_amd64.whl", hash = "sha256:c01dd04044c472e47193b54f68e84e08d6ebf4f29551885aa959b015f7cd9747", size = 672346, upload-time = "2026-08-16T22:56:03.792Z" }, + { url = "https://files.pythonhosted.org/packages/3f/77/55e020c9c576532ff7d20bf8b1dfa052ecbd5ada1949b02f76c44c966f7e/hypothesis-6.165.10-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9ccac776b2ca93b324806facd526ccb45da0fd035001c899a35b02c44431e209", size = 784833, upload-time = "2026-08-16T22:55:21.255Z" }, + { url = "https://files.pythonhosted.org/packages/4f/f2/01da2adf829cf549eaddcabb8e8072077fb3d26da4275f4c1e89b2c0af74/hypothesis-6.165.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e5f95f7b622e4171096d92175dda0a560f0955ade9b8a3a07bdcf151f7359611", size = 776545, upload-time = "2026-08-16T22:56:10.159Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8e/58d4f842895220b793c53fc94a6489705b3665bb4d0ae4d338ce03fdf9fb/hypothesis-6.165.10-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f76d1562643693b8a40066f1f96af795b93fd9bcfc9690a1af2ff4c5867ee29e", size = 1107271, upload-time = "2026-08-16T22:54:50.266Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b8/206468912d2153306bb8a41afdfc59e45b7a73a0495bbe4b9cb4f0e79c1d/hypothesis-6.165.10-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60cab3ab4ea468d31a33739ffd7e94ec3e37dea891d65a6582ecc8a477175191", size = 1156915, upload-time = "2026-08-16T22:54:25.89Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d3/bf5a22929b70a4cfd3edf69c5642b029b27ddb5cfda48fa295d384b01abb/hypothesis-6.165.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:22cf19388f0ff6ced8eb3e49c903d14938e4ed909d93bf28383eef451511e424", size = 1281205, upload-time = "2026-08-16T22:54:44.083Z" }, + { url = "https://files.pythonhosted.org/packages/07/a2/d7b2ba444d36fc84d4779f4431e74dd9b023dc63bcf282199f6e48ad39f4/hypothesis-6.165.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:057d0232f1224dcd0b7698902551a4341a7399f90670b036db6c4376715fe889", size = 1324243, upload-time = "2026-08-16T22:55:41.123Z" }, + { url = "https://files.pythonhosted.org/packages/d1/95/afe6b531fd01928c6f63d394ee413fa2338d088b2b44efcc23596b54477e/hypothesis-6.165.10-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ab0f2e9d7d7d4db257f7cf53de3706c2baf124269571f20ffc2bcd6781f03063", size = 616382, upload-time = "2026-08-16T22:55:18.449Z" }, + { url = "https://files.pythonhosted.org/packages/48/86/9b4fb75f520a028edec50ffc904a94d724180395d71feb6d7a0ce7bb6f00/hypothesis-6.165.10-cp314-cp314-win_amd64.whl", hash = "sha256:d1ea02fa8ab3d33eb1125eade81f7136341eb429152c6dbe2ae6f8bc33b3fbdd", size = 672145, upload-time = "2026-08-16T22:54:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ba/f7bbaae0c789bab7ddb764d2056ee1a463cc95a8acbccc90d4184e48b242/hypothesis-6.165.10-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:ed1a5891e59472884a03cb9875483e8fc131c80a275c60967f8afc5458a0c8ff", size = 783287, upload-time = "2026-08-16T22:54:23.751Z" }, + { url = "https://files.pythonhosted.org/packages/3a/83/01ef80772b4abd335c49405576dc503cede94fb5da30ba2643a119013aea/hypothesis-6.165.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:09772e328a26e50486ac572be34f9887f9aa185efe7ebb16bde4e8f6038db1f4", size = 774991, upload-time = "2026-08-16T22:55:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/a3/0b/f47506241f9d5a5a2efe4c65b6bf4830e9d9576e5d3779007a260699e608/hypothesis-6.165.10-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cf3b612542ba174c9da4000b59a4f4c81e8d66f87509be85d3a1b71b5c36413", size = 1105499, upload-time = "2026-08-16T22:54:51.864Z" }, + { url = "https://files.pythonhosted.org/packages/84/fe/abb3909b7089835112fbe75bf00d817d733b3a8032759783db0a24ff1e56/hypothesis-6.165.10-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f69ec5be85ef508e206153bed8eafd03f7995dc464356c8bbb279a1e2b7d56f3", size = 1155685, upload-time = "2026-08-16T22:54:30.94Z" }, + { url = "https://files.pythonhosted.org/packages/73/2f/1964738921640184067121ae77414522fc3f0463fc26c6e25a4f3b8e42ca/hypothesis-6.165.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:dd207497bb985918409a1bb5db85d1875f74e1269487332113b73d1ee7c77647", size = 1279177, upload-time = "2026-08-16T22:54:40.179Z" }, + { url = "https://files.pythonhosted.org/packages/34/c5/312af8ae038d3af9cf3f7f1021c1abfe31c0d9035e4cf63519e0a7dc983e/hypothesis-6.165.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:00de0abdcf8c05c9d0eab735a3c49a276376b55151e6fcb903c2b39a90e5e5c3", size = 1322921, upload-time = "2026-08-16T22:54:42.7Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e7/b0a2fde7570c090a1b914026266a421c751ef10138fffe37fe0ef9e675c0/hypothesis-6.165.10-cp314-cp314t-win_amd64.whl", hash = "sha256:cc2da5aa4edf14743fa9257e5ba3513963999f01211635702479d8e92b8207c8", size = 672147, upload-time = "2026-08-16T22:55:27.527Z" }, + { url = "https://files.pythonhosted.org/packages/47/fd/985aa564d6ffd06483d45a62b40d319df0a703cd8bc1d041de17d102fbaa/hypothesis-6.165.10-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:eeab73050ea58c13dd56e329f594c1dfe32ebd7bb169bbdf4f8ceefbc31ec6b5", size = 782882, upload-time = "2026-08-16T22:55:37.93Z" }, + { url = "https://files.pythonhosted.org/packages/f8/2c/6cc11151e450f72353a490940cd0db704680d07b78dc75dcc9f480e0d0e1/hypothesis-6.165.10-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:4c68e983d0007d014bb01ad4bcbba78bc432c73a1755ff36d5102ceefa18299a", size = 774584, upload-time = "2026-08-16T22:55:51.822Z" }, + { url = "https://files.pythonhosted.org/packages/10/39/ef26fa79c1738dfe9cdb1a3584fb6717d26429ca6c9d011cc4fdf08130c2/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7730d8197086f65d8969a991d6728a1d420a51b19fea06535c896cb43a1e05d0", size = 1104876, upload-time = "2026-08-16T22:54:58.937Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f4/3fcc84e7637f42bf00d987093b9418083ac8db81b87392608a60f4b7c5fd/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7a7980a898a3e6ebe4de1896a0507e3d519edb53fb9b4bda478c9fbeb6514558", size = 1133353, upload-time = "2026-08-16T22:54:28.635Z" }, + { url = "https://files.pythonhosted.org/packages/35/59/21c5c14179c38f8d0de3560e7f1825c083311b3013b63f817d7dc78dfcbd/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5820d009aedb7ae9cfd32f98b1ab0c0bbd6268379c4fab042218b6b655c63f8", size = 1132300, upload-time = "2026-08-16T22:56:08.539Z" }, + { url = "https://files.pythonhosted.org/packages/14/af/fbb56059961e416b2de7b9dc5352db2e8572bd5ea46892957e4c1e5548ab/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37a7ac3d34220800e1107871cc391bca1b00439875925d7d821878b8b791f245", size = 1155175, upload-time = "2026-08-16T22:55:19.824Z" }, + { url = "https://files.pythonhosted.org/packages/0f/53/77fb0c2dad445858555429c4e06cf94a59ae8d2407dd6426b5af97c84828/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:dafa7c9dbe3d802f9bcdf261b29c8a70700fb22839947f06e471f62c46b6257f", size = 1109881, upload-time = "2026-08-16T22:55:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7b/d187f673ff30e6ada640953636f978ffe64a6332f756b64163c2277f8d0c/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:90915635b9648071129b0f72c0673cf8eac9eb84cfd445c5bedef30c714b1ec2", size = 1144963, upload-time = "2026-08-16T22:56:13.428Z" }, + { url = "https://files.pythonhosted.org/packages/e0/60/31d504e364134d60af23e5f6365db0da3cf4a51b3ed3d4836e5a2cff12cf/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:e1bbeb7c506b07ee0422cf9b2f7212fefa4240957f03526d38d27bc6743a0a48", size = 1278684, upload-time = "2026-08-16T22:55:22.971Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e6/89d26834a08c02f8da149e541dd40d7a96f68d9722f43146e69a77436ed7/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:2b36aaffc88625a44f91074c5bbedfdefb9b376c38d1b3c342edcd2e4c8ed16c", size = 1407202, upload-time = "2026-08-16T22:55:14.949Z" }, + { url = "https://files.pythonhosted.org/packages/dc/61/20d1e72246867ea195440092e8bb422c7ddc2f271b87b5b65679d5532719/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:18a3ea838ddea183388f8788750afa8494d79abb5358823be9782585f34445d3", size = 1261395, upload-time = "2026-08-16T22:56:05.448Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9b/ebab6c3c2b90a16abb4119198178652d12aff83cc8ec2cfde5276c69fb1e/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:2a2567b3a03a4a5a7c575c191cfcce321a967df3727803817e75bffbbeaecabe", size = 1279213, upload-time = "2026-08-16T22:55:35.066Z" }, + { url = "https://files.pythonhosted.org/packages/23/78/69b219b524231d36eb20c792e1f01e7cb037e02bd0af1c29f77ed9a969c0/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:8001925fa3dde51cb574e4c9de4c7efe77c4e4d64bd2fd2ef61d5651f9d04f3d", size = 1322367, upload-time = "2026-08-16T22:54:21.279Z" }, + { url = "https://files.pythonhosted.org/packages/55/63/ad5cc153dcc72ae5e7905fb9b3585f3e48ce892a2d6366f90163e867a69d/hypothesis-6.165.10-cp315-abi3.abi3t-win32.whl", hash = "sha256:c6559380469295c4009215fe1cab561301591a3bee2e2fb3f4f96d2273a3affc", size = 666038, upload-time = "2026-08-16T22:56:11.797Z" }, + { url = "https://files.pythonhosted.org/packages/80/32/b62307b73fbc99f0a4381d6f9456df76fbcbb7a27ef7256e26f0376f48ea/hypothesis-6.165.10-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:30797f20ca45e57f526d2df872f63ba453cb4e1091ad542184a7a951af8da79d", size = 671941, upload-time = "2026-08-16T22:55:00.235Z" }, + { url = "https://files.pythonhosted.org/packages/c2/dd/e0f98add0548ef73ea7afac45da1fb8efc854d7f9931db568754d0f963f3/hypothesis-6.165.10-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:c53e9b1c36350df9965ec44d6c0d4e0bbbb38f720dd2b0e1256dc6524d411015", size = 669931, upload-time = "2026-08-16T22:55:50.205Z" }, +] + [[package]] name = "identify" version = "2.6.19" @@ -1174,11 +1270,11 @@ wheels = [ [[package]] name = "idna" -version = "3.15" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] @@ -1204,56 +1300,52 @@ wheels = [ [[package]] name = "jiter" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6e/c1/0cddc6eb17d4c53a99840953f95dd3accdc5cfc7a337b0e9b26476276be9/jiter-0.14.0.tar.gz", hash = "sha256:e8a39e66dac7153cf3f964a12aad515afa8d74938ec5cc0018adcdae5367c79e", size = 165725, upload-time = "2026-04-10T14:28:42.01Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/2a/09f70020898507a89279659a1afe3364d57fc1b2c89949081975d135f6f5/jiter-0.14.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:af72f204cf4d44258e5b4c1745130ac45ddab0e71a06333b01de660ab4187a94", size = 315502, upload-time = "2026-04-10T14:26:47.697Z" }, - { url = "https://files.pythonhosted.org/packages/d6/be/080c96a45cd74f9fce5db4fd68510b88087fb37ffe2541ff73c12db92535/jiter-0.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4b77da71f6e819be5fbcec11a453fde5b1d0267ef6ed487e2a392fd8e14e4e3a", size = 314870, upload-time = "2026-04-10T14:26:49.149Z" }, - { url = "https://files.pythonhosted.org/packages/7d/5e/2d0fee155826a968a832cc32438de5e2a193292c8721ca70d0b53e58245b/jiter-0.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f4ea612fe8b84b8b04e51d0e78029ecf3466348e25973f953de6e6a59aa4c1", size = 343406, upload-time = "2026-04-10T14:26:50.762Z" }, - { url = "https://files.pythonhosted.org/packages/70/af/bf9ee0d3a4f8dc0d679fc1337f874fe60cdbf841ebbb304b374e1c9aaceb/jiter-0.14.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:62fe2451f8fcc0240261e6a4df18ecbcd58327857e61e625b2393ea3b468aac9", size = 369415, upload-time = "2026-04-10T14:26:52.188Z" }, - { url = "https://files.pythonhosted.org/packages/0f/83/8e8561eadba31f4d3948a5b712fb0447ec71c3560b57a855449e7b8ddc98/jiter-0.14.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6112f26f5afc75bcb475787d29da3aa92f9d09c7858f632f4be6ffe607be82e9", size = 461456, upload-time = "2026-04-10T14:26:53.611Z" }, - { url = "https://files.pythonhosted.org/packages/f6/c9/c5299e826a5fe6108d172b344033f61c69b1bb979dd8d9ddd4278a160971/jiter-0.14.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:215a6cb8fb7dc702aa35d475cc00ddc7f970e5c0b1417fb4b4ac5d82fa2a29db", size = 378488, upload-time = "2026-04-10T14:26:55.211Z" }, - { url = "https://files.pythonhosted.org/packages/5d/37/c16d9d15c0a471b8644b1abe3c82668092a707d9bedcf076f24ff2e380cd/jiter-0.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4ab96a30fb3cb2c7e0cd33f7616c8860da5f5674438988a54ac717caccdbaa", size = 353242, upload-time = "2026-04-10T14:26:56.705Z" }, - { url = "https://files.pythonhosted.org/packages/58/ea/8050cb0dc654e728e1bfacbc0c640772f2181af5dedd13ae70145743a439/jiter-0.14.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:3a99c1387b1f2928f799a9de899193484d66206a50e98233b6b088a7f0c1edb2", size = 356823, upload-time = "2026-04-10T14:26:58.281Z" }, - { url = "https://files.pythonhosted.org/packages/b0/3b/cf71506d270e5f84d97326bf220e47aed9b95e9a4a060758fb07772170ab/jiter-0.14.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ab18d11074485438695f8d34a1b6da61db9754248f96d51341956607a8f39985", size = 392564, upload-time = "2026-04-10T14:27:00.018Z" }, - { url = "https://files.pythonhosted.org/packages/b0/cc/8c6c74a3efb5bd671bfd14f51e8a73375464ca914b1551bc3b40e26ac2c9/jiter-0.14.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:801028dcfc26ac0895e4964cbc0fd62c73be9fd4a7d7b1aaf6e5790033a719b7", size = 520322, upload-time = "2026-04-10T14:27:01.664Z" }, - { url = "https://files.pythonhosted.org/packages/41/24/68d7b883ec959884ddf00d019b2e0e82ba81b167e1253684fa90519ce33c/jiter-0.14.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ad425b087aafb4a1c7e1e98a279200743b9aaf30c3e0ba723aec93f061bd9bc8", size = 552619, upload-time = "2026-04-10T14:27:03.316Z" }, - { url = "https://files.pythonhosted.org/packages/b6/89/b1a0985223bbf3150ff9e8f46f98fc9360c1de94f48abe271bbe1b465682/jiter-0.14.0-cp313-cp313-win32.whl", hash = "sha256:882bcb9b334318e233950b8be366fe5f92c86b66a7e449e76975dfd6d776a01f", size = 205699, upload-time = "2026-04-10T14:27:04.662Z" }, - { url = "https://files.pythonhosted.org/packages/4c/19/3f339a5a7f14a11730e67f6be34f9d5105751d547b615ef593fa122a5ded/jiter-0.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:9b8c571a5dba09b98bd3462b5a53f27209a5cbbe85670391692ede71974e979f", size = 201323, upload-time = "2026-04-10T14:27:06.139Z" }, - { url = "https://files.pythonhosted.org/packages/50/56/752dd89c84be0e022a8ea3720bcfa0a8431db79a962578544812ce061739/jiter-0.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:34f19dcc35cb1abe7c369b3756babf8c7f04595c0807a848df8f26ef8298ef92", size = 191099, upload-time = "2026-04-10T14:27:07.564Z" }, - { url = "https://files.pythonhosted.org/packages/91/28/292916f354f25a1fe8cf2c918d1415c699a4a659ae00be0430e1c5d9ffea/jiter-0.14.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e89bcd7d426a75bb4952c696b267075790d854a07aad4c9894551a82c5b574ab", size = 320880, upload-time = "2026-04-10T14:27:09.326Z" }, - { url = "https://files.pythonhosted.org/packages/ad/c7/b002a7d8b8957ac3d469bd59c18ef4b1595a5216ae0de639a287b9816023/jiter-0.14.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b25beaa0d4447ea8c7ae0c18c688905d34840d7d0b937f2f7bdd52162c98a40", size = 346563, upload-time = "2026-04-10T14:27:11.287Z" }, - { url = "https://files.pythonhosted.org/packages/f9/3b/f8d07580d8706021d255a6356b8fab13ee4c869412995550ce6ed4ddf97d/jiter-0.14.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:651a8758dd413c51e3b7f6557cdc6921faf70b14106f45f969f091f5cda990ea", size = 357928, upload-time = "2026-04-10T14:27:12.729Z" }, - { url = "https://files.pythonhosted.org/packages/47/5b/ac1a974da29e35507230383110ffec59998b290a8732585d04e19a9eb5ba/jiter-0.14.0-cp313-cp313t-win_amd64.whl", hash = "sha256:e1a7eead856a5038a8d291f1447176ab0b525c77a279a058121b5fccee257f6f", size = 203519, upload-time = "2026-04-10T14:27:14.125Z" }, - { url = "https://files.pythonhosted.org/packages/96/6d/9fc8433d667d2454271378a79747d8c76c10b51b482b454e6190e511f244/jiter-0.14.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e692633a12cda97e352fdcd1c4acc971b1c28707e1e33aeef782b0cbf051975", size = 190113, upload-time = "2026-04-10T14:27:16.638Z" }, - { url = "https://files.pythonhosted.org/packages/4f/1e/354ed92461b165bd581f9ef5150971a572c873ec3b68a916d5aa91da3cc2/jiter-0.14.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:6f396837fc7577871ca8c12edaf239ed9ccef3bbe39904ae9b8b63ce0a48b140", size = 315277, upload-time = "2026-04-10T14:27:18.109Z" }, - { url = "https://files.pythonhosted.org/packages/a6/95/8c7c7028aa8636ac21b7a55faef3e34215e6ed0cbf5ae58258427f621aa3/jiter-0.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a4d50ea3d8ba4176f79754333bd35f1bbcd28e91adc13eb9b7ca91bc52a6cef9", size = 315923, upload-time = "2026-04-10T14:27:19.603Z" }, - { url = "https://files.pythonhosted.org/packages/47/40/e2a852a44c4a089f2681a16611b7ce113224a80fd8504c46d78491b47220/jiter-0.14.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce17f8a050447d1b4153bda4fb7d26e6a9e74eb4f4a41913f30934c5075bf615", size = 344943, upload-time = "2026-04-10T14:27:21.262Z" }, - { url = "https://files.pythonhosted.org/packages/fc/1f/670f92adee1e9895eac41e8a4d623b6da68c4d46249d8b556b60b63f949e/jiter-0.14.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4f1c4b125e1652aefbc2e2c1617b60a160ab789d180e3d423c41439e5f32850", size = 369725, upload-time = "2026-04-10T14:27:22.766Z" }, - { url = "https://files.pythonhosted.org/packages/01/2f/541c9ba567d05de1c4874a0f8f8c5e3fd78e2b874266623da9a775cf46e0/jiter-0.14.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be808176a6a3a14321d18c603f2d40741858a7c4fc982f83232842689fe86dd9", size = 461210, upload-time = "2026-04-10T14:27:24.315Z" }, - { url = "https://files.pythonhosted.org/packages/ce/a9/c31cbec09627e0d5de7aeaec7690dba03e090caa808fefd8133137cf45bc/jiter-0.14.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26679d58ba816f88c3849306dd58cb863a90a1cf352cdd4ef67e30ccf8a77994", size = 380002, upload-time = "2026-04-10T14:27:26.155Z" }, - { url = "https://files.pythonhosted.org/packages/50/02/3c05c1666c41904a2f607475a73e7a4763d1cbde2d18229c4f85b22dc253/jiter-0.14.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80381f5a19af8fa9aef743f080e34f6b25ebd89656475f8cf0470ec6157052aa", size = 354678, upload-time = "2026-04-10T14:27:27.701Z" }, - { url = "https://files.pythonhosted.org/packages/7d/97/e15b33545c2b13518f560d695f974b9891b311641bdcf178d63177e8801e/jiter-0.14.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:004df5fdb8ecbd6d99f3227df18ba1a259254c4359736a2e6f036c944e02d7c5", size = 358920, upload-time = "2026-04-10T14:27:29.256Z" }, - { url = "https://files.pythonhosted.org/packages/ad/d2/8b1461def6b96ba44530df20d07ef7a1c7da22f3f9bf1727e2d611077bf1/jiter-0.14.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cff5708f7ed0fa098f2b53446c6fa74c48469118e5cd7497b4f1cd569ab06928", size = 394512, upload-time = "2026-04-10T14:27:31.344Z" }, - { url = "https://files.pythonhosted.org/packages/e3/88/837566dd6ed6e452e8d3205355afd484ce44b2533edfa4ed73a298ea893e/jiter-0.14.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:2492e5f06c36a976d25c7cc347a60e26d5470178d44cde1b9b75e60b4e519f28", size = 521120, upload-time = "2026-04-10T14:27:33.299Z" }, - { url = "https://files.pythonhosted.org/packages/89/6b/b00b45c4d1b4c031777fe161d620b755b5b02cdade1e316dcb46e4471d63/jiter-0.14.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:7609cfbe3a03d37bfdbf5052012d5a879e72b83168a363deae7b3a26564d57de", size = 553668, upload-time = "2026-04-10T14:27:34.868Z" }, - { url = "https://files.pythonhosted.org/packages/ad/d8/6fe5b42011d19397433d345716eac16728ac241862a2aac9c91923c7509a/jiter-0.14.0-cp314-cp314-win32.whl", hash = "sha256:7282342d32e357543565286b6450378c3cd402eea333fc1ebe146f1fabb306fc", size = 207001, upload-time = "2026-04-10T14:27:36.455Z" }, - { url = "https://files.pythonhosted.org/packages/e5/43/5c2e08da1efad5e410f0eaaabeadd954812612c33fbbd8fd5328b489139d/jiter-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd77945f38866a448e73b0b7637366afa814d4617790ecd88a18ca74377e6c02", size = 202187, upload-time = "2026-04-10T14:27:38Z" }, - { url = "https://files.pythonhosted.org/packages/aa/1f/6e39ac0b4cdfa23e606af5b245df5f9adaa76f35e0c5096790da430ca506/jiter-0.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:f2d4c61da0821ee42e0cdf5489da60a6d074306313a377c2b35af464955a3611", size = 192257, upload-time = "2026-04-10T14:27:39.504Z" }, - { url = "https://files.pythonhosted.org/packages/05/57/7dbc0ffbbb5176a27e3518716608aa464aee2e2887dc938f0b900a120449/jiter-0.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1bf7ff85517dd2f20a5750081d2b75083c1b269cf75afc7511bdf1f9548beb3b", size = 323441, upload-time = "2026-04-10T14:27:41.039Z" }, - { url = "https://files.pythonhosted.org/packages/83/6e/7b3314398d8983f06b557aa21b670511ec72d3b79a68ee5e4d9bff972286/jiter-0.14.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8ef8791c3e78d6c6b157c6d360fbb5c715bebb8113bc6a9303c5caff012754a", size = 348109, upload-time = "2026-04-10T14:27:42.552Z" }, - { url = "https://files.pythonhosted.org/packages/ae/4f/8dc674bcd7db6dba566de73c08c763c337058baff1dbeb34567045b27cdc/jiter-0.14.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e74663b8b10da1fe0f4e4703fd7980d24ad17174b6bb35d8498d6e3ebce2ae6a", size = 368328, upload-time = "2026-04-10T14:27:44.574Z" }, - { url = "https://files.pythonhosted.org/packages/3b/5f/188e09a1f20906f98bbdec44ed820e19f4e8eb8aff88b9d1a5a497587ff3/jiter-0.14.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1aca29ba52913f78362ec9c2da62f22cdc4c3083313403f90c15460979b84d9b", size = 463301, upload-time = "2026-04-10T14:27:46.717Z" }, - { url = "https://files.pythonhosted.org/packages/ac/f0/19046ef965ed8f349e8554775bb12ff4352f443fbe12b95d31f575891256/jiter-0.14.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8b39b7d87a952b79949af5fef44d2544e58c21a28da7f1bae3ef166455c61746", size = 378891, upload-time = "2026-04-10T14:27:48.32Z" }, - { url = "https://files.pythonhosted.org/packages/c4/c3/da43bd8431ee175695777ee78cf0e93eacbb47393ff493f18c45231b427d/jiter-0.14.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d918a68b26e9fab068c2b5453577ef04943ab2807b9a6275df2a812599a310", size = 360749, upload-time = "2026-04-10T14:27:49.88Z" }, - { url = "https://files.pythonhosted.org/packages/72/26/e054771be889707c6161dbdec9c23d33a9ec70945395d70f07cfea1e9a6f/jiter-0.14.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:b08997c35aee1201c1a5361466a8fb9162d03ae7bf6568df70b6c859f1e654a4", size = 358526, upload-time = "2026-04-10T14:27:51.504Z" }, - { url = "https://files.pythonhosted.org/packages/c3/0f/7bea65ea2a6d91f2bf989ff11a18136644392bf2b0497a1fa50934c30a9c/jiter-0.14.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:260bf7ca20704d58d41f669e5e9fe7fe2fa72901a6b324e79056f5d52e9c9be2", size = 393926, upload-time = "2026-04-10T14:27:53.368Z" }, - { url = "https://files.pythonhosted.org/packages/3c/a1/b1ff7d70deef61ac0b7c6c2f12d2ace950cdeecb4fdc94500a0926802857/jiter-0.14.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:37826e3df29e60f30a382f9294348d0238ef127f4b5d7f5f8da78b5b9e050560", size = 521052, upload-time = "2026-04-10T14:27:55.058Z" }, - { url = "https://files.pythonhosted.org/packages/0b/7b/3b0649983cbaf15eda26a414b5b1982e910c67bd6f7b1b490f3cfc76896a/jiter-0.14.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:645be49c46f2900937ba0eaf871ad5183c96858c0af74b6becc7f4e367e36e06", size = 553716, upload-time = "2026-04-10T14:27:57.269Z" }, - { url = "https://files.pythonhosted.org/packages/97/f8/33d78c83bd93ae0c0af05293a6660f88a1977caef39a6d72a84afab94ce0/jiter-0.14.0-cp314-cp314t-win32.whl", hash = "sha256:2f7877ed45118de283786178eceaf877110abacd04fde31efff3940ae9672674", size = 207957, upload-time = "2026-04-10T14:27:59.285Z" }, - { url = "https://files.pythonhosted.org/packages/d6/ac/2b760516c03e2227826d1f7025d89bf6bf6357a28fe75c2a2800873c50bf/jiter-0.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:14c0cb10337c49f5eafe8e7364daca5e29a020ea03580b8f8e6c597fed4e1588", size = 204690, upload-time = "2026-04-10T14:28:00.962Z" }, - { url = "https://files.pythonhosted.org/packages/dc/2e/a44c20c58aeed0355f2d326969a181696aeb551a25195f47563908a815be/jiter-0.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:5419d4aa2024961da9fe12a9cfe7484996735dca99e8e090b5c88595ef1951ff", size = 191338, upload-time = "2026-04-10T14:28:02.853Z" }, +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/1f/10936e16d8860c70698a1aa939a46aa0224813b782bce4e000e637da0b2d/jiter-0.16.0.tar.gz", hash = "sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c", size = 176431, upload-time = "2026-06-29T13:05:13.657Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/c0/555fc60473d30d66894ba825e63615e3be7524fac23858356afa7a38906c/jiter-0.16.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:41977aa5654023948c2dae2a81cbf9c43343954bef1cd59a154dd15a4d84c195", size = 306203, upload-time = "2026-06-29T13:03:36.243Z" }, + { url = "https://files.pythonhosted.org/packages/d0/2b/c3eaf16f5d7c9bad66ea32f40a95bd169b29a91217fcc7f081375157e99c/jiter-0.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d28bb3c26762358dadf3e5bf0bccd29ae987d65e6988d2e6f49829c76b003c09", size = 306489, upload-time = "2026-06-29T13:03:37.846Z" }, + { url = "https://files.pythonhosted.org/packages/96/3f/02fdfc6705cad96127d883af5c34e4867f554f29ec7705ec1a46156400a9/jiter-0.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536", size = 335453, upload-time = "2026-06-29T13:03:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a6/e4bda5920d4b0d7c5dfb7174ce4a6b2e4d3e11c9162c452ef0eab4cdbdbd/jiter-0.16.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450", size = 361625, upload-time = "2026-06-29T13:03:40.597Z" }, + { url = "https://files.pythonhosted.org/packages/b7/97/4e6b59b2c6e55cbb3e183595f81ad65dcfb21c915fee5e19e335df21bc55/jiter-0.16.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e", size = 456958, upload-time = "2026-06-29T13:03:42.074Z" }, + { url = "https://files.pythonhosted.org/packages/15/e0/97e9557686d2f94f4b93786eccb7eed28e9228ad132ea8237f44727314a7/jiter-0.16.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8", size = 372017, upload-time = "2026-06-29T13:03:43.658Z" }, + { url = "https://files.pythonhosted.org/packages/0f/94/db768b6938e0df35c86beeba3dfbbb025c9ee5c19e1aa271f2396e50864d/jiter-0.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026", size = 343320, upload-time = "2026-06-29T13:03:45.226Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d6/5a59d938244a30735fe62d9433fd325f9021ea29d89780ea4596ea93bc89/jiter-0.16.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053", size = 350520, upload-time = "2026-06-29T13:03:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/c4a857f49c9af125f6bbcac7e3eee7f7978ed89682833062e2dbf62576b1/jiter-0.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e", size = 387550, upload-time = "2026-06-29T13:03:48.361Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d6/5fbc2f7d6b67b754caa61a993a2e626e815dec47ffc2f9e35f01adfebec7/jiter-0.16.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb", size = 515424, upload-time = "2026-06-29T13:03:49.881Z" }, + { url = "https://files.pythonhosted.org/packages/ed/54/284f0164b64a5fed915fea6ba7e9ba9b3d8d37c67d59cf2e3bb99d45cdfe/jiter-0.16.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84", size = 546981, upload-time = "2026-06-29T13:03:51.363Z" }, + { url = "https://files.pythonhosted.org/packages/13/c5/2a467585a576594384e1d2c43e1224deaafc085f24e243529cf98beef8e1/jiter-0.16.0-cp313-cp313-win32.whl", hash = "sha256:c9c53be232c2e206ef9cdbad81a48bfa74c3d3f08bcf8124630a8a748aad993e", size = 202853, upload-time = "2026-06-29T13:03:53.015Z" }, + { url = "https://files.pythonhosted.org/packages/88/6a/de61d04b9eec69c71719968d2f716532a3bc121170c44a39e14979c6be81/jiter-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:baad945ed47f163ad833314f8e3288c396118934f94e7bbb9e243ce4b341a4fd", size = 196160, upload-time = "2026-06-29T13:03:54.447Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/b390ed59bafb3f31d008d1218578f10327714484b334439947f7e5b11e7f/jiter-0.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:3c1fd2dbe1b0af19e987f03fe66c5f5bd105a2229c1aff4ab14890b24f41d21a", size = 189862, upload-time = "2026-06-29T13:03:55.754Z" }, + { url = "https://files.pythonhosted.org/packages/a7/89/bc4f1b57d5da938fd344a466396541e586d161320d70bffd929aaafcd8f4/jiter-0.16.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b2c61484666ad42726029af0c00ef4541f0f3b5cdc550221f56c2343208018ee", size = 308239, upload-time = "2026-06-29T13:03:57.205Z" }, + { url = "https://files.pythonhosted.org/packages/65/7a/c415453e5213001bf3b411ff65dec3d303b0e76a4a2cfea9768cd4960994/jiter-0.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:63efadc657488f45db1c676d81e704cac2abf3fdb892def1faea61db053127e2", size = 308928, upload-time = "2026-06-29T13:03:58.643Z" }, + { url = "https://files.pythonhosted.org/packages/11/fc/1f4fb7ebf9a724c7741994f4aae18fba1e2f3133df14521a79194952c34a/jiter-0.16.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf0d73f50e7b6935677854f6e8e31d499ca7064dd24734f703e060f5b237d883", size = 336998, upload-time = "2026-06-29T13:04:00.071Z" }, + { url = "https://files.pythonhosted.org/packages/a0/8d/72cadaac05ccfa7cc3a0a2232862e6c72443ca40cf300ba8b57f9f18b69b/jiter-0.16.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3ea07d9bc8e7d03a9fbc051295462e6dbc295b894fd72457c3136e3e43d898", size = 362112, upload-time = "2026-06-29T13:04:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/58/4a/c4b0d5f651fda90a24ffce9f8d56cde462a2e09d31ae3de3c68cef34c04e/jiter-0.16.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26798522707abb47d767db536e4148ceac1b14446bf028ee85e579a2e043cfe5", size = 459807, upload-time = "2026-06-29T13:04:03.214Z" }, + { url = "https://files.pythonhosted.org/packages/80/58/ef77879ea9aa56b50824edc5a445e226422c7a8d211f3fd2a56bcb9493cf/jiter-0.16.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc837c1b9631be10abfe0191537fe8009838204cec7e44827401ace390ddb567", size = 373181, upload-time = "2026-06-29T13:04:04.629Z" }, + { url = "https://files.pythonhosted.org/packages/49/2e/ffbc3f254e4d8a66da3062c624a7df4b7c2b2cf9e1fe43cf394b3e104041/jiter-0.16.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49060fd70737fad59d33ba9dcc0d83247dc9e77187de26053a19c16c9f32bd69", size = 344927, upload-time = "2026-06-29T13:04:06.067Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f6/0be5dc6d64a89f80aa8fec984f94dedb2973e251edcae55841d60786d578/jiter-0.16.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:adbb8edeadd431bc4477879d5d371ece7cb1334486584e0f252656dd7ffada29", size = 352754, upload-time = "2026-06-29T13:04:07.477Z" }, + { url = "https://files.pythonhosted.org/packages/da/6e/7d31243b3b91cd261dd19e9d3557fc3251a80883d3d8049c86174e7ab7af/jiter-0.16.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:31aaee5b80f672c1dc21272bcfb9cbdcfc1ea04ff50f00ed5af500b80c44fa93", size = 390553, upload-time = "2026-06-29T13:04:08.92Z" }, + { url = "https://files.pythonhosted.org/packages/25/33/51ae371fde3c88897520f62b4d5f8b27ad7103e2bb10812ff52195609853/jiter-0.16.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:6722bcef4ffc86c835574b1b2fac6b33b9fb4a889c781e67950e891591f3c55a", size = 516900, upload-time = "2026-06-29T13:04:10.407Z" }, + { url = "https://files.pythonhosted.org/packages/a0/45/6449b3d123ea439ba79507c657288f461d55049e7bcbdc2cf8eb8210f491/jiter-0.16.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:5ab4f50ff971b611d656554ea10b75f80097392c827bc32923c6eeb6386c8b00", size = 548754, upload-time = "2026-06-29T13:04:12.046Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e7/fd2fb11ae3e2649333da3aa170d04d7b3000bbdc3b270f6513382fdf4e04/jiter-0.16.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:710cc51d4ebdcd3c1f70b232c1db1ea1344a075770422bbd4bede5708335acbe", size = 122381, upload-time = "2026-06-29T13:04:13.413Z" }, + { url = "https://files.pythonhosted.org/packages/26/80/f0b147a62c315a164ed2168908286ca302310824c218d3aae52b06c0c9a9/jiter-0.16.0-cp314-cp314-win32.whl", hash = "sha256:57b37fc887a32d44798e4d8ebfa7c9683ff3da1d5bf38f08d1bb3573ccb39106", size = 204578, upload-time = "2026-06-29T13:04:14.813Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/4758a14304b4523a6f5adb2419340086aa3593bd4327c2b25b5948a90548/jiter-0.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:cbd18dd5e2df96b580487b5745adf57ef64ad89ba2d9662fc3c19386acce7db8", size = 198154, upload-time = "2026-06-29T13:04:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/26/be/41fa54a2e7ea41d6c99f1dc5b1f0fd4cb474680304b5d268dd518e81da3a/jiter-0.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:a32d2027a9fa67f109ff245a3252ece3ccc32cc56703e1deab6cc846a59e0585", size = 191458, upload-time = "2026-06-29T13:04:17.707Z" }, + { url = "https://files.pythonhosted.org/packages/81/6b/59127338b86d9fe4d99418f5a15118bea778103ee0fe9d9dd7e0af174e95/jiter-0.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2577196f4474ef3fc4779a088a23b0897bbf86f9ea3679c372d45b8383b43207", size = 316739, upload-time = "2026-06-29T13:04:19.663Z" }, + { url = "https://files.pythonhosted.org/packages/2d/95/49461034d5388196d3dabf98748935f017b7785d8f3f5349f834bcc4ed0d/jiter-0.16.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e89e008a93c01104161c75b4988e58716b01d62307ebfe161e52a56d2a818", size = 340911, upload-time = "2026-06-29T13:04:21.257Z" }, + { url = "https://files.pythonhosted.org/packages/cd/97/a4369f2fb82cb3dda13b98622f31249b2e014b223fe64ee534413ad72294/jiter-0.16.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2e9efbe042210df657bade597f66d6d75723e3d8f45a12ea6d8167ff8bbce3", size = 361747, upload-time = "2026-06-29T13:04:22.677Z" }, + { url = "https://files.pythonhosted.org/packages/28/51/49b6ed456261646e1906016a6760367a28aacd3c24805e4e5fe64116c1db/jiter-0.16.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f4d9e473a5ce7d27fef8b848df4dc16e283893d3f53b4a585e72c9595f3c284", size = 460225, upload-time = "2026-06-29T13:04:24.441Z" }, + { url = "https://files.pythonhosted.org/packages/33/b5/5689aff4f66c5b60be63106e591dbfcba2190df97d2c9c7cf052361ddb98/jiter-0.16.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d30a4a1c87713060c8d1cc59a7b6c8fb6b8ef0a6900368014c76c87922a2929", size = 373169, upload-time = "2026-06-29T13:04:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/a2/96/3ae1b85ee0d6d6cab254fb7f8da018272b932bbf2d69b07e98aa2a96c746/jiter-0.16.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae96332410f866e5900d809298b1ed82735932986c672495f9701daacd80620", size = 350332, upload-time = "2026-06-29T13:04:27.302Z" }, + { url = "https://files.pythonhosted.org/packages/15/32/c99d7bafd78986556c95bf60ce84c6cc98786eac56066c12d7f828bb6747/jiter-0.16.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:da3d7ec75dc83bb18bca888b5edfae0656a26849056c59e05a7728badd17e7af", size = 353377, upload-time = "2026-06-29T13:04:28.731Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/f99a8e571287c3dec766bcc18528bbe8e8fb5365522ab5e6d64c93e87066/jiter-0.16.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee6162b77d49a9939229df666dfa8af3e656b6701b54c4c84966d740e189264e", size = 387746, upload-time = "2026-06-29T13:04:30.319Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/c78a5b3f71040e34eb5917df26fb7ae9a2174cad1ccbf277512507c53a6e/jiter-0.16.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:63ffdbdae7d4499f4cda14eadc12ddcabef0fc0c081191bdc2247489cb698077", size = 517292, upload-time = "2026-06-29T13:04:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f7/095b38eda4c70d03651c403f29a5590f16d12ddc5d544aac9f9cddf72277/jiter-0.16.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a111256a7193bea0759267b10385e5870949c239ed7b6ddbaaf57573edb38734", size = 549259, upload-time = "2026-06-29T13:04:33.721Z" }, + { url = "https://files.pythonhosted.org/packages/2e/c5/6a0207d90e5f656d95af98ebd0934f382d37674416f215aeda2ff8063e51/jiter-0.16.0-cp314-cp314t-win32.whl", hash = "sha256:de5ba8763e56b793561f43bed197c9ea55776daa5e9a6b91eed68a909bc9cdbf", size = 206523, upload-time = "2026-06-29T13:04:35.068Z" }, + { url = "https://files.pythonhosted.org/packages/a5/31/c757d5f30a8980fd945ce7b98be10be9e4ff59c7c42f5fd86804c2e87db8/jiter-0.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b8a3f9a6008048fe9def7bf465180564a6e458047d2ce499149cfbe73c3ae9db", size = 200366, upload-time = "2026-06-29T13:04:36.61Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a2/d88de6d313d734a544a7901353ad5db67cb38dcfcd91713b7979dafc345d/jiter-0.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0fa25b09b13075c46f5bc174f2690525a925a4fc2f7c82969a2bbabff22386ce", size = 190516, upload-time = "2026-06-29T13:04:38.004Z" }, ] [[package]] @@ -1520,21 +1612,19 @@ wheels = [ [[package]] name = "openai" -version = "2.53.0" +version = "3.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, + { name = "httpx2" }, { name = "jiter" }, { name = "pydantic" }, { name = "sniffio" }, - { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ef/cf/36e3e7235fdf6d125c052acc0970924611b17a20a4fe580596faf4566a65/openai-2.53.0.tar.gz", hash = "sha256:baf5802ad08980e1d9d561e1b996e800c8bcd14af5847c6d0e7a5cc59e4d4116", size = 1099435, upload-time = "2026-08-03T21:42:01.664Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/9c/ba0c292b4032ede74c249ca314ad64eb1bb5a03a843f6e01facb02f80cd8/openai-3.3.1.tar.gz", hash = "sha256:6f22807de1a976c932cecda620e8172a8c3fdbaeed29c7f21564e0c2410edf56", size = 1282113, upload-time = "2026-08-19T16:31:35.006Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/78/0f/cc6afea3542a5142c5d8fc8211c5e059a8375105d004a41dfa2c7948dbb0/openai-2.53.0-py3-none-any.whl", hash = "sha256:c694ffc747a3c4d1663ef2b07b811315a476164ee5efa3a993967349ebca7618", size = 1659829, upload-time = "2026-08-03T21:41:59.581Z" }, + { url = "https://files.pythonhosted.org/packages/6a/db/2b7a1b3de659bb82aef979116c74e809982b13e42c057759767552b5155f/openai-3.3.1-py3-none-any.whl", hash = "sha256:9652df7fdf8ee6f5bd58e0a12f2b1d414a18e0f06bb7a9a57c8643a5f5469bd3", size = 1690337, upload-time = "2026-08-19T16:31:32.812Z" }, ] [[package]] @@ -1784,7 +1874,7 @@ wheels = [ [[package]] name = "pre-commit" -version = "4.6.1" +version = "4.6.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cfgv" }, @@ -1793,9 +1883,9 @@ dependencies = [ { name = "pyyaml" }, { name = "virtualenv" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/25/3a/ddb78f32a0814e66b18a099377a106a2dcdce92d86a034d69d65df9b256e/pre_commit-4.6.1.tar.gz", hash = "sha256:03e809865c7d178b9979d06c761fcbfe6808fdaded8581a745bb110e52050421", size = 198646, upload-time = "2026-07-21T20:56:58.225Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/89/1f3e8e1fc3e97de0fa963495832f581f025f29471602a309e48808244292/pre_commit-4.6.2.tar.gz", hash = "sha256:8f5d7bfb021ecdbcd9d49d89847082dd24172ccde534390081a679ad046e2441", size = 198670, upload-time = "2026-08-10T22:07:18.421Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/49/bc925106abcdac498074f2cbe6137e94e09f418dd2b7775df5b577dc0313/pre_commit-4.6.1-py2.py3-none-any.whl", hash = "sha256:0e3b2942510d1fb34eec167a3ec57331bf8442122f1153a9fb8b58f5c49b2717", size = 226186, upload-time = "2026-07-21T20:56:57.064Z" }, + { url = "https://files.pythonhosted.org/packages/45/e2/bbb7129c9e7999a6b8ee9cca3b66486c25c423ab5a75f34071798b74ce94/pre_commit-4.6.2-py2.py3-none-any.whl", hash = "sha256:e2dde9a75d3bce11bd3831c26d134df00a2803c1d818be6a0383c3dcda25dc4e", size = 226202, upload-time = "2026-08-10T22:07:16.942Z" }, ] [[package]] @@ -2059,11 +2149,11 @@ wheels = [ [[package]] name = "pygments" -version = "2.20.0" +version = "2.21.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, ] [[package]] @@ -2182,11 +2272,11 @@ wheels = [ [[package]] name = "python-dotenv" -version = "1.2.2" +version = "1.2.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/53/ed9d74092561d4b01a2ef1349d52cdbc135e526c245f366b089cfca6de49/python_dotenv-1.2.3.tar.gz", hash = "sha256:a20a594dabeaa385725aa239d5244871c143ecb356add8a20fcf23773a6c3a35", size = 58945, upload-time = "2026-08-16T16:54:54.067Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, + { url = "https://files.pythonhosted.org/packages/0d/17/c5c6b53ddc18f297992099b3d9ec16c855c0ccc83263a21fe4d1c625ec6c/python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9", size = 22780, upload-time = "2026-08-16T16:54:52.473Z" }, ] [[package]] @@ -2352,27 +2442,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.16.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/73/e1/4508a569211b35599016e84ba65c1a992b7a4004b4b6c4bea02a851cba1b/ruff-0.16.2.tar.gz", hash = "sha256:c3d7828d12e8927a6fc65fe38e2c2541b9e762d360a1786d752cb1b8883b3c9c", size = 4885811, upload-time = "2026-08-07T13:31:01.432Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/57/db19951540f98859c956b50bdb4d31089b4d91e9f15e2968e7d5193806d5/ruff-0.16.2-py3-none-linux_armv6l.whl", hash = "sha256:3c8de4cf2181f01d57946d87d777aa52916976fc09942aed89938fab5e013318", size = 10847925, upload-time = "2026-08-07T13:30:14.468Z" }, - { url = "https://files.pythonhosted.org/packages/13/5a/995fe85a8470d3e391ac0f7fa8054bb454eaf33ee138196d6172ed1079c0/ruff-0.16.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a48cc05c6fbc811ca81b5d7ba95375affea6582d1b8024e455e41afbbf55344", size = 11072662, upload-time = "2026-08-07T13:30:18.143Z" }, - { url = "https://files.pythonhosted.org/packages/32/53/370d767c61c71a971a4ace36703a7ecd8c393956349a7325d7fab2b56827/ruff-0.16.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2c0d14fcbb26c91f0f867a6dc9bd71bbc30b1b6151829c884f23faeab2e5700", size = 10566771, upload-time = "2026-08-07T13:30:20.899Z" }, - { url = "https://files.pythonhosted.org/packages/85/d6/9d96948caf5a632be62d62202d5ec914d6856f204fd79eb036e5915e79ea/ruff-0.16.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:335c621622c4650330be50842561c6586ac6971bb8ab5407fe34dcc9efb16bbe", size = 10975825, upload-time = "2026-08-07T13:30:23.517Z" }, - { url = "https://files.pythonhosted.org/packages/3b/92/ea87129b3414acb0b5770563779c51804d37ac67675c7ba35447ddb14773/ruff-0.16.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20e66910f2c37cc753f9ef6580c914a621b80c4fa3549d3e3521e29d0f5bfc3f", size = 10649437, upload-time = "2026-08-07T13:30:26.097Z" }, - { url = "https://files.pythonhosted.org/packages/ac/43/f8f291dcd4af5bb7872b74fdfa41a7cd7c856ca1d4069670971cf1b9f5cb/ruff-0.16.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7e36fbfba65510548156902bcf1350a979a958ce0347ce0f90d73894036b39f", size = 11446761, upload-time = "2026-08-07T13:30:28.752Z" }, - { url = "https://files.pythonhosted.org/packages/71/4a/ef991fb2fcf516ab71f0808adcdd8da5e18c8cde447f4ceaf5f47a5132a5/ruff-0.16.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f0eab35f80df8f134aae5d1630e751901321d317cc8e50dc39e36fa3ed34cd12", size = 12336364, upload-time = "2026-08-07T13:30:31.468Z" }, - { url = "https://files.pythonhosted.org/packages/f3/24/f615e74f307e6ca0e56a482872477b856c70d530aa356abfb6dfe5ca8a80/ruff-0.16.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ea8c0594feb894e89c8c61ab9c103d38b0ea72dfde6c594107147ca31b1140", size = 11630720, upload-time = "2026-08-07T13:30:34.426Z" }, - { url = "https://files.pythonhosted.org/packages/c5/d3/8ef50149e8412a77f7ab409efdef0e2b23803707a3863da4fc64cb23d459/ruff-0.16.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab3d62dde0b19facdd632008cc4827fc28ada7736c6bd35ab6f1050f0bfed53f", size = 11466130, upload-time = "2026-08-07T13:30:36.958Z" }, - { url = "https://files.pythonhosted.org/packages/dd/a7/a19334985c4dea8c381981fa252cd854c7ee52dc4b1686dc16f4a911c702/ruff-0.16.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e43e1f5b8388da9eca1b9e88328d47a5cec794633ccf6f7484ac2dd15eee92c0", size = 11523634, upload-time = "2026-08-07T13:30:39.822Z" }, - { url = "https://files.pythonhosted.org/packages/6e/6c/96d192b0e742412ceda08c0a50f9669b253dde9fd6a60ea1a10c9fa79a63/ruff-0.16.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c24788a980581e1d7ea3a0cbe4344c4fbeb0a6a9b1f4713aa46bb104f8294690", size = 10949807, upload-time = "2026-08-07T13:30:42.745Z" }, - { url = "https://files.pythonhosted.org/packages/fa/51/e26599ceca11e79ee255c7df515995561edf87e9ca1893284e44d98f5a86/ruff-0.16.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:81806b08329130005dd4a8a8394a0c9da8c6f4cafb16ba438d2a2ee6a18bedf1", size = 10646891, upload-time = "2026-08-07T13:30:45.522Z" }, - { url = "https://files.pythonhosted.org/packages/68/01/800c4b1f97bc8d7c6029e06b1f20473a3cf1e13c4933d8f3342add83fc55/ruff-0.16.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4ce4e02bad779bef557f541a1b31f20d6abeae1cc05ed1b1ac019d4ffd1044c8", size = 11162063, upload-time = "2026-08-07T13:30:48.131Z" }, - { url = "https://files.pythonhosted.org/packages/e4/d0/1477ea50fc5a0d4b0b71d1d63d50770bdd794d90b43e37a7618e63ec9894/ruff-0.16.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e0422abdf70070255fc4073ce9dfc814cc03db577013761ddd09bc1e4a9a4fbd", size = 11556038, upload-time = "2026-08-07T13:30:50.686Z" }, - { url = "https://files.pythonhosted.org/packages/b8/76/a7776f32048d991e16d4fa8ff91790b877342d3596cc3ed04acdbf1aaedc/ruff-0.16.2-py3-none-win32.whl", hash = "sha256:bf3a63d78fb39f4bf5ac8ae52051c5520505301abe19ba4e204c453b3f09bb0b", size = 10872850, upload-time = "2026-08-07T13:30:53.471Z" }, - { url = "https://files.pythonhosted.org/packages/00/0d/929c800d920e61397d82a01b60bffc68da3052c17d31de59efaad2e4ed75/ruff-0.16.2-py3-none-win_amd64.whl", hash = "sha256:bcabe2f6d0fc7819f1431793005af4e4de7371927d037345bf941252b195b9fa", size = 12023338, upload-time = "2026-08-07T13:30:56.193Z" }, - { url = "https://files.pythonhosted.org/packages/5b/6c/93e26c22c5f78ff87363e07da49c84955affbeb1098bd1936bf3b3f293bf/ruff-0.16.2-py3-none-win_arm64.whl", hash = "sha256:d614e95cedf38a2053fd351c55b103ba30d017d61688fdbfd40ee0412852a99f", size = 11374065, upload-time = "2026-08-07T13:30:58.775Z" }, +version = "0.16.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/8f/d8074b1f25e003164087a8bfe79a0f1a3945135764dbb6aaab04103dcaf9/ruff-0.16.4.tar.gz", hash = "sha256:13171aa9d9af2240ee3504e639de73122c67e74036de5ba2e1d01422cd17e3dc", size = 4899731, upload-time = "2026-08-20T17:43:59.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/80/779895ef584e089d22f2c6df0d0e99a65ec2df0805f1fffd439415b8c1f0/ruff-0.16.4-py3-none-linux_armv6l.whl", hash = "sha256:df4075f71ddac40b9934af60c3ec8a53047dd5a5fdc43224e6e4e8e9a27cb6f7", size = 10006909, upload-time = "2026-08-20T17:43:16.888Z" }, + { url = "https://files.pythonhosted.org/packages/a9/e6/f553199b5e8927a05cb5c422d921fd0656b29ab976e91c44802107c6b0da/ruff-0.16.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0c95538517af68004306b0fb3214ff2f2af67a65092aee77cd9eb86db6656604", size = 10240201, upload-time = "2026-08-20T17:43:19.337Z" }, + { url = "https://files.pythonhosted.org/packages/1c/70/4a6dc4bb34da4dee35e30f09bbd1bfbdd26f33b62fb9b8df31f08a199cd2/ruff-0.16.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:963f83df8e69e575b64d67dd447ebbc917db41a14bf38d4593a4183e7aaa8255", size = 9835122, upload-time = "2026-08-20T17:43:21.708Z" }, + { url = "https://files.pythonhosted.org/packages/24/12/c6e22d686372c15bcb7af99831f1a1be96df696491babf4f24e4f942c527/ruff-0.16.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32a5057c7ff3f6e6480a48fccfb3a412a690f48a3d03ac5cf08177d6c2da3ade", size = 9977162, upload-time = "2026-08-20T17:43:24.236Z" }, + { url = "https://files.pythonhosted.org/packages/46/49/72b10ec912f5ab5854992eaf7aa7cd36729b6937d9dc4e0fb41b3bf428ec/ruff-0.16.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3dce8d9b0c57c265b91885a66a567d8ea1372e8eb4e250fa8e5e3f579e99cff", size = 9829789, upload-time = "2026-08-20T17:43:26.966Z" }, + { url = "https://files.pythonhosted.org/packages/fa/80/0f30e32e7f6ee26edc39075502db9d368d788a44a79b55f763eb4ab03796/ruff-0.16.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7dc651db49283c69f8e72c834eec4fe5573e4c646856aebece0ce385dceb2a80", size = 10527949, upload-time = "2026-08-20T17:43:29.384Z" }, + { url = "https://files.pythonhosted.org/packages/52/3d/86e8ad3542169e56cac3859a343afdb9df2ad54d35a59ce1e67baee83421/ruff-0.16.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3817b87dbcabc92f13b05019257c5b89b5b4d51b5fb20f56fb5235ceb723cd07", size = 11333695, upload-time = "2026-08-20T17:43:31.872Z" }, + { url = "https://files.pythonhosted.org/packages/d0/16/481c29b380c20a0054a8261066665e1b3488e23636c49d0a43e75975b9bb/ruff-0.16.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9fce1499134b2c8c68e5166f95705a5812062bb93aacc5f9873bb1a27084bc7", size = 10727741, upload-time = "2026-08-20T17:43:34.596Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b6/56bc0b8cf45b54b28b3a5e6381c8945d51b5b18adf659454c32295209a31/ruff-0.16.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2d812e482f5a7e02eee26cd73d2a37ebbdf47d795ea63ba1b89110ae93e9fb3", size = 10286522, upload-time = "2026-08-20T17:43:37.288Z" }, + { url = "https://files.pythonhosted.org/packages/e8/8b/b345b4fb110f2fbe2bd31eabd271e5e8b3b7e4ee6c0e02f2dc6be78db000/ruff-0.16.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6baaf984aa7976edf93d3b627fe2d1d22ee94bbca05fa6f90fc76d73924e3454", size = 10584182, upload-time = "2026-08-20T17:43:39.984Z" }, + { url = "https://files.pythonhosted.org/packages/29/e5/827b34041c35f58774a9681a4213994c164fc987800f4dddabcf451da0bf/ruff-0.16.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:bdfcf0b28662eb890372d50f92c283bb94e67e7635ed93c7fd533970acff7b2b", size = 10134195, upload-time = "2026-08-20T17:43:42.351Z" }, + { url = "https://files.pythonhosted.org/packages/0f/10/d0bffcdd6729b87afc82ba0ef377173356a7dc8e972f5179968cf2fdf98c/ruff-0.16.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b66b02cb9b04f537643cadf5768e5f98dc461890d530cb67113d71c8c76e605d", size = 9825821, upload-time = "2026-08-20T17:43:44.532Z" }, + { url = "https://files.pythonhosted.org/packages/f5/32/0db2a863b796ca62d83e92a07a3ccf00921b14db02059347576a2fda3d4b/ruff-0.16.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8528bf9a4b291a60bf02ea453511e8ce6215bd2b982ee80405b66b008b6c30a0", size = 10267658, upload-time = "2026-08-20T17:43:46.989Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a0/fbdeb59e48c6261f523e56c8f12e9c08fbe693786595cc7e3959207a9232/ruff-0.16.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fbd85d2875fdd67e833213a651f613bbf25303abf6aa822a5121f4531195678d", size = 10697071, upload-time = "2026-08-20T17:43:49.891Z" }, + { url = "https://files.pythonhosted.org/packages/aa/28/0c6dd865859c6d17bc8ccc34cb72b0e02d6c7eb25e8a1e22b5bea681e2c0/ruff-0.16.4-py3-none-win32.whl", hash = "sha256:312769988007aaeb8e189b443ccdd03c0e6374489e053467be6d96518ebff76e", size = 10021687, upload-time = "2026-08-20T17:43:52.281Z" }, + { url = "https://files.pythonhosted.org/packages/a3/03/e724450f621698117f9aa6dd241c94d0274ae96781378dc86745ae29f0e7/ruff-0.16.4-py3-none-win_amd64.whl", hash = "sha256:05d9d27a18c4bcbefada602480ec9e01e0bc949d432e0ced5df77edac195919c", size = 10567657, upload-time = "2026-08-20T17:43:54.78Z" }, + { url = "https://files.pythonhosted.org/packages/0e/fe/da8b9e1347696bb22120b77280ec5ce25d500ca5cb39d5ad6e5c18de19c1/ruff-0.16.4-py3-none-win_arm64.whl", hash = "sha256:a3a61621c9b6f6a89573e938a080e648f1695baa3f58570a3a707bc51ff65a21", size = 10451579, upload-time = "2026-08-20T17:43:57.135Z" }, ] [[package]] @@ -2424,38 +2514,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "sqlalchemy" -version = "2.0.51" +version = "2.0.52" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/02/f1/a7a892f18d4d224e6b26f706531eafccc41e37594d37d304786969ee13cb/sqlalchemy-2.0.51.tar.gz", hash = "sha256:804dccd8a4a6242c4e30ad961e540e18a588f6527202f2d6791b01845d59fdc9", size = 9912201, upload-time = "2026-06-15T15:41:20.012Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/fe/a210d52fd1a90ecfae8a78e9d8b27e18d733d60818a8bf250ff690b75120/sqlalchemy-2.0.51-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7c2056838b6685b72fdb36c99996cf862753461a62f2e84f4196371d3b2d6a07", size = 2157184, upload-time = "2026-06-15T16:08:50.374Z" }, - { url = "https://files.pythonhosted.org/packages/17/6b/2dce8369b199cb855110e056032f94a9f66dacc2237d3d39c115a86eac56/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:483b11bd46bf35fc14c52faf338b04300c9e6ce554bce9b11be85bfec3bc3195", size = 3284735, upload-time = "2026-06-15T16:19:46.934Z" }, - { url = "https://files.pythonhosted.org/packages/53/ff/dbc495b8a14da840faffb353857a72d4190113cac33727906fb997047f0f/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1bed1ee8b01da6088210aa9412023326fb98a599ba502e6118308601dcbef77f", size = 3302756, upload-time = "2026-06-15T16:26:41.336Z" }, - { url = "https://files.pythonhosted.org/packages/cf/d5/fde8f4dddcf518ee15ab35a7c6a28acc32c8ba548d1d2aa451f96e6dbb0b/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:72ca54c952107ba5cd58854b67a5a6268631289d21651a1235396f3b98b47400", size = 3232055, upload-time = "2026-06-15T16:19:49.286Z" }, - { url = "https://files.pythonhosted.org/packages/67/d1/43d3a0ac955a58601c24fa23038b1c55ee3a1ec02c0f96ebb1eae2bcf614/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b3e693d15533a45cd5906f0589f9c35090bef6ef45bf1e8195c424aa0ae06a8d", size = 3269850, upload-time = "2026-06-15T16:26:43.017Z" }, - { url = "https://files.pythonhosted.org/packages/94/df/de669c7054cd47c4439ac34b1b2ee8b804a794791fbb10720e997a2c87c7/sqlalchemy-2.0.51-cp313-cp313-win32.whl", hash = "sha256:b93ab07b5292dbe7e6b8da89475275e7042744283921344b56105f3eeb0f828b", size = 2117721, upload-time = "2026-06-15T16:23:12.36Z" }, - { url = "https://files.pythonhosted.org/packages/d0/8a/403c51d064196bae20a0bc2476577f83a3f8dd299719a97417086b7f2ec5/sqlalchemy-2.0.51-cp313-cp313-win_amd64.whl", hash = "sha256:0f053118c30e53161857a953e4de667d90e274980dccbe5dd3829bbbeece72a5", size = 2143615, upload-time = "2026-06-15T16:23:13.906Z" }, - { url = "https://files.pythonhosted.org/packages/b1/49/a739be2e1d02a96a658eb71ab45d921c874249252358ad24a5bffdd02525/sqlalchemy-2.0.51-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6ea306caaae6bd5afd0a46050003c88f6bf33227377a49298c498c3cb88ff491", size = 2158999, upload-time = "2026-06-15T16:08:51.759Z" }, - { url = "https://files.pythonhosted.org/packages/23/6b/2e0e38cf75c8780eca78d9b2e78164f8bcfd70125e5caa588ff5cbb9c9f4/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c45a496d6bc05dec41dcd4c3a2b183723f47473255c159cd80b503c8f246424d", size = 3282539, upload-time = "2026-06-15T16:19:51.065Z" }, - { url = "https://files.pythonhosted.org/packages/dd/a1/e77854cb5336fd37dc3c6ae3b71de242c98caac5725120be0b526b31cbd0/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4004ada0aafe8ae1991b2cd1d99c6d9146126e123bd6f883c260d974aa012e54", size = 3287545, upload-time = "2026-06-15T16:26:44.735Z" }, - { url = "https://files.pythonhosted.org/packages/f6/ab/9e17272fd4dac8df3b83c4fbe52b998a1c9d89a843c8c35ff29b74ff7364/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0f6bcad487aee1c638d707235682fc96f741de00663619881ab235400d03289e", size = 3230929, upload-time = "2026-06-15T16:19:52.625Z" }, - { url = "https://files.pythonhosted.org/packages/02/3c/52f408ea701781caee975606beccc48845f2aee8711ac29843d612c0306c/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:39a76529db6305693d8d4affa58ad5b5e2e18edd62daea628b29b97930b3513d", size = 3252888, upload-time = "2026-06-15T16:26:46.454Z" }, - { url = "https://files.pythonhosted.org/packages/24/16/3efd2ee6bc4ca4693a30a1dd17a91b606cae15d517d2a4746611d9b73ce8/sqlalchemy-2.0.51-cp314-cp314-win32.whl", hash = "sha256:08a204d8b5638717c26a24df18fcf40af45a6b22e35b70b1d62f0113c2e278e8", size = 2120551, upload-time = "2026-06-15T16:23:15.629Z" }, - { url = "https://files.pythonhosted.org/packages/7b/78/55b12e70f45bccc40d9e483925c065027b3b98ea4cbbdf6f8c2546feaf6c/sqlalchemy-2.0.51-cp314-cp314-win_amd64.whl", hash = "sha256:96747bfbadb055466e5b46d572618170046b45ce5a4879167f50d70a5319a499", size = 2146318, upload-time = "2026-06-15T16:23:17.108Z" }, - { url = "https://files.pythonhosted.org/packages/21/db/a9574ed40fed418924b1b1a3e54f47ee3963053b3d3d325a0d36b41f2c08/sqlalchemy-2.0.51-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e5ea1a213be1fcd5e49d9904c3b9939211ded90bc2a64e93f4c01963474285de", size = 2178920, upload-time = "2026-06-15T15:59:56.285Z" }, - { url = "https://files.pythonhosted.org/packages/bf/90/a1bb5c7cbba76b7bc1fbd586d0a5479a7bc9c27b4a8298f22ec9423b2bb3/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c6b36ed71f41942bdcd2ad2522be46bfce09d5705be5640ecf19bbc7660e4b7", size = 3566534, upload-time = "2026-06-15T15:58:35.024Z" }, - { url = "https://files.pythonhosted.org/packages/15/4b/481f1fed30e0e9e8dd24aecbb49f29eb57fe7657ece5cf06ee9b84bb97d8/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c2c62877097e1a0db401fba5cb4debee33265e5b2a55c4ccb489c02c53b4f72", size = 3535844, upload-time = "2026-06-15T16:02:43.973Z" }, - { url = "https://files.pythonhosted.org/packages/02/71/0aa64aeda645510af0a43f7d9ee70932f0d1dc4263aed34c50ee891d9df3/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0378d055e9e8cd6ce4d8dff683bdd3d7d413533c4ee51d67a2b1e0f9eacc0f23", size = 3475355, upload-time = "2026-06-15T15:58:36.592Z" }, - { url = "https://files.pythonhosted.org/packages/05/db/6061db32316446135a3abae5f308d144ab988a34234726042da3e58b1c63/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6e46fc36029eff666391e0531e5387b62ce6c4f1d8e50b3fb3099eaca1b42522", size = 3486591, upload-time = "2026-06-15T16:02:45.346Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c9/f14fdf71bb8957e0c7e39db69bbdf12b5c80f4ef775fdfa127bf4e0d6760/sqlalchemy-2.0.51-cp314-cp314t-win32.whl", hash = "sha256:9161cfc9efce70d1715f47d6ff40f79c6778c00d53be4fbc09d70301e4b83ba7", size = 2151313, upload-time = "2026-06-15T16:03:39.127Z" }, - { url = "https://files.pythonhosted.org/packages/6a/c6/673e618e6f4f297e126d9b56ea2f6478708f6c1af4e3223835c22e2c3697/sqlalchemy-2.0.51-cp314-cp314t-win_amd64.whl", hash = "sha256:159bb6ba32059f57ad7375a8f50d844dd2f19d14954ecf820cd33e20debd46b2", size = 2186280, upload-time = "2026-06-15T16:03:40.569Z" }, - { url = "https://files.pythonhosted.org/packages/e2/22/dbf013a12ec759e54a34a119e9e217435b3f71b2dd5c61a7ade0a25dae87/sqlalchemy-2.0.51-py3-none-any.whl", hash = "sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5", size = 1944334, upload-time = "2026-06-15T16:09:22.418Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/3b/21/77b4c147963073040dc3c3a5cb7a8c3001a1893c0209432cb77f9df836aa/sqlalchemy-2.0.52.tar.gz", hash = "sha256:5e2d46356ac2ccb7d268ab6c2319ac6a2b42f1b8d5fd8bd3d46855cd82abee97", size = 9945637, upload-time = "2026-08-11T19:07:09.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/18/e30c6fe1eca1bf34a39fbdd6066121cc9974c850faf6f349eac563697a26/sqlalchemy-2.0.52-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2eb3c6a64b1bfe6704777cfd504e7b8ad093a5f3e03ce67663a5e6742f294e43", size = 2167724, upload-time = "2026-08-11T20:58:12.679Z" }, + { url = "https://files.pythonhosted.org/packages/d0/56/2e17d161a4f7ecc1c2ffb93e607b4e1898bb551b451b283235acb8f6ce47/sqlalchemy-2.0.52-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:923bb183c1dc64fdf7b717965e3d59938ec4f8b8710b419a21ce403e5da9a9e1", size = 3321189, upload-time = "2026-08-11T21:02:41.932Z" }, + { url = "https://files.pythonhosted.org/packages/cf/b8/8490916e893f3f8d74dc9cc54c078619364999dee37047a188e73abbc852/sqlalchemy-2.0.52-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:651d6d8782e80679e6151707c7b490834d46ada526328895abf567f25e63d29c", size = 3338185, upload-time = "2026-08-11T21:17:02.597Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f7/752cc8ee453da222829b3f5c4613614bf750d97429363b70414fa10478e4/sqlalchemy-2.0.52-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b08cddb8989775e3c88799d86704bdfc3ee6e9846118201aa5997f16f27e3a15", size = 3271698, upload-time = "2026-08-11T21:02:43.963Z" }, + { url = "https://files.pythonhosted.org/packages/51/e6/074ade0c07b9e4c8e8bca46820320ed94df9702afdb6f2af06623068d2e6/sqlalchemy-2.0.52-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ab66fa9618269390d4dfa222f2f2f88f7bc4bf5da13905131b818217db7e8057", size = 3308936, upload-time = "2026-08-11T21:17:04.172Z" }, + { url = "https://files.pythonhosted.org/packages/66/07/557c0d04716705599227945ac14e0a17ad0338e899f37d8c2ddff4dcc663/sqlalchemy-2.0.52-cp313-cp313-win32.whl", hash = "sha256:c63bda077685c85ca513286547a531ba57e7a68cf0a7ed3bafcc2bbd18896f4d", size = 2127308, upload-time = "2026-08-11T21:14:53.879Z" }, + { url = "https://files.pythonhosted.org/packages/96/4e/226eda27654318ce525d043025221f689abef883da2c7126f9065121618c/sqlalchemy-2.0.52-cp313-cp313-win_amd64.whl", hash = "sha256:9876b09b9f1ce7398b0ffece585c0a911244c53191187341f6bcae640e133751", size = 2153876, upload-time = "2026-08-11T21:14:55.527Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f5/71cb30af58c9b80a4e1fac0b73bb48f86d497a774a6a2eb6d2f1e657bb73/sqlalchemy-2.0.52-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:410d52be41d17f1a236d19520fbe776257dc16516ed06bd16d433311842aefd9", size = 2169537, upload-time = "2026-08-11T20:58:13.855Z" }, + { url = "https://files.pythonhosted.org/packages/4c/93/d07ebd645d1b07b6b5ed63450a70f063a346a7e0f2c8810daf2e532400cb/sqlalchemy-2.0.52-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfe9ce533dbe4d0a2ae1486546619bd30b76bcd670539a44d910361376175f5e", size = 3319606, upload-time = "2026-08-11T21:02:45.829Z" }, + { url = "https://files.pythonhosted.org/packages/ae/5c/290c84c7c2566ecd3b65baaae0fddec9bc33b033b398a06123bb86fbfc6e/sqlalchemy-2.0.52-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:812bae5138bfc0aa46fb0686da0fc7f581f68e2bbb05bc24c3713bebaedd1437", size = 3323642, upload-time = "2026-08-11T21:17:05.675Z" }, + { url = "https://files.pythonhosted.org/packages/13/f5/2cc160590ca49173359557880b92a0572293ccb899e8f6cedf150c5a3ddf/sqlalchemy-2.0.52-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:50bff43b632a56fbf5ed9afdd76307e1512b62051bcd5afb341ae67205bbb6c8", size = 3268125, upload-time = "2026-08-11T21:02:47.649Z" }, + { url = "https://files.pythonhosted.org/packages/35/f3/ea8933fc9f7d1353e9c2ff9965eae687c4cef181120574591ed2fa0633e1/sqlalchemy-2.0.52-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:49565daf5af554f538e23aef1fc81a95a4e49658f152285e45c02f5fc44f04cd", size = 3289516, upload-time = "2026-08-11T21:17:07.267Z" }, + { url = "https://files.pythonhosted.org/packages/45/67/05cf86541c1e1716fca1e4a996954a439cd74501707cda607fb7cb02ef50/sqlalchemy-2.0.52-cp314-cp314-win32.whl", hash = "sha256:ab9da41e61b9979b910499d633b241df20c51ee5037e5405b11c2faac3cbe1a2", size = 2130249, upload-time = "2026-08-11T21:14:57.273Z" }, + { url = "https://files.pythonhosted.org/packages/96/d7/8ac6ffa1e36169e762ef65bd835046abb2251b1bc17f8f6708e14ed8d31f/sqlalchemy-2.0.52-cp314-cp314-win_amd64.whl", hash = "sha256:a593db51b3bae75db17a5738ad5f992244b3a03863f83c28117ee482c6a3f76d", size = 2156718, upload-time = "2026-08-11T21:14:58.667Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4b/e01a737eef378e734cc6394a82248a6ce13b167dfa36c731075ce9fc9c64/sqlalchemy-2.0.52-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1e61d08bdf4ee2f41024569e3400de7d6734ba498144766b11260936ccfa582", size = 2190344, upload-time = "2026-08-11T19:53:21.393Z" }, + { url = "https://files.pythonhosted.org/packages/b3/3f/3582293d1e185e71d19d7c731c3e2ee20ba21981c4a1115c0806c1f62120/sqlalchemy-2.0.52-py3-none-any.whl", hash = "sha256:3b81b8363a919ce53453591cdb93702e6bd54ade6c4fa2f468fc053baee5ed89", size = 1950700, upload-time = "2026-08-11T20:47:21.603Z" }, ] [[package]] @@ -2471,40 +2564,37 @@ wheels = [ ] [[package]] -name = "tqdm" -version = "4.67.3" +name = "truststore" +version = "0.10.4" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598, upload-time = "2026-02-03T17:35:53.048Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, ] [[package]] name = "ty" -version = "0.0.69" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8e/5b/7a618632dfe9373b7df572ecd7a08c8f799d772fbc317da82dd3aa363207/ty-0.0.69.tar.gz", hash = "sha256:b65106e9ff24fa76e25e1142fb09c85244e815c40450e3021d2bf652c231bb43", size = 6565094, upload-time = "2026-08-06T10:04:25.667Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/60/6534092f4d2c15e2491807edd609c2e50d527c1fed957acf40b9f110b64a/ty-0.0.69-py3-none-linux_armv6l.whl", hash = "sha256:98bfd383b273540829af673e7f98b9c1c4bcc8547d12a1a3806cd0bec7f0e087", size = 12364185, upload-time = "2026-08-06T10:03:47.137Z" }, - { url = "https://files.pythonhosted.org/packages/34/2b/5c29689bd4f74c2e3394d983d85e4011b629f2ce3730c9442553b8554bf8/ty-0.0.69-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:964621ddd05771660017c51b4e74078d861d9fc863c21ef2a500db1ab62c9ccf", size = 12042510, upload-time = "2026-08-06T10:03:49.481Z" }, - { url = "https://files.pythonhosted.org/packages/09/46/fa085bde4d23516d7ef14b24736fc5dd7dc498f60f52b3d077e59ffdea20/ty-0.0.69-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3ffea4048dd0da4c9c97393b4be0901098a9065b06fa81be2477cbde65d8a151", size = 11549397, upload-time = "2026-08-06T10:03:51.747Z" }, - { url = "https://files.pythonhosted.org/packages/25/cc/97b9efb2061dcab6fef1e94a4ad99df0bb45bd2cc15d4f5794c787ee0552/ty-0.0.69-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8684d4a70aadd1eab0f41bdba835e3288ef49db8402a8e6ca81bab52ed5d610", size = 12115567, upload-time = "2026-08-06T10:03:53.79Z" }, - { url = "https://files.pythonhosted.org/packages/e0/c1/a5e0404965093835f3e62544e661784ec0aa8ef0b006ed50af50b19c107e/ty-0.0.69-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:afaaba240ab4122e2069a796836d10be81b4ddb053ae268b3dff962a0b4ca5c7", size = 12149770, upload-time = "2026-08-06T10:03:55.993Z" }, - { url = "https://files.pythonhosted.org/packages/e2/39/8cad6b205a4abe8a044ca0c84aea71e8ccda29b07a75a5f090e310605580/ty-0.0.69-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11ea63ef07d4e33aeb1a775cf5f2c736b3ed22fa6f8b1b608591612c36795044", size = 12941278, upload-time = "2026-08-06T10:03:58.324Z" }, - { url = "https://files.pythonhosted.org/packages/d6/8b/8766d96b732c2a060d70dc8ccafcc4d6a54109a2a95f1deb0705de88892b/ty-0.0.69-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cb3730b1268e92a2907d7aea3afe8dd1b360ae65862f0557080cf479d481b424", size = 13426509, upload-time = "2026-08-06T10:04:00.621Z" }, - { url = "https://files.pythonhosted.org/packages/02/1f/e991b2cde953ea5b94d6a9a4c45c87937bd916bc09235f764407bf471c0a/ty-0.0.69-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a544ff57a752ef186ed40b5a2f44c17402af4cdefeb74a311ca02ebd57c4fca0", size = 13106582, upload-time = "2026-08-06T10:04:02.818Z" }, - { url = "https://files.pythonhosted.org/packages/ea/bb/73538f1b99e3558fd9db87b98698426f0f60fc8666da0b1efd0e70e275eb/ty-0.0.69-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87ed2cbca20caddfdf8e3e14d213ce91b67e75feed78900f4aaf3ef884954028", size = 12708931, upload-time = "2026-08-06T10:04:05.233Z" }, - { url = "https://files.pythonhosted.org/packages/87/cd/484a5208d74c4ad1155933906295ccdce9aa81a257d8df2ab9e41bd60133/ty-0.0.69-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2684efcbce5b6fe45045faf610b377b50781b6d2aa7e61ea23ecf5b3d2bce421", size = 12985322, upload-time = "2026-08-06T10:04:07.587Z" }, - { url = "https://files.pythonhosted.org/packages/6e/81/b75003f0d4da9ab3bc8fd4f4802f836cb9921ff7e70f460604f7b769a0b5/ty-0.0.69-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:da9aeb26fdac1d2214937542b59e0d4d1ba94ec7a3f45444f33c846de1eb1d63", size = 12063910, upload-time = "2026-08-06T10:04:09.835Z" }, - { url = "https://files.pythonhosted.org/packages/8a/76/088469f547ef63dceefc4a75826aedee5014f9371dc5171cde931896a82c/ty-0.0.69-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:00e7677cd14ede381f705f71104ea7b8ea0ce217a8634e19a89781953de0e9ad", size = 12166823, upload-time = "2026-08-06T10:04:12.114Z" }, - { url = "https://files.pythonhosted.org/packages/0a/c9/ce88a0bec0d46d8ae180b99c6ec014866fecc4cba1727b5feec8877b2765/ty-0.0.69-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d91965eb799649833d0d6042db09cd03d15289125245337cc46a2606effb7bda", size = 12483136, upload-time = "2026-08-06T10:04:14.33Z" }, - { url = "https://files.pythonhosted.org/packages/63/9e/6fae0ff225a0012642cf72c077e20f8f448c0a80771bc3360e8178fe2f32/ty-0.0.69-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1f03359cd8e5c412aa0c181118fa9b9061a4dddaedbb61bac0a424fb0814d402", size = 12799025, upload-time = "2026-08-06T10:04:16.445Z" }, - { url = "https://files.pythonhosted.org/packages/e4/43/78a658d18b2a4ccf35b053392f2213bf12e3c63b2abea512d3b6751d1f4c/ty-0.0.69-py3-none-win32.whl", hash = "sha256:ec460e01586b1eb91894c4a8403bee3e045a47e7a4ada943cc27ce8e348e88cf", size = 11787774, upload-time = "2026-08-06T10:04:18.622Z" }, - { url = "https://files.pythonhosted.org/packages/3a/5e/88db1f674403f2b81316a853a44a81ed220621fa96f8f7ae586fb6ca7513/ty-0.0.69-py3-none-win_amd64.whl", hash = "sha256:18976ca26a4e28fc3249477f79a695d5502e670803f2e080d89ac905baef3c6e", size = 12864038, upload-time = "2026-08-06T10:04:20.748Z" }, - { url = "https://files.pythonhosted.org/packages/4d/7b/6fc6efd00c69103d70f2bdbe824343089cd70b17b3079170057d3e5a3ac0/ty-0.0.69-py3-none-win_arm64.whl", hash = "sha256:7d4ca3bb74d91cb9947ba3f3b4cb131ad6a2b3ecc76d34040c4ec6092d2e411d", size = 12196693, upload-time = "2026-08-06T10:04:22.902Z" }, +version = "0.0.73" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/90/c4e1bb4cead3b644c3e258a27f9b05c7dc5eb0ec96a4f5282194edae9e0d/ty-0.0.73.tar.gz", hash = "sha256:823d4ce0d237bfc7eb6bcee70842f2c0706113813a16951077840743712f4b74", size = 6712739, upload-time = "2026-08-19T03:12:43.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/0f/f5e1801e55cc631f2db193276675b30561b963a2403da832bffb5d100267/ty-0.0.73-py3-none-linux_armv6l.whl", hash = "sha256:90a946082bf9bc446b5e72973d9f4ff1222a240b2ca4c9e6eed61eb913e30810", size = 12715452, upload-time = "2026-08-19T03:12:06.673Z" }, + { url = "https://files.pythonhosted.org/packages/54/32/515dd05074c213b433524ab97eb003b0132ae7e358e0d75633ba7a314ed8/ty-0.0.73-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b7d6b5c6a6db7ea95fbbc16af514ef44a27a29a2fe1dc798900790364d170209", size = 12301870, upload-time = "2026-08-19T03:12:08.924Z" }, + { url = "https://files.pythonhosted.org/packages/50/4d/085b4889f0d4bbe4af8b96242d4a1cb209fff95967cfa239ea141983719b/ty-0.0.73-py3-none-macosx_11_0_arm64.whl", hash = "sha256:dd6f657f463e01372d8688f235be164750c8db722c97da27fa4903aa8d40b203", size = 12111741, upload-time = "2026-08-19T03:12:11.067Z" }, + { url = "https://files.pythonhosted.org/packages/95/f6/d6ec277cadfecf03ad4c18551b67c4c6eb7807a0560d801db14be99d7a89/ty-0.0.73-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc2de468e33fd44c9ff1c43473a7316f4289480f5cba8995a67b6d22aee39ca9", size = 12196124, upload-time = "2026-08-19T03:12:13.14Z" }, + { url = "https://files.pythonhosted.org/packages/75/b7/ce78d8707563af9cae9bbd25328bfbc4931035085bd20089adf0c418f70e/ty-0.0.73-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2942fa0ef795a66034cdc8d75a72f453442f3b58ff2f69b4da05b7b954765b55", size = 12488557, upload-time = "2026-08-19T03:12:15.252Z" }, + { url = "https://files.pythonhosted.org/packages/d8/e8/329b9851b23502758c5c98e8cc875ea2a1b4c9674b4ca3a86da56a5063d3/ty-0.0.73-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e0f1ef14f642e18ac4e7a616a2796dcf7a5d82e28cd17f9796494acc7c4aabb", size = 13215606, upload-time = "2026-08-19T03:12:17.225Z" }, + { url = "https://files.pythonhosted.org/packages/36/38/67fedfd2cb77516ef0066b1642f487dba0eb3006493cf3475b15f5b8b228/ty-0.0.73-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16981e15fdceedb37d0aff76c5ac25914595dfee2675af95335550064251ad22", size = 13665497, upload-time = "2026-08-19T03:12:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/8e/b3/154f4dd48ec5eebc186ab4b822c6e62f982fc5ddfd262d6e3903c2acba44/ty-0.0.73-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:644b2bec8a2e2e4957a942ae81d6cff5571c489bb5a8675e4d3886de537a694d", size = 13351231, upload-time = "2026-08-19T03:12:21.353Z" }, + { url = "https://files.pythonhosted.org/packages/35/5f/d462496903fbe453fb76363f8478be929c8e6ff21e6928c57dcd7e5fa21f/ty-0.0.73-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338d565be3186f50ff8e9d10483685549c2d23f0754485d5ede3b54f4319188a", size = 12782586, upload-time = "2026-08-19T03:12:23.667Z" }, + { url = "https://files.pythonhosted.org/packages/87/52/ec6d24b74abe3ec324204c1c71e6d0c6c76a17ffc15fd51d603b0a302abe/ty-0.0.73-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:11c7b6d839309d2c102cb3a4c03d817176bbfab5b2fccc95a75ec5c9597421c9", size = 13247134, upload-time = "2026-08-19T03:12:25.956Z" }, + { url = "https://files.pythonhosted.org/packages/26/20/cc74650fec56a54786c6d7c89e09576fcad3092be34cf21715d39a406a9b/ty-0.0.73-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:488572db7ff97fb50ea36a76250f2d617c9727d143da6c7bf0623276eb0fc507", size = 12309344, upload-time = "2026-08-19T03:12:28.122Z" }, + { url = "https://files.pythonhosted.org/packages/89/bd/4b0a9087f4315d7fbadf77a3ce44c816cc9ffabed1ced06cc5be81fbc414/ty-0.0.73-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1b958ebceefbbf594e59eb8d3d55bbd033ce634026fcba3e4bc3179e78e45bb7", size = 12502319, upload-time = "2026-08-19T03:12:30.128Z" }, + { url = "https://files.pythonhosted.org/packages/11/80/0a925074911fe111912ea29d9eed309bcc183f43d2fb3eef07db056a0beb/ty-0.0.73-py3-none-musllinux_1_2_i686.whl", hash = "sha256:91a32993b3c34e42c3f323ad6c0399cb596bd1c27e9b7f20db7cd64c1067b68e", size = 12753688, upload-time = "2026-08-19T03:12:32.433Z" }, + { url = "https://files.pythonhosted.org/packages/24/6b/aeccaf89efbc2e112bd415340a22e2669ec998aa397242503e747b712ca4/ty-0.0.73-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bab8a19fbf51f479bddb2a12c5fabfe52f918a5590362321ed5d89b44eb62c15", size = 13069050, upload-time = "2026-08-19T03:12:35.398Z" }, + { url = "https://files.pythonhosted.org/packages/d7/3e/eae485fd86c1585943fd4e1746b0757b2da01e2c43136ebe8c686fe1c7f1/ty-0.0.73-py3-none-win32.whl", hash = "sha256:03347a612f0fa020b19bfd8dbd521db6ecc75d377a3e4d4f6e6c2e62871da4cc", size = 12053187, upload-time = "2026-08-19T03:12:37.565Z" }, + { url = "https://files.pythonhosted.org/packages/a7/01/9b8b983786e3ce34924e372e8b76b92b508273ab65c589fc7e88cc03ee17/ty-0.0.73-py3-none-win_amd64.whl", hash = "sha256:cedd05122ded0b5dcc55431a370e974b747f99c41c290a3d2ab8c1867f197519", size = 12693838, upload-time = "2026-08-19T03:12:39.483Z" }, + { url = "https://files.pythonhosted.org/packages/ea/88/25333bbfea6a5dc064371d2002d3d4807db90b84d5448f9106b2712b0fbc/ty-0.0.73-py3-none-win_arm64.whl", hash = "sha256:e47068f8369dea5d641a26a2ad0a947a320b02ff87099b07e95de0323245a4dc", size = 12443573, upload-time = "2026-08-19T03:12:41.449Z" }, ] [[package]] @@ -2563,15 +2653,15 @@ wheels = [ [[package]] name = "uvicorn" -version = "0.52.1" +version = "0.52.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "h11" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/03/18/ccce41535dee1be77735592bd19965f3972c82e07ee703d324709496b716/uvicorn-0.52.1.tar.gz", hash = "sha256:112ec661814189acbccd3f7b86460147cc065fc92c0821afa78918780e4354dd", size = 100571, upload-time = "2026-08-01T18:19:30.732Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/0f/3f86e61397dd33bf2ccf28188c40db6a740658aeebbbf6e7dbc101a1f487/uvicorn-0.52.4.tar.gz", hash = "sha256:73acfee47a0b133c5de13d219492d62d8a31e935f4fe6e41a232451a15379f86", size = 100627, upload-time = "2026-08-19T06:27:41.821Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/d5/68e6e9bca63c0badf67002890a46d3784c958de45b65e1275ec583ca1f06/uvicorn-0.52.1-py3-none-any.whl", hash = "sha256:e4403f9d93188cf9d1088e9f40e3acd12630e2df8675316704379a7fc20fff6a", size = 79859, upload-time = "2026-08-01T18:19:29.294Z" }, + { url = "https://files.pythonhosted.org/packages/f1/79/4a20b54ab0491485ccd8c077db2d39187c7f12b3e15485d38a7be37c81b4/uvicorn-0.52.4-py3-none-any.whl", hash = "sha256:f86e41a149d7d05a9969337e3946a9c171c06a5d42680896daaba624aeac8da1", size = 79871, upload-time = "2026-08-19T06:27:40.36Z" }, ] [package.optional-dependencies]